diff options
Diffstat (limited to 'lib')
459 files changed, 8733 insertions, 3580 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 0d2d74b43f..56d60a29da 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -232,6 +232,50 @@ remove dependent packages *before* removing their dependencies or use the .. _nondownloadable: +^^^^^^^^^^^^^^^^^^ +Garbage collection +^^^^^^^^^^^^^^^^^^ + +When Spack builds software from sources, if often installs tools that are needed +just to build or test other software. These are not necessary at runtime. +To support cases where removing these tools can be a benefit Spack provides +the ``spack gc`` ("garbage collector") command, which will uninstall all unneeded packages: + +.. code-block:: console + + $ spack find + ==> 24 installed packages + -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ---------------------- + autoconf@2.69 findutils@4.6.0 libiconv@1.16 libszip@2.1.1 m4@1.4.18 openjpeg@2.3.1 pkgconf@1.6.3 util-macros@1.19.1 + automake@1.16.1 gdbm@1.18.1 libpciaccess@0.13.5 libtool@2.4.6 mpich@3.3.2 openssl@1.1.1d readline@8.0 xz@5.2.4 + cmake@3.16.1 hdf5@1.10.5 libsigsegv@2.12 libxml2@2.9.9 ncurses@6.1 perl@5.30.0 texinfo@6.5 zlib@1.2.11 + + $ spack gc + ==> The following packages will be uninstalled: + + -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ---------------------- + vn47edz autoconf@2.69 6m3f2qn findutils@4.6.0 ubl6bgk libtool@2.4.6 pksawhz openssl@1.1.1d urdw22a readline@8.0 + ki6nfw5 automake@1.16.1 fklde6b gdbm@1.18.1 b6pswuo m4@1.4.18 k3s2csy perl@5.30.0 lp5ya3t texinfo@6.5 + ylvgsov cmake@3.16.1 5omotir libsigsegv@2.12 leuzbbh ncurses@6.1 5vmfbrq pkgconf@1.6.3 5bmv4tg util-macros@1.19.1 + + ==> Do you want to proceed? [y/N] y + + [ ... ] + + $ spack find + ==> 9 installed packages + -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ---------------------- + hdf5@1.10.5 libiconv@1.16 libpciaccess@0.13.5 libszip@2.1.1 libxml2@2.9.9 mpich@3.3.2 openjpeg@2.3.1 xz@5.2.4 zlib@1.2.11 + +In the example above Spack went through all the packages in the DB +and removed everything that is not either: + +1. A package installed upon explicit request of the user +2. A ``link`` or ``run`` dependency, even transitive, of one of the packages at point 1. + +You can check :ref:`cmd-spack-find-metadata` to see how to query for explicitly installed packages +or :ref:`dependency-types` for a more thorough treatment of dependency types. + ^^^^^^^^^^^^^^^^^^^^^^^^^ Non-Downloadable Tarballs ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -414,6 +458,8 @@ Packages are divided into groups according to their architecture and compiler. Within each group, Spack tries to keep the view simple, and only shows the version of installed packages. +.. _cmd-spack-find-metadata: + """""""""""""""""""""""""""""""" Viewing more metadata """""""""""""""""""""""""""""""" @@ -883,11 +929,13 @@ in GNU Autotools. If all flags are set, the order is Compiler environment variables and additional RPATHs ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In the exceptional case a compiler requires setting special environment -variables, like an explicit library load path. These can bet set in an -extra section in the compiler configuration (the supported environment -modification commands are: ``set``, ``unset``, ``append-path``, and -``prepend-path``). The user can also specify additional ``RPATHs`` that the +Sometimes compilers require setting special environment variables to +operate correctly. Spack handles these cases by allowing custom environment +modifications in the ``environment`` attribute of the compiler configuration +section. See also the :ref:`configuration_environment_variables` section +of the configuration files docs for more information. + +It is also possible to specify additional ``RPATHs`` that the compiler will add to all executables generated by that compiler. This is useful for forcing certain compilers to RPATH their own runtime libraries, so that executables will run without the need to set ``LD_LIBRARY_PATH``. @@ -904,28 +952,19 @@ that executables will run without the need to set ``LD_LIBRARY_PATH``. fc: /opt/gcc/bin/gfortran environment: unset: - BAD_VARIABLE: # The colon is required but the value must be empty + - BAD_VARIABLE set: GOOD_VARIABLE_NUM: 1 GOOD_VARIABLE_STR: good - prepend-path: + prepend_path: PATH: /path/to/binutils - append-path: + append_path: LD_LIBRARY_PATH: /opt/gcc/lib extra_rpaths: - /path/to/some/compiler/runtime/directory - /path/to/some/other/compiler/runtime/directory -.. note:: - - The section `environment` is interpreted as an ordered dictionary, which - means two things. First, environment modification are applied in the order - they are specified in the configuration file. Second, you cannot express - environment modifications that require mixing different commands, i.e. you - cannot `set` one variable, than `prepend-path` to another one, and than - again `set` a third one. - ^^^^^^^^^^^^^^^^^^^^^^^ Architecture specifiers ^^^^^^^^^^^^^^^^^^^^^^^ @@ -1271,10 +1310,9 @@ directly when you run ``python``: Using Extensions ^^^^^^^^^^^^^^^^ -There are three ways to get ``numpy`` working in Python. The first is -to use :ref:`shell-support`. You can simply ``load`` the -module for the extension, and it will be added to the ``PYTHONPATH`` -in your current shell: +There are four ways to get ``numpy`` working in Python. The first is +to use :ref:`shell-support`. You can simply ``load`` the extension, +and it will be added to the ``PYTHONPATH`` in your current shell: .. code-block:: console @@ -1284,11 +1322,29 @@ in your current shell: Now ``import numpy`` will succeed for as long as you keep your current session open. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Loading Extensions via Modules +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Instead of using Spack's environment modification capabilities through +the ``spack load`` command, you can load numpy through your +environment modules (using ``environment-modules`` or ``lmod``). This +will also add the extension to the ``PYTHONPATH`` in your current +shell. + +.. code-block:: console + + $ module load <name of numpy module> + +If you do not know the name of the specific numpy module you wish to +load, you can use the ``spack module tcl|lmod loads`` command to get +the name of the module from the Spack spec. + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Activating Extensions in a View ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The second way to use extensions is to create a view, which merges the +Another way to use extensions is to create a view, which merges the python installation along with the extensions into a single prefix. See :ref:`filesystem-views` for a more in-depth description of views and :ref:`cmd-spack-view` for usage of the ``spack view`` command. diff --git a/lib/spack/docs/binary_caches.rst b/lib/spack/docs/binary_caches.rst index a54c0541c2..614ad2864c 100644 --- a/lib/spack/docs/binary_caches.rst +++ b/lib/spack/docs/binary_caches.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_settings.rst b/lib/spack/docs/build_settings.rst index fbd73d7a52..45f54084ae 100644 --- a/lib/spack/docs/build_settings.rst +++ b/lib/spack/docs/build_settings.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems.rst b/lib/spack/docs/build_systems.rst index 93b8e5c7a8..a68dd99911 100644 --- a/lib/spack/docs/build_systems.rst +++ b/lib/spack/docs/build_systems.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -56,6 +56,7 @@ on these ideas for each distinct build system that Spack supports: :maxdepth: 1 :caption: Other + build_systems/bundlepackage build_systems/cudapackage build_systems/intelpackage build_systems/custompackage diff --git a/lib/spack/docs/build_systems/autotoolspackage.rst b/lib/spack/docs/build_systems/autotoolspackage.rst index 4b6ea68726..a2baedd576 100644 --- a/lib/spack/docs/build_systems/autotoolspackage.rst +++ b/lib/spack/docs/build_systems/autotoolspackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/bundlepackage.rst b/lib/spack/docs/build_systems/bundlepackage.rst new file mode 100644 index 0000000000..e22e4b26be --- /dev/null +++ b/lib/spack/docs/build_systems/bundlepackage.rst @@ -0,0 +1,52 @@ +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _bundlepackage: + +------------- +BundlePackage +------------- + +``BundlePackage`` represents a set of packages that are expected to work well +together, such as a collection of commonly used software libraries. The +associated software is specified as bundle dependencies. + + +^^^^^^^^ +Creation +^^^^^^^^ + +Be sure to specify the ``bundle`` template if you are using ``spack create`` +to generate a package from the template. For example, use the following +command to create a bundle package whose class name will be ``Mybundle``: + +.. code-block:: console + + $ spack create --template bundle --name mybundle + + + +^^^^^^ +Phases +^^^^^^ + +The ``BundlePackage`` base class does not provide any phases by default +since the bundle does not represent a build system. + + +^^^ +URL +^^^ + +The ``url`` property does not have meaning since there is no package-specific +code to fetch. + + +^^^^^^^ +Version +^^^^^^^ + +At least one ``version`` must be specified in order for the package to +build. diff --git a/lib/spack/docs/build_systems/cmakepackage.rst b/lib/spack/docs/build_systems/cmakepackage.rst index 7e246b11a1..0a771edad3 100644 --- a/lib/spack/docs/build_systems/cmakepackage.rst +++ b/lib/spack/docs/build_systems/cmakepackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/cudapackage.rst b/lib/spack/docs/build_systems/cudapackage.rst index d2097ac5ec..b4561c69ed 100644 --- a/lib/spack/docs/build_systems/cudapackage.rst +++ b/lib/spack/docs/build_systems/cudapackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/custompackage.rst b/lib/spack/docs/build_systems/custompackage.rst index 611ead22d4..8a20dada92 100644 --- a/lib/spack/docs/build_systems/custompackage.rst +++ b/lib/spack/docs/build_systems/custompackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/intelpackage.rst b/lib/spack/docs/build_systems/intelpackage.rst index f7d975899e..c14fc34f51 100644 --- a/lib/spack/docs/build_systems/intelpackage.rst +++ b/lib/spack/docs/build_systems/intelpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/makefilepackage.rst b/lib/spack/docs/build_systems/makefilepackage.rst index 41a66a8137..406ec8ce89 100644 --- a/lib/spack/docs/build_systems/makefilepackage.rst +++ b/lib/spack/docs/build_systems/makefilepackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/mesonpackage.rst b/lib/spack/docs/build_systems/mesonpackage.rst index 22fc8d3e11..75b0e1caae 100644 --- a/lib/spack/docs/build_systems/mesonpackage.rst +++ b/lib/spack/docs/build_systems/mesonpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/octavepackage.rst b/lib/spack/docs/build_systems/octavepackage.rst index 2405dbb387..90e582e36c 100644 --- a/lib/spack/docs/build_systems/octavepackage.rst +++ b/lib/spack/docs/build_systems/octavepackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/perlpackage.rst b/lib/spack/docs/build_systems/perlpackage.rst index 4eafe1248e..11968e1560 100644 --- a/lib/spack/docs/build_systems/perlpackage.rst +++ b/lib/spack/docs/build_systems/perlpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/pythonpackage.rst b/lib/spack/docs/build_systems/pythonpackage.rst index 595f905e4a..3021f5b6dc 100644 --- a/lib/spack/docs/build_systems/pythonpackage.rst +++ b/lib/spack/docs/build_systems/pythonpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/qmakepackage.rst b/lib/spack/docs/build_systems/qmakepackage.rst index 051ff1fe0b..81d0f7f798 100644 --- a/lib/spack/docs/build_systems/qmakepackage.rst +++ b/lib/spack/docs/build_systems/qmakepackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/rpackage.rst b/lib/spack/docs/build_systems/rpackage.rst index 5533792541..60c76816c5 100644 --- a/lib/spack/docs/build_systems/rpackage.rst +++ b/lib/spack/docs/build_systems/rpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/rubypackage.rst b/lib/spack/docs/build_systems/rubypackage.rst index 44667257d3..6310507809 100644 --- a/lib/spack/docs/build_systems/rubypackage.rst +++ b/lib/spack/docs/build_systems/rubypackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/sconspackage.rst b/lib/spack/docs/build_systems/sconspackage.rst index fa0c190e6d..f29d6c5f45 100644 --- a/lib/spack/docs/build_systems/sconspackage.rst +++ b/lib/spack/docs/build_systems/sconspackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/sippackage.rst b/lib/spack/docs/build_systems/sippackage.rst index 1f0fb7b426..b8c08ec513 100644 --- a/lib/spack/docs/build_systems/sippackage.rst +++ b/lib/spack/docs/build_systems/sippackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/build_systems/wafpackage.rst b/lib/spack/docs/build_systems/wafpackage.rst index 9876106468..1916630f4c 100644 --- a/lib/spack/docs/build_systems/wafpackage.rst +++ b/lib/spack/docs/build_systems/wafpackage.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/chain.rst b/lib/spack/docs/chain.rst index ca6d0d4779..d84db87ff3 100644 --- a/lib/spack/docs/chain.rst +++ b/lib/spack/docs/chain.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index e5305a9317..e19dd05511 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -176,7 +176,25 @@ exclude_patterns = ['_build', '_spack_root', '.spack-env'] #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +# We use our own extension of the default style with a few modifications +from pygments.style import Style +from pygments.styles.default import DefaultStyle +from pygments.token import Generic, Comment, Text + +class SpackStyle(DefaultStyle): + styles = DefaultStyle.styles.copy() + background_color = "#f4f4f8" + styles[Generic.Output] = "#355" + styles[Generic.Prompt] = "bold #346ec9" + +import pkg_resources +dist = pkg_resources.Distribution(__file__) +sys.path.append('.') # make 'conf' module findable +ep = pkg_resources.EntryPoint.parse('spack = conf:SpackStyle', dist=dist) +dist._ep_map = {'pygments.styles': {'plugin1': ep}} +pkg_resources.working_set.add(dist) + +pygments_style = 'spack' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index d8d07b505a..865845b274 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -30,11 +30,21 @@ Default is ``$spack/opt/spack``. ``install_hash_length`` and ``install_path_scheme`` --------------------------------------------------- -The default Spack installation path can be very long and can create -problems for scripts with hardcoded shebangs. There are two parameters -to help with that. Firstly, the ``install_hash_length`` parameter can -set the length of the hash in the installation path from 1 to 32. The -default path uses the full 32 characters. +The default Spack installation path can be very long and can create problems +for scripts with hardcoded shebangs. Additionally, when using the Intel +compiler, and if there is also a long list of dependencies, the compiler may +segfault. If you see the following: + + .. code-block:: console + + : internal error: ** The compiler has encountered an unexpected problem. + ** Segmentation violation signal raised. ** + Access violation or stack overflow. Please contact Intel Support for assistance. + +it may be because variables containing dependency specs may be too long. There +are two parameters to help with long path names. Firstly, the +``install_hash_length`` parameter can set the length of the hash in the +installation path from 1 to 32. The default path uses the full 32 characters. Secondly, it is also possible to modify the entire installation scheme. By default Spack uses diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 7721c07a8e..41c9de79f9 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -427,6 +427,33 @@ home directory, and ``~user`` will expand to a specified user's home directory. The ``~`` must appear at the beginning of the path, or Spack will not expand it. +.. _configuration_environment_variables: + +------------------------- +Environment Modifications +------------------------- + +Spack allows to prescribe custom environment modifications in a few places +within its configuration files. Every time these modifications are allowed +they are specified as a dictionary, like in the following example: + +.. code-block:: yaml + + environment: + set: + LICENSE_FILE: '/path/to/license' + unset: + - CPATH + - LIBRARY_PATH + append_path: + PATH: '/new/bin/dir' + +The possible actions that are permitted are ``set``, ``unset``, ``append_path``, +``prepend_path`` and finally ``remove_path``. They all require a dictionary +of variable names mapped to the values used for the modification. +The only exception is ``unset`` that requires just a list of variable names. +No particular order is ensured on the execution of each of these modifications. + ---------------------------- Seeing Spack's Configuration ---------------------------- diff --git a/lib/spack/docs/containers.rst b/lib/spack/docs/containers.rst new file mode 100644 index 0000000000..bbb21a2e00 --- /dev/null +++ b/lib/spack/docs/containers.rst @@ -0,0 +1,307 @@ +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _containers: + +================ +Container Images +================ + +Spack can be an ideal tool to setup images for containers since all the +features discussed in :ref:`environments` can greatly help to manage +the installation of software during the image build process. Nonetheless, +building a production image from scratch still requires a lot of +boilerplate to: + +- Get Spack working within the image, possibly running as root +- Minimize the physical size of the software installed +- Properly update the system software in the base image + +To facilitate users with these tedious tasks, Spack provides a command +to automatically generate recipes for container images based on +Environments: + +.. code-block:: console + + $ ls + spack.yaml + + $ spack containerize + # Build stage with Spack pre-installed and ready to be used + FROM spack/centos7:latest as builder + + # What we want to install and how we want to install it + # is specified in a manifest file (spack.yaml) + RUN mkdir /opt/spack-environment \ + && (echo "spack:" \ + && echo " specs:" \ + && echo " - gromacs+mpi" \ + && echo " - mpich" \ + && echo " concretization: together" \ + && echo " config:" \ + && echo " install_tree: /opt/software" \ + && echo " view: /opt/view") > /opt/spack-environment/spack.yaml + + # Install the software, remove unecessary deps + RUN cd /opt/spack-environment && spack install && spack gc -y + + # Strip all the binaries + RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s + + # Modifications to the environment that are necessary to run + RUN cd /opt/spack-environment && \ + spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh + + + # Bare OS image to run the installed executables + FROM centos:7 + + COPY --from=builder /opt/spack-environment /opt/spack-environment + COPY --from=builder /opt/software /opt/software + COPY --from=builder /opt/view /opt/view + COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh + + RUN yum update -y && yum install -y epel-release && yum update -y \ + && yum install -y libgomp \ + && rm -rf /var/cache/yum && yum clean all + + RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc + + + LABEL "app"="gromacs" + LABEL "mpi"="mpich" + + ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] + + +The bits that make this automation possible are discussed in details +below. All the images generated in this way will be based on +multi-stage builds with: + +- A fat ``build`` stage containing common build tools and Spack itself +- A minimal ``final`` stage containing only the software requested by the user + +----------------- +Spack Base Images +----------------- + +Docker images with Spack preinstalled and ready to be used are +built on `Docker Hub <https://hub.docker.com/u/spack>`_ +at every push to ``develop`` or to a release branch. The OS that +are currently supported are summarized in the table below: + +.. _containers-supported-os: + +.. list-table:: Supported operating systems + :header-rows: 1 + + * - Operating System + - Base Image + - Spack Image + * - Ubuntu 16.04 + - ``ubuntu:16.04`` + - ``spack/ubuntu-xenial`` + * - Ubuntu 18.04 + - ``ubuntu:16.04`` + - ``spack/ubuntu-bionic`` + * - CentOS 6 + - ``centos:6`` + - ``spack/centos6`` + * - CentOS 7 + - ``centos:7`` + - ``spack/centos7`` + +All the images are tagged with the corresponding release of Spack: + +.. image:: dockerhub_spack.png + +with the exception of the ``latest`` tag that points to the HEAD +of the ``develop`` branch. These images are available for anyone +to use and take care of all the repetitive tasks that are necessary +to setup Spack within a container. All the container recipes generated +automatically by Spack use them as base images for their ``build`` stage. + + +------------------------- +Environment Configuration +------------------------- + +Any Spack Environment can be used for the automatic generation of container +recipes. Sensible defaults are provided for things like the base image or the +version of Spack used in the image. If a finer tuning is needed it can be +obtained by adding the relevant metadata under the ``container`` attribute +of environments: + +.. code-block:: yaml + + spack: + specs: + - gromacs+mpi + - mpich + + container: + # Select the format of the recipe e.g. docker, + # singularity or anything else that is currently supported + format: docker + + # Select from a valid list of images + base: + image: "centos:7" + spack: develop + + # Whether or not to strip binaries + strip: true + + # Additional system packages that are needed at runtime + os_packages: + - libgomp + + # Extra instructions + extra_instructions: + final: | + RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc + + # Labels for the image + labels: + app: "gromacs" + mpi: "mpich" + +The tables below describe the configuration options that are currently supported: + +.. list-table:: General configuration options for the ``container`` section of ``spack.yaml`` + :header-rows: 1 + + * - Option Name + - Description + - Allowed Values + - Required + * - ``format`` + - The format of the recipe + - ``docker`` or ``singularity`` + - Yes + * - ``base:image`` + - Base image for ``final`` stage + - See :ref:`containers-supported-os` + - Yes + * - ``base:spack`` + - Version of Spack + - Valid tags for ``base:image`` + - Yes + * - ``strip`` + - Whether to strip binaries + - ``true`` (default) or ``false`` + - No + * - ``os_packages`` + - System packages to be installed + - Valid packages for the ``final`` OS + - No + * - ``extra_instructions:build`` + - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``build`` stage + - Anything understood by the current ``format`` + - No + * - ``extra_instructions:final`` + - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``final`` stage + - Anything understood by the current ``format`` + - No + * - ``labels`` + - Labels to tag the image + - Pairs of key-value strings + - No + +.. list-table:: Configuration options specific to Singularity + :header-rows: 1 + + * - Option Name + - Description + - Allowed Values + - Required + * - ``singularity:runscript`` + - Content of ``%runscript`` + - Any valid script + - No + * - ``singularity:startscript`` + - Content of ``%startscript`` + - Any valid script + - No + * - ``singularity:test`` + - Content of ``%test`` + - Any valid script + - No + * - ``singularity:help`` + - Description of the image + - Description string + - No + +Once the Environment is properly configured a recipe for a container +image can be printed to standard output by issuing the following +command from the directory where the ``spack.yaml`` resides: + +.. code-block:: console + + $ spack containerize + +The example ``spack.yaml`` above would produce for instance the +following ``Dockerfile``: + +.. code-block:: docker + + # Build stage with Spack pre-installed and ready to be used + FROM spack/centos7:latest as builder + + # What we want to install and how we want to install it + # is specified in a manifest file (spack.yaml) + RUN mkdir /opt/spack-environment \ + && (echo "spack:" \ + && echo " specs:" \ + && echo " - gromacs+mpi" \ + && echo " - mpich" \ + && echo " concretization: together" \ + && echo " config:" \ + && echo " install_tree: /opt/software" \ + && echo " view: /opt/view") > /opt/spack-environment/spack.yaml + + # Install the software, remove unecessary deps + RUN cd /opt/spack-environment && spack install && spack gc -y + + # Strip all the binaries + RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s + + # Modifications to the environment that are necessary to run + RUN cd /opt/spack-environment && \ + spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh + + + # Bare OS image to run the installed executables + FROM centos:7 + + COPY --from=builder /opt/spack-environment /opt/spack-environment + COPY --from=builder /opt/software /opt/software + COPY --from=builder /opt/view /opt/view + COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh + + RUN yum update -y && yum install -y epel-release && yum update -y \ + && yum install -y libgomp \ + && rm -rf /var/cache/yum && yum clean all + + RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc + + + LABEL "app"="gromacs" + LABEL "mpi"="mpich" + + ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] + +.. note:: + Spack can also produce Singularity definition files to build the image. The + minimum version of Singularity required to build a SIF (Singularity Image Format) + from them is ``3.5.3``.
\ No newline at end of file diff --git a/lib/spack/docs/contribution_guide.rst b/lib/spack/docs/contribution_guide.rst index 73f25c6f03..0b79141ee3 100644 --- a/lib/spack/docs/contribution_guide.rst +++ b/lib/spack/docs/contribution_guide.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -64,6 +64,8 @@ If you take a look in ``$SPACK_ROOT/.travis.yml``, you'll notice that we test against Python 2.6, 2.7, and 3.4-3.7 on both macOS and Linux. We currently perform 3 types of tests: +.. _cmd-spack-test: + ^^^^^^^^^^ Unit Tests ^^^^^^^^^^ @@ -86,40 +88,83 @@ To run *all* of the unit tests, use: $ spack test -These tests may take several minutes to complete. If you know you are only -modifying a single Spack feature, you can run a single unit test at a time: +These tests may take several minutes to complete. If you know you are +only modifying a single Spack feature, you can run subsets of tests at a +time. For example, this would run all the tests in +``lib/spack/spack/test/architecture.py``: + +.. code-block:: console + + $ spack test architecture.py + +And this would run the ``test_platform`` test from that file: .. code-block:: console - $ spack test architecture + $ spack test architecture.py::test_platform -This allows you to develop iteratively: make a change, test that change, make -another change, test that change, etc. To get a list of all available unit -tests, run: +This allows you to develop iteratively: make a change, test that change, +make another change, test that change, etc. We use `pytest +<http://pytest.org/>`_ as our tests fromework, and these types of +arguments are just passed to the ``pytest`` command underneath. See `the +pytest docs +<http://doc.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests>`_ +for more details on test selection syntax. + +``spack test`` has a few special options that can help you understand +what tests are available. To get a list of all available unit test +files, run: .. command-output:: spack test --list + :ellipsis: 5 + +To see a more detailed list of available unit tests, use ``spack test +--list-long``: + +.. command-output:: spack test --list-long + :ellipsis: 10 + +And to see the fully qualified names of all tests, use ``--list-names``: -A more detailed list of available unit tests can be found by running -``spack test --long-list``. +.. command-output:: spack test --list-names + :ellipsis: 5 -By default, ``pytest`` captures the output of all unit tests. If you add print -statements to a unit test and want to see the output, simply run: +You can combine these with ``pytest`` arguments to restrict which tests +you want to know about. For example, to see just the tests in +``architecture.py``: + +.. command-output:: spack test --list-long architecture.py + +You can also combine any of these options with a ``pytest`` keyword +search. For example, to see the names of all tests that have "spec" +or "concretize" somewhere in their names: + +.. command-output:: spack test --list-names -k "spec and concretize" + +By default, ``pytest`` captures the output of all unit tests, and it will +print any captured output for failed tests. Sometimes it's helpful to see +your output interactively, while the tests run (e.g., if you add print +statements to a unit tests). To see the output *live*, use the ``-s`` +argument to ``pytest``: .. code-block:: console - $ spack test -s -k architecture + $ spack test -s architecture.py::test_platform -Unit tests are crucial to making sure bugs aren't introduced into Spack. If you -are modifying core Spack libraries or adding new functionality, please consider -adding new unit tests or strengthening existing tests. +Unit tests are crucial to making sure bugs aren't introduced into +Spack. If you are modifying core Spack libraries or adding new +functionality, please add new unit tests for your feature, and consider +strengthening existing tests. You will likely be asked to do this if you +submit a pull request to the Spack project on GitHub. Check out the +`pytest docs <http://pytest.org/>`_ and feel free to ask for guidance on +how to write tests! .. note:: - There is also a ``run-unit-tests`` script in ``share/spack/qa`` that - runs the unit tests. Afterwards, it reports back to Codecov with the - percentage of Spack that is covered by unit tests. This script is - designed for Travis CI. If you want to run the unit tests yourself, we - suggest you use ``spack test``. + You may notice the ``share/spack/qa/run-unit-tests`` script in the + repository. This script is designed for Travis CI. It runs the unit + tests and reports coverage statistics back to Codecov. If you want to + run the unit tests yourself, we suggest you use ``spack test``. ^^^^^^^^^^^^ Flake8 Tests @@ -223,8 +268,7 @@ documentation. In order to prevent things like broken links and missing imports, we added documentation tests that build the documentation and fail if there are any warning or error messages. -Building the documentation requires several dependencies, all of which can be -installed with Spack: +Building the documentation requires several dependencies: * sphinx * sphinxcontrib-programoutput @@ -234,11 +278,18 @@ installed with Spack: * mercurial * subversion +All of these can be installed with Spack, e.g. + +.. code-block:: console + + $ spack install py-sphinx py-sphinxcontrib-programoutput py-sphinx-rtd-theme graphviz git mercurial subversion + .. warning:: Sphinx has `several required dependencies <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/py-sphinx/package.py>`_. - If you installed ``py-sphinx`` with Spack, make sure to add all of these - dependencies to your ``PYTHONPATH``. The easiest way to do this is to run: + If you're using a ``python`` from Spack and you installed + ``py-sphinx`` and friends, you need to make them available to your + ``python``. The easiest way to do this is to run: .. code-block:: console @@ -246,8 +297,10 @@ installed with Spack: $ spack activate py-sphinx-rtd-theme $ spack activate py-sphinxcontrib-programoutput - so that all of the dependencies are symlinked to a central location. - If you see an error message like: + so that all of the dependencies are symlinked into that Python's + tree. Alternatively, you could arrange for their library + directories to be added to PYTHONPATH. If you see an error message + like: .. code-block:: console diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index d25b40e45a..de2fe80f85 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -363,12 +363,12 @@ Developer commands ``spack doc`` ^^^^^^^^^^^^^ -.. _cmd-spack-test: - ^^^^^^^^^^^^^^ ``spack test`` ^^^^^^^^^^^^^^ +See the :ref:`contributor guide section <cmd-spack-test>` on ``spack test``. + .. _cmd-spack-python: ^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/docker_for_developers.rst b/lib/spack/docs/docker_for_developers.rst index f60c648b8c..2b489cdec4 100644 --- a/lib/spack/docs/docker_for_developers.rst +++ b/lib/spack/docs/docker_for_developers.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/dockerhub_spack.png b/lib/spack/docs/dockerhub_spack.png Binary files differnew file mode 100644 index 0000000000..44ff0ed7ed --- /dev/null +++ b/lib/spack/docs/dockerhub_spack.png diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 1ce765210f..5ec1ec9032 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -49,6 +49,8 @@ Spack uses a "manifest and lock" model similar to `Bundler gemfiles managers. The user input file is named ``spack.yaml`` and the lock file is named ``spack.lock`` +.. _environments-using: + ------------------ Using Environments ------------------ @@ -382,11 +384,12 @@ the Environment. Loading ^^^^^^^ -Once an environment has been installed, the following creates a load script for it: +Once an environment has been installed, the following creates a load +script for it: .. code-block:: console - $ spack env myenv loads -r + $ spack env loads -r This creates a file called ``loads`` in the environment directory. Sourcing that file in Bash will make the environment available to the diff --git a/lib/spack/docs/extensions.rst b/lib/spack/docs/extensions.rst index 8f4c54b435..c71a6511ed 100644 --- a/lib/spack/docs/extensions.rst +++ b/lib/spack/docs/extensions.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,12 +9,6 @@ Custom Extensions ================= -.. warning:: - - The support for extending Spack with custom commands is still experimental. - Users should expect APIs or prescribed directory structures to - change at any time. - *Spack extensions* permit you to extend Spack capabilities by deploying your own custom commands or logic in an arbitrary location on your filesystem. This might be extremely useful e.g. to develop and maintain a command whose purpose is diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index a242eab595..cd7d3a083f 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 7f3a34b62b..3a9b2d42b2 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -97,7 +97,7 @@ Check Installation With Spack installed, you should be able to run some basic Spack commands. For example: -.. command-output:: spack spec netcdf +.. command-output:: spack spec netcdf-c ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 48af67982a..8170f152a4 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -66,6 +66,7 @@ or refer to the full manual below. config_yaml build_settings environments + containers mirrors module_file_support repositories @@ -74,6 +75,7 @@ or refer to the full manual below. package_list chain extensions + pipelines .. toctree:: :maxdepth: 2 diff --git a/lib/spack/docs/known_issues.rst b/lib/spack/docs/known_issues.rst index bdd8db2f4c..7d47137392 100644 --- a/lib/spack/docs/known_issues.rst +++ b/lib/spack/docs/known_issues.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index b161682374..5268c87067 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/module_file_support.rst b/lib/spack/docs/module_file_support.rst index 1b5de3a0e8..aa7eb57653 100644 --- a/lib/spack/docs/module_file_support.rst +++ b/lib/spack/docs/module_file_support.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -119,7 +119,7 @@ For example this will add the ``mpich`` package built with ``gcc`` to your path: # ... wait for install ... - $ spack load mpich %gcc@4.4.7 # modules + $ spack load mpich %gcc@4.4.7 $ which mpicc ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpich@3.0.4/bin/mpicc @@ -129,27 +129,29 @@ want to use a package, you can type unload or unuse similarly: .. code-block:: console - $ spack unload mpich %gcc@4.4.7 # modules + $ spack unload mpich %gcc@4.4.7 .. note:: - The ``load`` and ``unload`` subcommands are - only available if you have enabled Spack's shell support *and* you - have environment-modules installed on your machine. + The ``load`` and ``unload`` subcommands are only available if you + have enabled Spack's shell support. These command DO NOT use the + underlying Spack-generated module files. -^^^^^^^^^^^^^^^^^^^^^^ -Ambiguous module names -^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^ +Ambiguous specs +^^^^^^^^^^^^^^^ -If a spec used with load/unload or use/unuse is ambiguous (i.e. more -than one installed package matches it), then Spack will warn you: +If a spec used with load/unload or is ambiguous (i.e. more than one +installed package matches it), then Spack will warn you: .. code-block:: console $ spack load libelf - ==> Error: Multiple matches for spec libelf. Choose one: - libelf@0.8.13%gcc@4.4.7 arch=linux-debian7-x86_64 - libelf@0.8.13%intel@15.0.0 arch=linux-debian7-x86_64 + ==> Error: libelf matches multiple packages. + Matching packages: + libelf@0.8.13%gcc@4.4.7 arch=linux-debian7-x86_64 + libelf@0.8.13%intel@15.0.0 arch=linux-debian7-x86_64 + Use a more specific spec You can either type the ``spack load`` command again with a fully qualified argument, or you can add just enough extra constraints to @@ -171,8 +173,15 @@ To identify just the one built with the Intel compiler. ``spack module tcl loads`` ^^^^^^^^^^^^^^^^^^^^^^^^^^ -In some cases, it is desirable to load not just a module, but also all -the modules it depends on. This is not required for most modules +In some cases, it is desirable to use a Spack-generated module, rather +than relying on Spack's built-in user-environment modification +capabilities. To translate a spec into a module name, use ``spack +module tcl loads`` or ``spack module lmod loads`` depending on the +module system desired. + + +To load not just a module, but also all the modules it depends on, use +the ``--dependencies`` option. This is not required for most modules because Spack builds binaries with RPATH support. However, not all packages use RPATH to find their dependencies: this can be true in particular for Python extensions, which are currently *not* built with diff --git a/lib/spack/docs/package_list.rst b/lib/spack/docs/package_list.rst index 6c1e053e9e..8b5a4d332f 100644 --- a/lib/spack/docs/package_list.rst +++ b/lib/spack/docs/package_list.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 7fc9d09e2d..6e1271b12c 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -553,6 +553,34 @@ version. This is useful for packages that have an easy to extrapolate URL, but keep changing their URL format every few releases. With this method, you only need to specify the ``url`` when the URL changes. +""""""""""""""""""""""" +Mirrors of the main URL +""""""""""""""""""""""" + +Spack supports listing mirrors of the main URL in a package by defining +the ``urls`` attribute: + +.. code-block:: python + + class Foo(Package): + + urls = [ + 'http://example.com/foo-1.0.tar.gz', + 'http://mirror.com/foo-1.0.tar.gz' + ] + +instead of just a single ``url``. This attribute is a list of possible URLs that +will be tried in order when fetching packages. Notice that either one of ``url`` +or ``urls`` can be present in a package, but not both at the same time. + +A well-known case of packages that can be fetched from multiple mirrors is that +of GNU. For that, Spack goes a step further and defines a mixin class that +takes care of all of the plumbing and requires packagers to just define a proper +``gnu_mirror_path`` attribute: + +.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/autoconf/package.py + :lines: 9-18 + ^^^^^^^^^^^^^^^^^^^^^^^^ Skipping the expand step ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -901,6 +929,9 @@ Git fetching supports the following parameters to ``version``: * ``tag``: Name of a tag to fetch. * ``commit``: SHA hash (or prefix) of a commit to fetch. * ``submodules``: Also fetch submodules recursively when checking out this repository. +* ``submodules_delete``: A list of submodules to forcibly delete from the repository + after fetching. Useful if a version in the repository has submodules that + have disappeared/are no longer accessible. * ``get_full_repo``: Ensure the full git history is checked out with all remote branch information. Normally (``get_full_repo=False``, the default), the git option ``--depth 1`` will be used if the version of git and the specified @@ -1479,8 +1510,8 @@ that the same package with different patches applied will have different hash identifiers. To ensure that the hashing scheme is consistent, you must use a ``sha256`` checksum for the patch. Patches will be fetched from their URLs, checked, and applied to your source code. You can use -the ``spack sha256`` command to generate a checksum for a patch file or -URL. +the GNU utils ``sha256sum`` or the macOS ``shasum -a 256`` commands to +generate a checksum for a patch file. Spack can also handle compressed patches. If you use these, Spack needs a little more help. Specifically, it needs *two* checksums: the @@ -1922,6 +1953,8 @@ issues with 1.64.0, 1.65.0, and 1.66.0, you can say: depends_on('boost@1.59.0:1.63,1.65.1,1.67.0:') +.. _dependency-types: + ^^^^^^^^^^^^^^^^ Dependency types ^^^^^^^^^^^^^^^^ @@ -1959,6 +1992,28 @@ inject the dependency's ``prefix/lib`` directory, but the package needs to be in ``PATH`` and ``PYTHONPATH`` during the build process and later when a user wants to run the package. +^^^^^^^^^^^^^^^^^^^^^^^^ +Conditional dependencies +^^^^^^^^^^^^^^^^^^^^^^^^ + +You may have a package that only requires a dependency under certain +conditions. For example, you may have a package that has optional MPI support, +- MPI is only a dependency when you want to enable MPI support for the +package. In that case, you could say something like: + +.. code-block:: python + + variant('mpi', default=False) + depends_on('mpi', when='+mpi') + +``when`` can include constraints on the variant, version, compiler, etc. and +the :mod:`syntax<spack.spec>` is the same as for Specs written on the command +line. + +If a dependency/feature of a package isn't typically used, you can save time +by making it conditional (since Spack will not build the dependency unless it +is required for the Spec). + .. _dependency_dependency_patching: ^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/pipelines.rst b/lib/spack/docs/pipelines.rst new file mode 100644 index 0000000000..f70b39a16d --- /dev/null +++ b/lib/spack/docs/pipelines.rst @@ -0,0 +1,439 @@ +.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _pipelines: + +========= +Pipelines +========= + +Spack provides commands that support generating and running automated build +pipelines designed for Gitlab CI. At the highest level it works like this: +provide a spack environment describing the set of packages you care about, +and include within that environment file a description of how those packages +should be mapped to Gitlab runners. Spack can then generate a ``.gitlab-ci.yml`` +file containing job descriptions for all your packages that can be run by a +properly configured Gitlab CI instance. When run, the generated pipeline will +build and deploy binaries, and it can optionally report to a CDash instance +regarding the health of the builds as they evolve over time. + +------------------------------ +Getting started with pipelines +------------------------------ + +It is fairly straightforward to get started with automated build pipelines. At +a minimum, you'll need to set up a Gitlab instance (more about Gitlab CI +`here <https://about.gitlab.com/product/continuous-integration/>`_) and configure +at least one `runner <https://docs.gitlab.com/runner/>`_. Then the basic steps +for setting up a build pipeline are as follows: + +#. Create a repository on your gitlab instance +#. Add a ``spack.yaml`` at the root containing your pipeline environment (see + below for details) +#. Add a ``.gitlab-ci.yml`` at the root containing a single job, similar to + this one: + + .. code-block:: yaml + + pipeline-job: + tags: + - <custom-tag> + ... + script: + - spack ci start + +#. Add any secrets required by the CI process to environment variables using the + CI web ui +#. Push a commit containing the ``spack.yaml`` and ``.gitlab-ci.yml`` mentioned above + to the gitlab repository + +The ``<custom-tag>``, above, is used to pick one of your configured runners, +while the use of the ``spack ci start`` command implies that runner has an +appropriate version of spack installed and configured for use. Of course, there +are myriad ways to customize the process. You can configure CDash reporting +on the progress of your builds, set up S3 buckets to mirror binaries built by +the pipeline, clone a custom spack repository/ref for use by the pipeline, and +more. + +While it is possible to set up pipelines on gitlab.com, the builds there are +limited to 60 minutes and generic hardware. It is also possible to +`hook up <https://about.gitlab.com/blog/2018/04/24/getting-started-gitlab-ci-gcp>`_ +Gitlab to Google Kubernetes Engine (`GKE <https://cloud.google.com/kubernetes-engine/>`_) +or Amazon Elastic Kubernetes Service (`EKS <https://aws.amazon.com/eks>`_), though those +topics are outside the scope of this document. + +----------------------------------- +Spack commands supporting pipelines +----------------------------------- + +Spack provides a command `ci` with sub-commands for doing various things related +to automated build pipelines. All of the ``spack ci ...`` commands must be run +from within a environment, as each one makes use of the environment for different +purposes. Additionally, some options to the commands (or conditions present in +the spack environment file) may require particular environment variables to be +set in order to function properly. Examples of these are typically secrets +needed for pipeline operation that should not be visible in a spack environment +file. These environment variables are described in more detail +:ref:`ci_environment_variables`. + +.. _cmd_spack_ci: + +^^^^^^^^^^^^^^^^^^ +``spack ci`` +^^^^^^^^^^^^^^^^^^ + +Super-command for functionality related to generating pipelines and executing +pipeline jobs. + +.. _cmd_spack_ci_start: + +^^^^^^^^^^^^^^^^^^ +``spack ci start`` +^^^^^^^^^^^^^^^^^^ + +Currently this command is a short-cut to first run ``spack ci generate``, followed +by ``spack ci pushyaml``. + +.. _cmd_spack_ci_generate: + +^^^^^^^^^^^^^^^^^^^^^ +``spack ci generate`` +^^^^^^^^^^^^^^^^^^^^^ + +Concretizes the specs in the active environment, stages them (as described in +:ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk. + +.. _cmd_spack_ci_pushyaml: + +^^^^^^^^^^^^^^^^^^^^^ +``spack ci pushyaml`` +^^^^^^^^^^^^^^^^^^^^^ + +Generates a commit containing the generated ``.gitlab-ci.yml`` and pushes it to a +``DOWNSTREAM_CI_REPO``, which is frequently the same repository. The branch +created has the same name as the current branch being tested, but has ``multi-ci-`` +prepended to the branch name. Once Gitlab CI has full support for dynamically +defined workloads, this command will be deprecated. + +.. _cmd_spack_ci_rebuild: + +^^^^^^^^^^^^^^^^^^^^ +``spack ci rebuild`` +^^^^^^^^^^^^^^^^^^^^ + +This sub-command is responsible for ensuring a single spec from the release +environment is up to date on the remote mirror configured in the environment, +and as such, corresponds to a single job in the ``.gitlab-ci.yml`` file. + +------------------------------------ +A pipeline-enabled spack environment +------------------------------------ + +Here's an example of a spack environment file that has been enhanced with +sections desribing a build pipeline: + +.. code-block:: yaml + + spack: + definitions: + - pkgs: + - readline@7.0 + - compilers: + - '%gcc@5.5.0' + - oses: + - os=ubuntu18.04 + - os=centos7 + specs: + - matrix: + - [$pkgs] + - [$compilers] + - [$oses] + mirrors: + cloud_gitlab: https://mirror.spack.io + gitlab-ci: + mappings: + - match: + - os=ubuntu18.04 + runner-attributes: + tags: + - spack-k8s + image: spack/spack_builder_ubuntu_18.04 + - match: + - os=centos7 + runner-attributes: + tags: + - spack-k8s + image: spack/spack_builder_centos_7 + cdash: + build-group: Release Testing + url: https://cdash.spack.io + project: Spack + site: Spack AWS Gitlab Instance + +Hopefully, the ``definitions``, ``specs``, ``mirrors``, etc. sections are already +familiar, as they are part of spack :ref:`environments`. So let's take a more +in-depth look some of the pipeline-related sections in that environment file +that might not be as familiar. + +The ``gitlab-ci`` section is used to configure how the pipeline workload should be +generated, mainly how the jobs for building specs should be assigned to the +configured runners on your instance. Each entry within the list of ``mappings`` +corresponds to a known gitlab runner, where the ``match`` section is used +in assigning a release spec to one of the runners, and the ``runner-attributes`` +section is used to configure the spec/job for that particular runner. + +There are other pipeline options you can configure within the ``gitlab-ci`` section +as well. The ``bootstrap`` section allows you to specify lists of specs from +your ``definitions`` that should be staged ahead of the environment's ``specs`` (this +section is described in more detail below). The ``enable-artifacts-buildcache`` key +takes a boolean and determines whether the pipeline uses artifacts to store and +pass along the buildcaches from one stage to the next (the default if you don't +provide this option is ``False``). The ``enable-debug-messages`` key takes a boolean +and allows you to choose whether the pipeline build jobs are run as ``spack -d ci rebuild`` +or just ``spack ci rebuild`` (the default is not to enable debug messages). The +``final-stage-rebuild-index`` section controls whether an extra job is added to the +end of your pipeline (in a stage by itself) which will regenerate the mirror's +buildcache index. Under normal operation, each pipeline job that rebuilds a package +will re-generate the mirror's buildcache index after the buildcache entry for that +job has been created and pushed to the mirror. Since jobs in the same stage can run in +parallel, there is the possibility that at the end of some stage, the index may not +reflect all the binaries in the buildcache. Adding the ``final-stage-rebuild-index`` +section ensures that at the end of the pipeline, the index will be in sync with the +binaries on the mirror. If the mirror lives in an S3 bucket, this job will need to +run on a machine with the Python ``boto3`` module installed, and consequently the +``final-stage-rebuild-index`` needs to specify a list of ``tags`` to pick a runner +satisfying that condition. It can also take an ``image`` key so Docker executor type +runners can pick the right image for the index regeneration job. + +The optional ``cdash`` section provides information that will be used by the +``spack ci generate`` command (invoked by ``spack ci start``) for reporting +to CDash. All the jobs generated from this environment will belong to a +"build group" within CDash that can be tracked over time. As the release +progresses, this build group may have jobs added or removed. The url, project, +and site are used to specify the CDash instance to which build results should +be reported. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Assignment of specs to runners +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``mappings`` section corresponds to a list of runners, and during assignment +of specs to runners, the list is traversed in order looking for matches, the +first runner that matches a release spec is assigned to build that spec. The +``match`` section within each runner mapping section is a list of specs, and +if any of those specs match the release spec (the ``spec.satisfies()`` method +is used), then that runner is considered a match. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Configuration of specs/jobs for a runner +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once a runner has been chosen to build a release spec, the ``runner-attributes`` +section provides information determining details of the job in the context of +the runner. The ``runner-attributes`` section must have a ``tags`` key, which +is a list containing at least one tag used to select the runner from among the +runners known to the gitlab instance. For Docker executor type runners, the +``image`` key is used to specify the Docker image used to build the release spec +(and could also appear as a dictionary with a ``name`` specifying the image name, +as well as an ``entrypoint`` to override whatever the default for that image is). +For other types of runners the ``variables`` key will be useful to pass any +information on to the runner that it needs to do its work (e.g. scheduler +parameters, etc.). + +.. _staging_algorithm: + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Summary of ``.gitlab-ci.yml`` generation algorithm +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +All specs yielded by the matrix (or all the specs in the environment) have their +dependencies computed, and the entire resulting set of specs are staged together +before being run through the ``gitlab-ci/mappings`` entries, where each staged +spec is assigned a runner. "Staging" is the name we have given to the process +of figuring out in what order the specs should be built, taking into consideration +Gitlab CI rules about jobs/stages. In the staging process the goal is to maximize +the number of jobs in any stage of the pipeline, while ensuring that the jobs in +any stage only depend on jobs in previous stages (since those jobs are guaranteed +to have completed already). As a runner is determined for a job, the information +in the ``runner-attributes`` is used to populate various parts of the job +description that will be used by Gitlab CI. Once all the jobs have been assigned +a runner, the ``.gitlab-ci.yml`` is written to disk. + +The short example provided above would result in the ``readline``, ``ncurses``, +and ``pkgconf`` packages getting staged and built on the runner chosen by the +``spack-k8s`` tag. In this example, we assume the runner is a Docker executor +type runner, and thus certain jobs will be run in the ``centos7`` container, +and others in the ``ubuntu-18.04`` container. The resulting ``.gitlab-ci.yml`` +will contain 6 jobs in three stages. Once the jobs have been generated, the +presence of a ``SPACK_CDASH_AUTH_TOKEN`` environment variable during the +``spack ci generate`` command would result in all of the jobs being put in a +build group on CDash called "Release Testing" (that group will be created if +it didn't already exist). + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Optional compiler bootstrapping +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Spack pipelines also have support for bootstrapping compilers on systems that +may not already have the desired compilers installed. The idea here is that +you can specify a list of things to bootstrap in your ``definitions``, and +spack will guarantee those will be installed in a phase of the pipeline before +your release specs, so that you can rely on those packages being available in +the binary mirror when you need them later on in the pipeline. At the moment +the only viable use-case for bootstrapping is to install compilers. + +Here's an example of what bootstrapping some compilers might look like: + +.. code-block:: yaml + + spack: + definitions: + - compiler-pkgs: + - 'llvm+clang@6.0.1 os=centos7' + - 'gcc@6.5.0 os=centos7' + - 'llvm+clang@6.0.1 os=ubuntu18.04' + - 'gcc@6.5.0 os=ubuntu18.04' + - pkgs: + - readline@7.0 + - compilers: + - '%gcc@5.5.0' + - '%gcc@6.5.0' + - '%gcc@7.3.0' + - '%clang@6.0.0' + - '%clang@6.0.1' + - oses: + - os=ubuntu18.04 + - os=centos7 + specs: + - matrix: + - [$pkgs] + - [$compilers] + - [$oses] + exclude: + - '%gcc@7.3.0 os=centos7' + - '%gcc@5.5.0 os=ubuntu18.04' + gitlab-ci: + bootstrap: + - name: compiler-pkgs + compiler-agnostic: true + mappings: + # mappings similar to the example higher up in this description + ... + +In the example above, we have added a list to the ``definitions`` called +``compiler-pkgs`` (you can add any number of these), which lists compiler packages +we want to be staged ahead of the full matrix of release specs (which consists +only of readline in our example). Then within the ``gitlab-ci`` section, we +have added a ``bootstrap`` section, which can contain a list of items, each +referring to a list in the ``definitions`` section. These items can either +be a dictionary or a string. If you supply a dictionary, it must have a name +key whose value must match one of the lists in definitions and it can have a +``compiler-agnostic`` key whose value is a boolean. If you supply a string, +then it needs to match one of the lists provided in ``definitions``. You can +think of the bootstrap list as an ordered list of pipeline "phases" that will +be staged before your actual release specs. While this introduces another +layer of bottleneck in the pipeline (all jobs in all stages of one phase must +complete before any jobs in the next phase can begin), it also means you are +guaranteed your bootstrapped compilers will be available when you need them. + +The ``compiler-agnostic`` key can be provided with each item in the +bootstrap list. It tells the ``spack ci generate`` command that any jobs staged +from that particular list should have the compiler removed from the spec, so +that any compiler available on the runner where the job is run can be used to +build the package. + +When including a bootstrapping phase as in the example above, the result is that +the bootstrapped compiler packages will be pushed to the binary mirror (and the +local artifacts mirror) before the actual release specs are built. In this case, +the jobs corresponding to subsequent release specs are configured to +``install_missing_compilers``, so that if spack is asked to install a package +with a compiler it doesn't know about, it can be quickly installed from the +binary mirror first. + +Since bootstrapping compilers is optional, those items can be left out of the +environment/stack file, and in that case no bootstrapping will be done (only the +specs will be staged for building) and the runners will be expected to already +have all needed compilers installed and configured for spack to use. + +------------------------------------- +Using a custom spack in your pipeline +------------------------------------- + +If your runners will not have a version of spack ready to invoke, or if for some +other reason you want to use a custom version of spack to run your pipelines, +this can be accomplished fairly simply. First, create CI environment variables +containing the url and branch/tag you want to clone (calling them, for example, +``SPACK_REPO`` and ``SPACK_REF``), use them to clone spack in your pre-ci +``before_script``, and finally pass those same values along to the workload +generation process via the ``spack-repo`` and ``spack-ref`` cli args. Here's +an example: + +.. code-block:: yaml + + pipeline-job: + tags: + - <some-other-tag> + before_script: + - git clone ${SPACK_REPO} --branch ${SPACK_REF} + - . ./spack/share/spack/setup-env.sh + script: + - spack ci start --spack-repo ${SPACK_REPO} --spack-ref ${SPACK_REF} <...args> + after_script: + - rm -rf ./spack + +If the ``spack ci start`` command receives those extra command line arguments, +then it adds similar ``before_script`` and ``after_script`` sections for each of +the ``spack ci rebuild`` jobs it generates (cloning and sourcing a custom +spack in the ``before_script`` and removing it again in the ``after_script``). +This gives you control over the version of spack used when the rebuild jobs +are actually run on the gitlab runner. + +.. _ci_environment_variables: + +-------------------------------------------------- +Environment variables affecting pipeline operation +-------------------------------------------------- + +Certain secrets and some other information should be provided to the pipeline +infrastructure via environment variables, usually for reasons of security, but +in some cases to support other pipeline use cases such as PR testing. The +environment variables used by the pipeline infrastructure are described here. + +^^^^^^^^^^^^^^^^^ +AWS_ACCESS_KEY_ID +^^^^^^^^^^^^^^^^^ + +Needed when binary mirror is an S3 bucket. + +^^^^^^^^^^^^^^^^^^^^^ +AWS_SECRET_ACCESS_KEY +^^^^^^^^^^^^^^^^^^^^^ + +Needed when binary mirror is an S3 bucket. + +^^^^^^^^^^^^^^^ +S3_ENDPOINT_URL +^^^^^^^^^^^^^^^ + +Needed when binary mirror is an S3 bucket that is *not* on AWS. + +^^^^^^^^^^^^^^^^^ +CDASH_AUTH_TOKEN +^^^^^^^^^^^^^^^^^ + +Needed in order to report build groups to CDash. + +^^^^^^^^^^^^^^^^^ +SPACK_SIGNING_KEY +^^^^^^^^^^^^^^^^^ + +Needed to sign/verify binary packages from the remote binary mirror. + +^^^^^^^^^^^^^^^^^^ +DOWNSTREAM_CI_REPO +^^^^^^^^^^^^^^^^^^ + +Needed until Gitlab CI supports dynamic job generation. Can contain connection +credentials, and could be the same repository or a different one. diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index 45693fd2c5..6c2050a4be 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/requirements.txt b/lib/spack/docs/requirements.txt index 190b018a0b..f5c50b0199 100644 --- a/lib/spack/docs/requirements.txt +++ b/lib/spack/docs/requirements.txt @@ -1,7 +1,7 @@ # These dependencies should be installed using pip in order # to build the documentation. -sphinx==2.0.1 -sphinxcontrib-programoutput==0.14 -sphinx-rtd-theme==0.4.3 +sphinx +sphinxcontrib-programoutput +sphinx-rtd-theme python-levenshtein diff --git a/lib/spack/docs/spack.yaml b/lib/spack/docs/spack.yaml index bade961fce..3c8cd14d10 100644 --- a/lib/spack/docs/spack.yaml +++ b/lib/spack/docs/spack.yaml @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/docs/workflows.rst b/lib/spack/docs/workflows.rst index 454cb15ecc..b329a0205c 100644 --- a/lib/spack/docs/workflows.rst +++ b/lib/spack/docs/workflows.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -253,14 +253,14 @@ However, other more powerful methods are generally preferred for user environments. -^^^^^^^^^^^^^^^^^^^^^^^ -Spack-Generated Modules -^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Using ``spack load`` to Manage the User Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Suppose that Spack has been used to install a set of command-line programs, which users now wish to use. One can in principle put a number of ``spack load`` commands into ``.bashrc``, for example, to -load a set of Spack-generated modules: +load a set of Spack packages: .. code-block:: sh @@ -273,7 +273,7 @@ load a set of Spack-generated modules: Although simple load scripts like this are useful in many cases, they have some drawbacks: -1. The set of modules loaded by them will in general not be +1. The set of packages loaded by them will in general not be consistent. They are a decent way to load commands to be called from command shells. See below for better ways to assemble a consistent set of packages for building application programs. @@ -285,19 +285,24 @@ have some drawbacks: other hand, are not very smart: if the user-supplied spec matches more than one installed package, then ``spack module tcl loads`` will fail. This may change in the future. For now, the workaround is to - be more specific on any ``spack module tcl loads`` lines that fail. + be more specific on any ``spack load`` commands that fail. """""""""""""""""""""" Generated Load Scripts """""""""""""""""""""" -Another problem with using `spack load` is, it is slow; a typical user -environment could take several seconds to load, and would not be -appropriate to put into ``.bashrc`` directly. It is preferable to use -a series of ``spack module tcl loads`` commands to pre-compute which -modules to load. These can be put in a script that is run whenever -installed Spack packages change. For example: +Another problem with using `spack load` is, it can be slow; a typical +user environment could take several seconds to load, and would not be +appropriate to put into ``.bashrc`` directly. This is because it +requires the full start-up overhead of python/Spack for each command. +In some circumstances it is preferable to use a series of ``spack +module tcl loads`` (or ``spack module lmod loads``) commands to +pre-compute which modules to load. This will generate the modulenames +to load the packages using environment modules, rather than Spack's +built-in support for environment modifications. These can be put in a +script that is run whenever installed Spack packages change. For +example: .. code-block:: sh @@ -634,7 +639,7 @@ Global Activations Python (and similar systems) packages directly or creating a view. If extensions are globally activated, then ``spack load python`` will also load all the extensions activated for the given ``python``. -This reduces the need for users to load a large number of modules. +This reduces the need for users to load a large number of packages. However, Spack global activations have two potential drawbacks: @@ -1090,6 +1095,248 @@ or filesystem views. However, it has some drawbacks: integrate Spack explicitly in their workflow. Not all users are willing to do this. +------------------------------------- +Using Spack to Replace Homebrew/Conda +------------------------------------- + +Spack is an incredibly powerful package manager, designed for supercomputers +where users have diverse installation needs. But Spack can also be used to +handle simple single-user installations on your laptop. Most macOS users are +already familiar with package managers like Homebrew and Conda, where all +installed packages are symlinked to a single central location like ``/usr/local``. +In this section, we will show you how to emulate the behavior of Homebrew/Conda +using :ref:`environments`! + +^^^^^ +Setup +^^^^^ + +First, let's create a new environment. We'll assume that Spack is already set up +correctly, and that you've already sourced the setup script for your shell. +To create a new environment, simply run: + +.. code-block:: console + + $ spack env create myenv + ==> Updating view at /Users/me/spack/var/spack/environments/myenv/.spack-env/view + ==> Created environment 'myenv' in /Users/me/spack/var/spack/environments/myenv + $ spack env activate myenv + +Here, *myenv* can be anything you want to name your environment. Next, we can add +a list of packages we would like to install into our environment. Let's say we +want a newer version of Bash than the one that comes with macOS, and we want a +few Python libraries. We can run: + +.. code-block:: console + + $ spack add bash + ==> Adding bash to environment myenv + ==> Updating view at /Users/me/spack/var/spack/environments/myenv/.spack-env/view + $ spack add python@3: + ==> Adding python@3: to environment myenv + ==> Updating view at /Users/me/spack/var/spack/environments/myenv/.spack-env/view + $ spack add py-numpy py-scipy py-matplotlib + ==> Adding py-numpy to environment myenv + ==> Adding py-scipy to environment myenv + ==> Adding py-matplotlib to environment myenv + ==> Updating view at /Users/me/spack/var/spack/environments/myenv/.spack-env/view + +Each package can be listed on a separate line, or combined into a single line. +Notice that we're explicitly asking for Python 3 here. You can use any spec +you would normally use on the command line with other Spack commands. + +Next, we want to manually configure a couple of things. In the ``myenv`` +directory, we can find the ``spack.yaml`` that actually defines our environment. + +.. code-block:: console + + $ vim ~/spack/var/spack/environments/myenv/spack.yaml + +.. code-block:: yaml + + # This is a Spack Environment file. + # + # It describes a set of packages to be installed, along with + # configuration settings. + spack: + # add package specs to the `specs` list + specs: [bash, 'python@3:', py-numpy, py-scipy, py-matplotlib] + view: + default: + root: /Users/me/spack/var/spack/environments/myenv/.spack-env/view + projections: {} + config: {} + mirrors: {} + modules: + enable: [] + packages: {} + repos: [] + upstreams: {} + definitions: [] + concretization: separately + +You can see the packages we added earlier in the ``specs:`` section. If you +ever want to add more packages, you can either use ``spack add`` or manually +edit this file. + +We also need to change the ``concretization:`` option. By default, Spack +concretizes each spec *separately*, allowing multiple versions of the same +package to coexist. Since we want a single consistent environment, we want to +concretize all of the specs *together*. + +Here is what your ``spack.yaml`` looks like with these new settings, and with +some of the sections we don't plan on using removed: + +.. code-block:: diff + + spack: + - specs: [bash, 'python@3:', py-numpy, py-scipy, py-matplotlib] + + specs: + + - bash + + - 'python@3:' + + - py-numpy + + - py-scipy + + - py-matplotlib + - view: + - default: + - root: /Users/me/spack/var/spack/environments/myenv/.spack-env/view + - projections: {} + + view: /Users/me/spack/var/spack/environments/myenv/.spack-env/view + - config: {} + - mirrors: {} + - modules: + - enable: [] + - packages: {} + - repos: [] + - upstreams: {} + - definitions: [] + + concretization: together + - concretization: separately + +"""""""""""""""" +Symlink location +"""""""""""""""" + +In the ``spack.yaml`` file above, you'll notice that by default, Spack symlinks +all installations to ``/Users/me/spack/var/spack/environments/myenv/.spack-env/view``. +You can actually change this to any directory you want. For example, Homebrew +uses ``/usr/local``, while Conda uses ``/Users/me/anaconda``. In order to access +files in these locations, you need to update ``PATH`` and other environment variables +to point to them. Activating the Spack environment does this automatically, but +you can also manually set them in your ``.bashrc``. + +.. warning:: + + There are several reasons why you shouldn't use ``/usr/local``: + + 1. If you are on macOS 10.11+ (El Capitan and newer), Apple makes it hard + for you. You may notice permissions issues on ``/usr/local`` due to their + `System Integrity Protection <https://support.apple.com/en-us/HT204899>`_. + By default, users don't have permissions to install anything in ``/usr/local``, + and you can't even change this using ``sudo chown`` or ``sudo chmod``. + 2. Other package managers like Homebrew will try to install things to the + same directory. If you plan on using Homebrew in conjunction with Spack, + don't symlink things to ``/usr/local``. + 3. If you are on a shared workstation, or don't have sudo priveleges, you + can't do this. + + If you still want to do this anyway, there are several ways around SIP. + You could disable SIP by booting into recovery mode and running + ``csrutil disable``, but this is not recommended, as it can open up your OS + to security vulnerabilities. Another technique is to run ``spack concretize`` + and ``spack install`` using ``sudo``. This is also not recommended. + + The safest way I've found is to create your installation directories using + sudo, then change ownership back to the user like so: + + .. code-block:: bash + + for directory in .spack bin contrib include lib man share + do + sudo mkdir -p /usr/local/$directory + sudo chown $(id -un):$(id -gn) /usr/local/$directory + done + + Depending on the packages you install in your environment, the exact list of + directories you need to create may vary. You may also find some packages + like Java libraries that install a single file to the installation prefix + instead of in a subdirectory. In this case, the action is the same, just replace + ``mkdir -p`` with ``touch`` in the for-loop above. + + But again, it's safer just to use the default symlink location. + + +^^^^^^^^^^^^ +Installation +^^^^^^^^^^^^ + +To actually concretize the environment, run: + +.. code-block:: console + + $ spack concretize + +This will tell you which if any packages are already installed, and alert you +to any conflicting specs. + +To actually install these packages and symlink them to your ``view:`` +directory, simply run: + +.. code-block:: console + + $ spack install + +Now, when you type ``which python3``, it should find the one you just installed. + +In order to change the default shell to our newer Bash installation, we first +need to add it to this list of acceptable shells. Run: + +.. code-block:: console + + $ sudo vim /etc/shells + +and add the absolute path to your bash executable. Then run: + +.. code-block:: console + + $ chsh -s /path/to/bash + +Now, when you log out and log back in, ``echo $SHELL`` should point to the +newer version of Bash. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Updating Installed Packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Let's say you upgraded to a new version of macOS, or a new version of Python +was released, and you want to rebuild your entire software stack. To do this, +simply run the following commands: + +.. code-block:: console + + $ spack env activate myenv + $ spack concretize --force + $ spack install + +The ``--force`` flag tells Spack to overwrite its previous concretization +decisions, allowing you to choose a new version of Python. If any of the new +packages like Bash are already installed, ``spack install`` won't re-install +them, it will keep the symlinks in place. + +^^^^^^^^^^^^^^ +Uninstallation +^^^^^^^^^^^^^^ + +If you decide that Spack isn't right for you, uninstallation is simple. +Just run: + +.. code-block:: console + + $ spack env activate myenv + $ spack uninstall --all + +This will uninstall all packages in your environment and remove the symlinks. + ------------------------ Using Spack on Travis-CI ------------------------ @@ -1254,7 +1501,7 @@ In order to build and run the image, execute: RUN spack install tar \ && spack clean -a - # need the modules already during image build? + # need the executables from a package already during image build? #RUN /bin/bash -l -c ' \ # spack load tar \ # && which tar' diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 2d6fe9998a..73c5759dfe 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index 1001ff5fff..27f00efb09 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -82,14 +82,6 @@ py ini-parsing, io, code, and log facilities. * Version: 1.4.34 (last version supporting Python 2.6) -pyqver ------- - -* Homepage: https://github.com/ghewgill/pyqver -* Usage: External script to query required python version of - python source code. Used for ensuring 2.6 compatibility. -* Version: Unversioned - pytest ------ diff --git a/lib/spack/external/_pytest/pytester.py b/lib/spack/external/_pytest/pytester.py index 2db85dff22..82aa00e0d2 100644 --- a/lib/spack/external/_pytest/pytester.py +++ b/lib/spack/external/_pytest/pytester.py @@ -569,7 +569,7 @@ class Testdir: def _possibly_invalidate_import_caches(self): # invalidate caches if we can (py33 and above) try: - import importlib # nopyqver + import importlib except ImportError: pass else: diff --git a/lib/spack/external/ordereddict_backport.py b/lib/spack/external/ordereddict_backport.py index ecb7c29966..5ec8493cc9 100644 --- a/lib/spack/external/ordereddict_backport.py +++ b/lib/spack/external/ordereddict_backport.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/external/pyqver2.py b/lib/spack/external/pyqver2.py deleted file mode 100755 index 07b191425b..0000000000 --- a/lib/spack/external/pyqver2.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python -# -# pyqver2.py -# by Greg Hewgill -# https://github.com/ghewgill/pyqver -# -# This software is provided 'as-is', without any express or implied -# warranty. In no event will the author be held liable for any damages -# arising from the use of this software. -# -# Permission is granted to anyone to use this software for any purpose, -# including commercial applications, and to alter it and redistribute it -# freely, subject to the following restrictions: -# -# 1. The origin of this software must not be misrepresented; you must not -# claim that you wrote the original software. If you use this software -# in a product, an acknowledgment in the product documentation would be -# appreciated but is not required. -# 2. Altered source versions must be plainly marked as such, and must not be -# misrepresented as being the original software. -# 3. This notice may not be removed or altered from any source distribution. -# -# Copyright (c) 2009-2013 Greg Hewgill http://hewgill.com -# - -import compiler -import platform -import sys - -StandardModules = { - "__future__": (2, 1), - "abc": (2, 6), -# skip argparse now that it's in lib/spack/external -# "argparse": (2, 7), - "ast": (2, 6), - "atexit": (2, 0), - "bz2": (2, 3), - "cgitb": (2, 2), - "collections": (2, 4), - "contextlib": (2, 5), - "cookielib": (2, 4), - "cProfile": (2, 5), - "csv": (2, 3), - "ctypes": (2, 5), - "datetime": (2, 3), - "decimal": (2, 4), - "difflib": (2, 1), - "DocXMLRPCServer": (2, 3), - "dummy_thread": (2, 3), - "dummy_threading": (2, 3), - "email": (2, 2), - "fractions": (2, 6), - "functools": (2, 5), - "future_builtins": (2, 6), - "hashlib": (2, 5), - "heapq": (2, 3), - "hmac": (2, 2), - "hotshot": (2, 2), - "HTMLParser": (2, 2), - "importlib": (2, 7), - "inspect": (2, 1), - "io": (2, 6), - "itertools": (2, 3), - "json": (2, 6), - "logging": (2, 3), - "modulefinder": (2, 3), - "msilib": (2, 5), - "multiprocessing": (2, 6), - "netrc": (1, 5, 2), - "numbers": (2, 6), - "optparse": (2, 3), - "ossaudiodev": (2, 3), - "pickletools": (2, 3), - "pkgutil": (2, 3), - "platform": (2, 3), - "pydoc": (2, 1), - "runpy": (2, 5), - "sets": (2, 3), - "shlex": (1, 5, 2), - "SimpleXMLRPCServer": (2, 2), - "spwd": (2, 5), - "sqlite3": (2, 5), - "ssl": (2, 6), - "stringprep": (2, 3), - "subprocess": (2, 4), - "sysconfig": (2, 7), - "tarfile": (2, 3), - "textwrap": (2, 3), - "timeit": (2, 3), - "unittest": (2, 1), - "uuid": (2, 5), - "warnings": (2, 1), - "weakref": (2, 1), - "winsound": (1, 5, 2), - "wsgiref": (2, 5), - "xml.dom": (2, 0), - "xml.dom.minidom": (2, 0), - "xml.dom.pulldom": (2, 0), - "xml.etree.ElementTree": (2, 5), - "xml.parsers.expat":(2, 0), - "xml.sax": (2, 0), - "xml.sax.handler": (2, 0), - "xml.sax.saxutils": (2, 0), - "xml.sax.xmlreader":(2, 0), - "xmlrpclib": (2, 2), - "zipfile": (1, 6), - "zipimport": (2, 3), - "_ast": (2, 5), - "_winreg": (2, 0), -} - -Functions = { - "all": (2, 5), - "any": (2, 5), - "collections.Counter": (2, 7), - "collections.defaultdict": (2, 5), - "collections.OrderedDict": (2, 7), - "functools.total_ordering": (2, 7), - "enumerate": (2, 3), - "frozenset": (2, 4), - "itertools.compress": (2, 7), - "math.erf": (2, 7), - "math.erfc": (2, 7), - "math.expm1": (2, 7), - "math.gamma": (2, 7), - "math.lgamma": (2, 7), - "memoryview": (2, 7), - "next": (2, 6), - "os.getresgid": (2, 7), - "os.getresuid": (2, 7), - "os.initgroups": (2, 7), - "os.setresgid": (2, 7), - "os.setresuid": (2, 7), - "reversed": (2, 4), - "set": (2, 4), - "subprocess.check_call": (2, 5), - "subprocess.check_output": (2, 7), - "sum": (2, 3), - "symtable.is_declared_global": (2, 7), - "weakref.WeakSet": (2, 7), -} - -Identifiers = { - "False": (2, 2), - "True": (2, 2), -} - -def uniq(a): - if len(a) == 0: - return [] - else: - return [a[0]] + uniq([x for x in a if x != a[0]]) - -class NodeChecker(object): - def __init__(self): - self.vers = dict() - self.vers[(2,0)] = [] - def add(self, node, ver, msg): - if ver not in self.vers: - self.vers[ver] = [] - self.vers[ver].append((node.lineno, msg)) - def default(self, node): - for child in node.getChildNodes(): - self.visit(child) - def visitCallFunc(self, node): - def rollup(n): - if isinstance(n, compiler.ast.Name): - return n.name - elif isinstance(n, compiler.ast.Const): - return type(n.value).__name__ - elif isinstance(n, compiler.ast.Getattr): - r = rollup(n.expr) - if r: - return r + "." + n.attrname - name = rollup(node.node) - if name: - # Special handling for empty format strings, which aren't - # allowed in Python 2.6 - if name in ('unicode.format', 'str.format'): - n = node.node - if isinstance(n, compiler.ast.Getattr): - n = n.expr - if isinstance(n, compiler.ast.Const): - if '{}' in n.value: - self.add(node, (2,7), name + ' with {} format string') - - v = Functions.get(name) - if v is not None: - self.add(node, v, name) - self.default(node) - def visitClass(self, node): - if node.bases: - self.add(node, (2,2), "new-style class") - if node.decorators: - self.add(node, (2,6), "class decorator") - self.default(node) - def visitDictComp(self, node): - self.add(node, (2,7), "dictionary comprehension") - self.default(node) - def visitFloorDiv(self, node): - self.add(node, (2,2), "// operator") - self.default(node) - def visitFrom(self, node): - v = StandardModules.get(node.modname) - if v is not None: - self.add(node, v, node.modname) - for n in node.names: - name = node.modname + "." + n[0] - v = Functions.get(name) - if v is not None: - self.add(node, v, name) - def visitFunction(self, node): - if node.decorators: - self.add(node, (2,4), "function decorator") - self.default(node) - def visitGenExpr(self, node): - self.add(node, (2,4), "generator expression") - self.default(node) - def visitGetattr(self, node): - if (isinstance(node.expr, compiler.ast.Const) - and isinstance(node.expr.value, str) - and node.attrname == "format"): - self.add(node, (2,6), "string literal .format()") - self.default(node) - def visitIfExp(self, node): - self.add(node, (2,5), "inline if expression") - self.default(node) - def visitImport(self, node): - for n in node.names: - v = StandardModules.get(n[0]) - if v is not None: - self.add(node, v, n[0]) - self.default(node) - def visitName(self, node): - v = Identifiers.get(node.name) - if v is not None: - self.add(node, v, node.name) - self.default(node) - def visitSet(self, node): - self.add(node, (2,7), "set literal") - self.default(node) - def visitSetComp(self, node): - self.add(node, (2,7), "set comprehension") - self.default(node) - def visitTryFinally(self, node): - # try/finally with a suite generates a Stmt node as the body, - # but try/except/finally generates a TryExcept as the body - if isinstance(node.body, compiler.ast.TryExcept): - self.add(node, (2,5), "try/except/finally") - self.default(node) - def visitWith(self, node): - if isinstance(node.body, compiler.ast.With): - self.add(node, (2,7), "with statement with multiple contexts") - else: - self.add(node, (2,5), "with statement") - self.default(node) - def visitYield(self, node): - self.add(node, (2,2), "yield expression") - self.default(node) - -def get_versions(source, filename=None): - """Return information about the Python versions required for specific features. - - The return value is a dictionary with keys as a version number as a tuple - (for example Python 2.6 is (2,6)) and the value are a list of features that - require the indicated Python version. - """ - tree = compiler.parse(source) - checker = compiler.walk(tree, NodeChecker()) - return checker.vers - -def v27(source): - if sys.version_info >= (2, 7): - return qver(source) - else: - print >>sys.stderr, "Not all features tested, run --test with Python 2.7" - return (2, 7) - -def qver(source): - """Return the minimum Python version required to run a particular bit of code. - - >>> qver('print "hello world"') - (2, 0) - >>> qver('class test(object): pass') - (2, 2) - >>> qver('yield 1') - (2, 2) - >>> qver('a // b') - (2, 2) - >>> qver('True') - (2, 2) - >>> qver('enumerate(a)') - (2, 3) - >>> qver('total = sum') - (2, 0) - >>> qver('sum(a)') - (2, 3) - >>> qver('(x*x for x in range(5))') - (2, 4) - >>> qver('class C:\\n @classmethod\\n def m(): pass') - (2, 4) - >>> qver('y if x else z') - (2, 5) - >>> qver('import hashlib') - (2, 5) - >>> qver('from hashlib import md5') - (2, 5) - >>> qver('import xml.etree.ElementTree') - (2, 5) - >>> qver('try:\\n try: pass;\\n except: pass;\\nfinally: pass') - (2, 0) - >>> qver('try: pass;\\nexcept: pass;\\nfinally: pass') - (2, 5) - >>> qver('from __future__ import with_statement\\nwith x: pass') - (2, 5) - >>> qver('collections.defaultdict(list)') - (2, 5) - >>> qver('from collections import defaultdict') - (2, 5) - >>> qver('"{0}".format(0)') - (2, 6) - >>> qver('memoryview(x)') - (2, 7) - >>> v27('{1, 2, 3}') - (2, 7) - >>> v27('{x for x in s}') - (2, 7) - >>> v27('{x: y for x in s}') - (2, 7) - >>> qver('from __future__ import with_statement\\nwith x:\\n with y: pass') - (2, 5) - >>> v27('from __future__ import with_statement\\nwith x, y: pass') - (2, 7) - >>> qver('@decorator\\ndef f(): pass') - (2, 4) - >>> qver('@decorator\\nclass test:\\n pass') - (2, 6) - - #>>> qver('0o0') - #(2, 6) - #>>> qver('@foo\\nclass C: pass') - #(2, 6) - """ - return max(get_versions(source).keys()) diff --git a/lib/spack/external/pyqver3.py b/lib/spack/external/pyqver3.py deleted file mode 100755 index b63576a064..0000000000 --- a/lib/spack/external/pyqver3.py +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env python3 -# -# pyqver3.py -# by Greg Hewgill -# https://github.com/ghewgill/pyqver -# -# This software is provided 'as-is', without any express or implied -# warranty. In no event will the author be held liable for any damages -# arising from the use of this software. -# -# Permission is granted to anyone to use this software for any purpose, -# including commercial applications, and to alter it and redistribute it -# freely, subject to the following restrictions: -# -# 1. The origin of this software must not be misrepresented; you must not -# claim that you wrote the original software. If you use this software -# in a product, an acknowledgment in the product documentation would be -# appreciated but is not required. -# 2. Altered source versions must be plainly marked as such, and must not be -# misrepresented as being the original software. -# 3. This notice may not be removed or altered from any source distribution. -# -# Copyright (c) 2009-2013 Greg Hewgill http://hewgill.com -# -import ast -import platform -import sys - -StandardModules = { -# skip argparse now that it's in lib/spack/external -# "argparse": (3, 2), - "faulthandler": (3, 3), - "importlib": (3, 1), - "ipaddress": (3, 3), - "lzma": (3, 3), - "tkinter.ttk": (3, 1), - "unittest.mock": (3, 3), - "venv": (3, 3), -} - -Functions = { - "bytearray.maketrans": (3, 1), - "bytes.maketrans": (3, 1), - "bz2.open": (3, 3), - "collections.Counter": (3, 1), - "collections.OrderedDict": (3, 1), - "crypt.mksalt": (3, 3), - "email.generator.BytesGenerator": (3, 2), - "email.message_from_binary_file": (3, 2), - "email.message_from_bytes": (3, 2), - "functools.lru_cache": (3, 2), - "gzip.compress": (3, 2), - "gzip.decompress": (3, 2), - "inspect.getclosurevars": (3, 3), - "inspect.getgeneratorlocals": (3, 3), - "inspect.getgeneratorstate": (3, 2), - "itertools.combinations_with_replacement": (3, 1), - "itertools.compress": (3, 1), - "logging.config.dictConfig": (3, 2), - "logging.NullHandler": (3, 1), - "math.erf": (3, 2), - "math.erfc": (3, 2), - "math.expm1": (3, 2), - "math.gamma": (3, 2), - "math.isfinite": (3, 2), - "math.lgamma": (3, 2), - "math.log2": (3, 3), - "os.environb": (3, 2), - "os.fsdecode": (3, 2), - "os.fsencode": (3, 2), - "os.fwalk": (3, 3), - "os.getenvb": (3, 2), - "os.get_exec_path": (3, 2), - "os.getgrouplist": (3, 3), - "os.getpriority": (3, 3), - "os.getresgid": (3, 2), - "os.getresuid": (3, 2), - "os.get_terminal_size": (3, 3), - "os.getxattr": (3, 3), - "os.initgroups": (3, 2), - "os.listxattr": (3, 3), - "os.lockf": (3, 3), - "os.pipe2": (3, 3), - "os.posix_fadvise": (3, 3), - "os.posix_fallocate": (3, 3), - "os.pread": (3, 3), - "os.pwrite": (3, 3), - "os.readv": (3, 3), - "os.removexattr": (3, 3), - "os.replace": (3, 3), - "os.sched_get_priority_max": (3, 3), - "os.sched_get_priority_min": (3, 3), - "os.sched_getaffinity": (3, 3), - "os.sched_getparam": (3, 3), - "os.sched_getscheduler": (3, 3), - "os.sched_rr_get_interval": (3, 3), - "os.sched_setaffinity": (3, 3), - "os.sched_setparam": (3, 3), - "os.sched_setscheduler": (3, 3), - "os.sched_yield": (3, 3), - "os.sendfile": (3, 3), - "os.setpriority": (3, 3), - "os.setresgid": (3, 2), - "os.setresuid": (3, 2), - "os.setxattr": (3, 3), - "os.sync": (3, 3), - "os.truncate": (3, 3), - "os.waitid": (3, 3), - "os.writev": (3, 3), - "shutil.chown": (3, 3), - "shutil.disk_usage": (3, 3), - "shutil.get_archive_formats": (3, 3), - "shutil.get_terminal_size": (3, 3), - "shutil.get_unpack_formats": (3, 3), - "shutil.make_archive": (3, 3), - "shutil.register_archive_format": (3, 3), - "shutil.register_unpack_format": (3, 3), - "shutil.unpack_archive": (3, 3), - "shutil.unregister_archive_format": (3, 3), - "shutil.unregister_unpack_format": (3, 3), - "shutil.which": (3, 3), - "signal.pthread_kill": (3, 3), - "signal.pthread_sigmask": (3, 3), - "signal.sigpending": (3, 3), - "signal.sigtimedwait": (3, 3), - "signal.sigwait": (3, 3), - "signal.sigwaitinfo": (3, 3), - "socket.CMSG_LEN": (3, 3), - "socket.CMSG_SPACE": (3, 3), - "socket.fromshare": (3, 3), - "socket.if_indextoname": (3, 3), - "socket.if_nameindex": (3, 3), - "socket.if_nametoindex": (3, 3), - "socket.sethostname": (3, 3), - "ssl.match_hostname": (3, 2), - "ssl.RAND_bytes": (3, 3), - "ssl.RAND_pseudo_bytes": (3, 3), - "ssl.SSLContext": (3, 2), - "ssl.SSLEOFError": (3, 3), - "ssl.SSLSyscallError": (3, 3), - "ssl.SSLWantReadError": (3, 3), - "ssl.SSLWantWriteError": (3, 3), - "ssl.SSLZeroReturnError": (3, 3), - "stat.filemode": (3, 3), - "textwrap.indent": (3, 3), - "threading.get_ident": (3, 3), - "time.clock_getres": (3, 3), - "time.clock_gettime": (3, 3), - "time.clock_settime": (3, 3), - "time.get_clock_info": (3, 3), - "time.monotonic": (3, 3), - "time.perf_counter": (3, 3), - "time.process_time": (3, 3), - "types.new_class": (3, 3), - "types.prepare_class": (3, 3), -} - -def uniq(a): - if len(a) == 0: - return [] - else: - return [a[0]] + uniq([x for x in a if x != a[0]]) - -class NodeChecker(ast.NodeVisitor): - def __init__(self): - self.vers = dict() - self.vers[(3,0)] = [] - def add(self, node, ver, msg): - if ver not in self.vers: - self.vers[ver] = [] - self.vers[ver].append((node.lineno, msg)) - def visit_Call(self, node): - def rollup(n): - if isinstance(n, ast.Name): - return n.id - elif isinstance(n, ast.Attribute): - r = rollup(n.value) - if r: - return r + "." + n.attr - name = rollup(node.func) - if name: - v = Functions.get(name) - if v is not None: - self.add(node, v, name) - self.generic_visit(node) - def visit_Import(self, node): - for n in node.names: - v = StandardModules.get(n.name) - if v is not None: - self.add(node, v, n.name) - self.generic_visit(node) - def visit_ImportFrom(self, node): - v = StandardModules.get(node.module) - if v is not None: - self.add(node, v, node.module) - for n in node.names: - name = node.module + "." + n.name - v = Functions.get(name) - if v is not None: - self.add(node, v, name) - def visit_Raise(self, node): - if isinstance(node.cause, ast.Name) and node.cause.id == "None": - self.add(node, (3,3), "raise ... from None") - def visit_YieldFrom(self, node): - self.add(node, (3,3), "yield from") - -def get_versions(source, filename=None): - """Return information about the Python versions required for specific features. - - The return value is a dictionary with keys as a version number as a tuple - (for example Python 3.1 is (3,1)) and the value are a list of features that - require the indicated Python version. - """ - tree = ast.parse(source, filename=filename) - checker = NodeChecker() - checker.visit(tree) - return checker.vers - -def v33(source): - if sys.version_info >= (3, 3): - return qver(source) - else: - print("Not all features tested, run --test with Python 3.3", file=sys.stderr) - return (3, 3) - -def qver(source): - """Return the minimum Python version required to run a particular bit of code. - - >>> qver('print("hello world")') - (3, 0) - >>> qver("import importlib") - (3, 1) - >>> qver("from importlib import x") - (3, 1) - >>> qver("import tkinter.ttk") - (3, 1) - >>> qver("from collections import Counter") - (3, 1) - >>> qver("collections.OrderedDict()") - (3, 1) - >>> qver("import functools\\n@functools.lru_cache()\\ndef f(x): x*x") - (3, 2) - >>> v33("yield from x") - (3, 3) - >>> v33("raise x from None") - (3, 3) - """ - return max(get_versions(source).keys()) diff --git a/lib/spack/external/ruamel/yaml/compat.py b/lib/spack/external/ruamel/yaml/compat.py index 6eee151c51..a1778bef28 100644 --- a/lib/spack/external/ruamel/yaml/compat.py +++ b/lib/spack/external/ruamel/yaml/compat.py @@ -12,7 +12,7 @@ try: from ruamel.ordereddict import ordereddict except: try: - from collections import OrderedDict # nopyqver + from collections import OrderedDict except ImportError: from ordereddict import OrderedDict # to get the right name import ... as ordereddict doesn't do that diff --git a/lib/spack/llnl/__init__.py b/lib/spack/llnl/__init__.py index 94f8ac4d9e..9f87532b85 100644 --- a/lib/spack/llnl/__init__.py +++ b/lib/spack/llnl/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/__init__.py b/lib/spack/llnl/util/__init__.py index 94f8ac4d9e..9f87532b85 100644 --- a/lib/spack/llnl/util/__init__.py +++ b/lib/spack/llnl/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/argparsewriter.py b/lib/spack/llnl/util/argparsewriter.py index 16bb570a77..f43595145e 100644 --- a/lib/spack/llnl/util/argparsewriter.py +++ b/lib/spack/llnl/util/argparsewriter.py @@ -1,203 +1,379 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function + import re import argparse import errno import sys - -class ArgparseWriter(object): +from six import StringIO + + +class Command(object): + """Parsed representation of a command from argparse. + + This is a single command from an argparse parser. ``ArgparseWriter`` + creates these and returns them from ``parse()``, and it passes one of + these to each call to ``format()`` so that we can take an action for + a single command. + + Parts of a Command: + - prog: command name (str) + - description: command description (str) + - usage: command usage (str) + - positionals: list of positional arguments (list) + - optionals: list of optional arguments (list) + - subcommands: list of subcommand parsers (list) + """ + def __init__(self, prog, description, usage, + positionals, optionals, subcommands): + self.prog = prog + self.description = description + self.usage = usage + self.positionals = positionals + self.optionals = optionals + self.subcommands = subcommands + + +# NOTE: The only reason we subclass argparse.HelpFormatter is to get access +# to self._expand_help(), ArgparseWriter is not intended to be used as a +# formatter_class. +class ArgparseWriter(argparse.HelpFormatter): """Analyzes an argparse ArgumentParser for easy generation of help.""" - def __init__(self, out=sys.stdout): + + def __init__(self, prog, out=sys.stdout, aliases=False): + """Initializes a new ArgparseWriter instance. + + Parameters: + prog (str): the program name + out (file object): the file to write to + aliases (bool): whether or not to include subparsers for aliases + """ + super(ArgparseWriter, self).__init__(prog) self.level = 0 + self.prog = prog self.out = out + self.aliases = aliases + + def parse(self, parser, prog): + """Parses the parser object and returns the relavent components. - def _write(self, parser, root=True, level=0): + Parameters: + parser (argparse.ArgumentParser): the parser + prog (str): the command name + + Returns: + (Command) information about the command from the parser + """ self.parser = parser - self.level = level - actions = parser._actions - # allow root level to be flattened with rest of commands - if type(root) == int: - self.level = root - root = True + split_prog = parser.prog.split(' ') + split_prog[-1] = prog + prog = ' '.join(split_prog) + description = parser.description - # go through actions and split them into optionals, positionals, + fmt = parser._get_formatter() + actions = parser._actions + groups = parser._mutually_exclusive_groups + usage = fmt._format_usage(None, actions, groups, '').strip() + + # Go through actions and split them into optionals, positionals, # and subcommands optionals = [] positionals = [] subcommands = [] for action in actions: if action.option_strings: - optionals.append(action) + flags = action.option_strings + dest_flags = fmt._format_action_invocation(action) + help = self._expand_help(action) if action.help else '' + help = help.replace('\n', ' ') + optionals.append((flags, dest_flags, help)) elif isinstance(action, argparse._SubParsersAction): for subaction in action._choices_actions: subparser = action._name_parser_map[subaction.dest] - subcommands.append(subparser) + subcommands.append((subparser, subaction.dest)) + + # Look for aliases of the form 'name (alias, ...)' + if self.aliases: + match = re.match(r'(.*) \((.*)\)', subaction.metavar) + if match: + aliases = match.group(2).split(', ') + for alias in aliases: + subparser = action._name_parser_map[alias] + subcommands.append((subparser, alias)) else: - positionals.append(action) + args = fmt._format_action_invocation(action) + help = self._expand_help(action) if action.help else '' + help = help.replace('\n', ' ') + positionals.append((args, help)) - groups = parser._mutually_exclusive_groups - fmt = parser._get_formatter() - description = parser.description + return Command( + prog, description, usage, positionals, optionals, subcommands) - def action_group(function, actions): - for action in actions: - arg = fmt._format_action_invocation(action) - help = action.help if action.help else '' - function(arg, re.sub('\n', ' ', help)) + def format(self, cmd): + """Returns the string representation of a single node in the + parser tree. - if root: - self.begin_command(parser.prog) + Override this in subclasses to define how each subcommand + should be displayed. - if description: - self.description(parser.description) + Parameters: + (Command): parsed information about a command or subcommand - usage = fmt._format_usage(None, actions, groups, '').strip() - self.usage(usage) + Returns: + str: the string representation of this subcommand + """ + raise NotImplementedError - if positionals: - self.begin_positionals() - action_group(self.positional, positionals) - self.end_positionals() + def _write(self, parser, prog, level=0): + """Recursively writes a parser. - if optionals: - self.begin_optionals() - action_group(self.optional, optionals) - self.end_optionals() + Parameters: + parser (argparse.ArgumentParser): the parser + prog (str): the command name + level (int): the current level + """ + self.level = level - if subcommands: - self.begin_subcommands(subcommands) - for subparser in subcommands: - self._write(subparser, root=True, level=level + 1) - self.end_subcommands(subcommands) + cmd = self.parse(parser, prog) + self.out.write(self.format(cmd)) - if root: - self.end_command(parser.prog) + for subparser, prog in cmd.subcommands: + self._write(subparser, prog, level=level + 1) - def write(self, parser, root=True): + def write(self, parser): """Write out details about an ArgumentParser. Args: - parser (ArgumentParser): an ``argparse`` parser - root (bool or int): if bool, whether to include the root parser; - or ``1`` to flatten the root parser with first-level - subcommands + parser (argparse.ArgumentParser): the parser """ try: - self._write(parser, root, level=0) + self._write(parser, self.prog) except IOError as e: - # swallow pipe errors + # Swallow pipe errors + # Raises IOError in Python 2 and BrokenPipeError in Python 3 if e.errno != errno.EPIPE: raise + +_rst_levels = ['=', '-', '^', '~', ':', '`'] + + +class ArgparseRstWriter(ArgparseWriter): + """Write argparse output as rst sections.""" + + def __init__(self, prog, out=sys.stdout, aliases=False, + rst_levels=_rst_levels): + """Create a new ArgparseRstWriter. + + Parameters: + prog (str): program name + out (file object): file to write to + aliases (bool): whether or not to include subparsers for aliases + rst_levels (list of str): list of characters + for rst section headings + """ + super(ArgparseRstWriter, self).__init__(prog, out, aliases) + self.rst_levels = rst_levels + + def format(self, cmd): + string = StringIO() + string.write(self.begin_command(cmd.prog)) + + if cmd.description: + string.write(self.description(cmd.description)) + + string.write(self.usage(cmd.usage)) + + if cmd.positionals: + string.write(self.begin_positionals()) + for args, help in cmd.positionals: + string.write(self.positional(args, help)) + string.write(self.end_positionals()) + + if cmd.optionals: + string.write(self.begin_optionals()) + for flags, dest_flags, help in cmd.optionals: + string.write(self.optional(dest_flags, help)) + string.write(self.end_optionals()) + + if cmd.subcommands: + string.write(self.begin_subcommands(cmd.subcommands)) + + return string.getvalue() + def begin_command(self, prog): - pass + return """ +---- + +.. _{0}: - def end_command(self, prog): - pass +{1} +{2} + +""".format(prog.replace(' ', '-'), prog, + self.rst_levels[self.level] * len(prog)) def description(self, description): - pass + return description + '\n\n' def usage(self, usage): - pass + return """\ +.. code-block:: console + + {0} + +""".format(usage) def begin_positionals(self): - pass + return '\n**Positional arguments**\n\n' def positional(self, name, help): - pass + return """\ +{0} + {1} + +""".format(name, help) def end_positionals(self): - pass + return '' def begin_optionals(self): - pass + return '\n**Optional arguments**\n\n' + + def optional(self, opts, help): + return """\ +``{0}`` + {1} - def optional(self, option, help): - pass +""".format(opts, help) def end_optionals(self): - pass + return '' def begin_subcommands(self, subcommands): - pass + string = """ +**Subcommands** - def end_subcommands(self, subcommands): - pass +.. hlist:: + :columns: 4 +""" -_rst_levels = ['=', '-', '^', '~', ':', '`'] + for cmd, _ in subcommands: + prog = re.sub(r'^[^ ]* ', '', cmd.prog) + string += ' * :ref:`{0} <{1}>`\n'.format( + prog, cmd.prog.replace(' ', '-')) + return string + '\n' -class ArgparseRstWriter(ArgparseWriter): - """Write argparse output as rst sections.""" - def __init__(self, out=sys.stdout, rst_levels=_rst_levels, - strip_root_prog=True): - """Create a new ArgparseRstWriter. +class ArgparseCompletionWriter(ArgparseWriter): + """Write argparse output as shell programmable tab completion functions.""" - Args: - out (file object): file to write to - rst_levels (list of str): list of characters - for rst section headings - strip_root_prog (bool): if ``True``, strip the base command name - from subcommands in output + def format(self, cmd): + """Returns the string representation of a single node in the + parser tree. + + Override this in subclasses to define how each subcommand + should be displayed. + + Parameters: + (Command): parsed information about a command or subcommand + + Returns: + str: the string representation of this subcommand """ - super(ArgparseRstWriter, self).__init__(out) - self.rst_levels = rst_levels - self.strip_root_prog = strip_root_prog - def line(self, string=''): - self.out.write('%s\n' % string) + assert cmd.optionals # we should always at least have -h, --help + assert not (cmd.positionals and cmd.subcommands) # one or the other - def begin_command(self, prog): - self.line() - self.line('----') - self.line() - self.line('.. _%s:\n' % prog.replace(' ', '-')) - self.line('%s' % prog) - self.line(self.rst_levels[self.level] * len(prog) + '\n') + # We only care about the arguments/flags, not the help messages + positionals = [] + if cmd.positionals: + positionals, _ = zip(*cmd.positionals) + optionals, _, _ = zip(*cmd.optionals) + subcommands = [] + if cmd.subcommands: + _, subcommands = zip(*cmd.subcommands) - def description(self, description): - self.line('%s\n' % description) + # Flatten lists of lists + optionals = [x for xx in optionals for x in xx] - def usage(self, usage): - self.line('.. code-block:: console\n') - self.line(' %s\n' % usage) + return (self.start_function(cmd.prog) + + self.body(positionals, optionals, subcommands) + + self.end_function(cmd.prog)) - def begin_positionals(self): - self.line() - self.line('**Positional arguments**\n') + def start_function(self, prog): + """Returns the syntax needed to begin a function definition. - def positional(self, name, help): - self.line(name) - self.line(' %s\n' % help) + Parameters: + prog (str): the command name - def begin_optionals(self): - self.line() - self.line('**Optional arguments**\n') + Returns: + str: the function definition beginning + """ + name = prog.replace('-', '_').replace(' ', '_') + return '\n_{0}() {{'.format(name) - def optional(self, opts, help): - self.line('``%s``' % opts) - self.line(' %s\n' % help) + def end_function(self, prog=None): + """Returns the syntax needed to end a function definition. - def begin_subcommands(self, subcommands): - self.line() - self.line('**Subcommands**\n') - self.line('.. hlist::') - self.line(' :columns: 4\n') - - for cmd in subcommands: - prog = cmd.prog - if self.strip_root_prog: - prog = re.sub(r'^[^ ]* ', '', prog) - - self.line(' * :ref:`%s <%s>`' - % (prog, cmd.prog.replace(' ', '-'))) - self.line() + Parameters: + prog (str, optional): the command name + + Returns: + str: the function definition ending + """ + return '}\n' + + def body(self, positionals, optionals, subcommands): + """Returns the body of the function. + + Parameters: + positionals (list): list of positional arguments + optionals (list): list of optional arguments + subcommands (list): list of subcommand parsers + + Returns: + str: the function body + """ + return '' + + def positionals(self, positionals): + """Returns the syntax for reporting positional arguments. + + Parameters: + positionals (list): list of positional arguments + + Returns: + str: the syntax for positional arguments + """ + return '' + + def optionals(self, optionals): + """Returns the syntax for reporting optional flags. + + Parameters: + optionals (list): list of optional arguments + + Returns: + str: the syntax for optional flags + """ + return '' + + def subcommands(self, subcommands): + """Returns the syntax for reporting subcommands. + + Parameters: + subcommands (list): list of subcommand parsers + + Returns: + str: the syntax for subcommand parsers + """ + return '' diff --git a/lib/spack/llnl/util/cpu/__init__.py b/lib/spack/llnl/util/cpu/__init__.py index cf3c3ef50c..5a4943c965 100644 --- a/lib/spack/llnl/util/cpu/__init__.py +++ b/lib/spack/llnl/util/cpu/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/cpu/alias.py b/lib/spack/llnl/util/cpu/alias.py index f52ecf7354..abb4ea22f5 100644 --- a/lib/spack/llnl/util/cpu/alias.py +++ b/lib/spack/llnl/util/cpu/alias.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/cpu/detect.py b/lib/spack/llnl/util/cpu/detect.py index c89f67c852..66b09f5e9e 100644 --- a/lib/spack/llnl/util/cpu/detect.py +++ b/lib/spack/llnl/util/cpu/detect.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -13,6 +13,7 @@ import warnings import six from .microarchitecture import generic_microarchitecture, targets +from .schema import targets_json #: Mapping from operating systems to chain of commands #: to obtain a dictionary of raw info on the current cpu @@ -108,21 +109,37 @@ def sysctl_info_dict(): 'model': sysctl('-n', 'machdep.cpu.model'), 'model name': sysctl('-n', 'machdep.cpu.brand_string') } + return info - # Super hacky way to deal with slight representation differences - # Would be better to somehow consider these "identical" - if 'sse4.1' in info['flags']: - info['flags'] += ' sse4_1' - if 'sse4.2' in info['flags']: - info['flags'] += ' sse4_2' - if 'avx1.0' in info['flags']: - info['flags'] += ' avx' - if 'clfsopt' in info['flags']: - info['flags'] += ' clflushopt' - if 'xsave' in info['flags']: - info['flags'] += ' xsavec xsaveopt' - return info +def adjust_raw_flags(info): + """Adjust the flags detected on the system to homogenize + slightly different representations. + """ + # Flags detected on Darwin turned to their linux counterpart + flags = info.get('flags', []) + d2l = targets_json['conversions']['darwin_flags'] + for darwin_flag, linux_flag in d2l.items(): + if darwin_flag in flags: + info['flags'] += ' ' + linux_flag + + +def adjust_raw_vendor(info): + """Adjust the vendor field to make it human readable""" + if 'CPU implementer' not in info: + return + + # Mapping numeric codes to vendor (ARM). This list is a merge from + # different sources: + # + # https://github.com/karelzak/util-linux/blob/master/sys-utils/lscpu-arm.c + # https://developer.arm.com/docs/ddi0487/latest/arm-architecture-reference-manual-armv8-for-armv8-a-architecture-profile + # https://github.com/gcc-mirror/gcc/blob/master/gcc/config/aarch64/aarch64-cores.def + # https://patchwork.kernel.org/patch/10524949/ + arm_vendors = targets_json['conversions']['arm_vendors'] + arm_code = info['CPU implementer'] + if arm_code in arm_vendors: + info['CPU implementer'] = arm_vendors[arm_code] def raw_info_dictionary(): @@ -139,6 +156,8 @@ def raw_info_dictionary(): warnings.warn(str(e)) if info: + adjust_raw_flags(info) + adjust_raw_vendor(info) break return info @@ -223,3 +242,15 @@ def compatibility_check_for_x86_64(info, target): return (target == arch_root or arch_root in target.ancestors) \ and (target.vendor == vendor or target.vendor == 'generic') \ and target.features.issubset(features) + + +@compatibility_check(architecture_family='aarch64') +def compatibility_check_for_aarch64(info, target): + basename = 'aarch64' + features = set(info.get('Features', '').split()) + vendor = info.get('CPU implementer', 'generic') + + arch_root = targets[basename] + return (target == arch_root or arch_root in target.ancestors) \ + and (target.vendor == vendor or target.vendor == 'generic') \ + and target.features.issubset(features) diff --git a/lib/spack/llnl/util/cpu/microarchitecture.py b/lib/spack/llnl/util/cpu/microarchitecture.py index 3d1590376a..f507837f89 100644 --- a/lib/spack/llnl/util/cpu/microarchitecture.py +++ b/lib/spack/llnl/util/cpu/microarchitecture.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,7 +8,7 @@ import re import warnings try: - from collections.abc import Sequence + from collections.abc import Sequence # novm except ImportError: from collections import Sequence diff --git a/lib/spack/llnl/util/cpu/microarchitectures.json b/lib/spack/llnl/util/cpu/microarchitectures.json index bb3b4db21a..51411d4a24 100644 --- a/lib/spack/llnl/util/cpu/microarchitectures.json +++ b/lib/spack/llnl/util/cpu/microarchitectures.json @@ -64,13 +64,13 @@ "clang": [ { "versions": "0.0.0-apple:", - "family": "x86-64", - "flags": "-march={family}" + "name": "x86-64", + "flags": "-march={name}" }, { "versions": ":", - "family": "x86-64", - "flags": "-march={family} -mcpu=generic" + "name": "x86-64", + "flags": "-march={name} -mtune=generic" } ], "intel": { @@ -96,8 +96,7 @@ }, "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -122,8 +121,7 @@ }, "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -157,8 +155,7 @@ ], "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -188,8 +185,7 @@ }, "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -227,8 +223,7 @@ ], "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": [ { @@ -274,8 +269,7 @@ ], "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": [ { @@ -326,8 +320,7 @@ ], "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": [ { @@ -373,8 +366,7 @@ }, "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "18.0:", @@ -416,8 +408,7 @@ }, "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "18.0:", @@ -463,8 +454,7 @@ "clang": { "versions": "3.9:", "name": "knl", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "18.0:", @@ -515,8 +505,7 @@ "clang": { "versions": "3.9:", "name": "skylake-avx512", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "18.0:", @@ -568,8 +557,7 @@ }, "clang": { "versions": "3.9:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "18.0:", @@ -609,7 +597,7 @@ "avx512bw", "avx512dq", "avx512cd", - "avx512vnni" + "avx512_vnni" ], "compilers": { "gcc": { @@ -618,8 +606,7 @@ }, "clang": { "versions": "8.0:", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "19.0:", @@ -685,13 +672,11 @@ { "versions": "7.0:", "name": "icelake-client", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, { "versions": "6.0:6.9", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" } ], "intel": { @@ -723,8 +708,7 @@ "clang": { "versions": "3.9:", "name": "amdfam10", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -761,8 +745,7 @@ "clang": { "versions": "3.9:", "name": "bdver1", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -803,8 +786,7 @@ "clang": { "versions": "3.9:", "name": "bdver2", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -846,8 +828,7 @@ "clang": { "versions": "3.9:", "name": "bdver3", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -892,8 +873,7 @@ "clang": { "versions": "3.9:", "name": "bdver4", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -942,8 +922,7 @@ "clang": { "versions": "4.0:", "name": "znver1", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -993,8 +972,7 @@ "clang": { "versions": "9.0:", "name": "znver2", - "family": "x86-64", - "flags": "-march={family} -mcpu={name}" + "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", @@ -1016,8 +994,7 @@ }, "clang": { "versions": ":", - "family": "ppc64", - "flags": "-march={family} -mcpu=generic" + "flags": "-mcpu={name} -mtune={name}" } } }, @@ -1033,9 +1010,7 @@ }, "clang": { "versions": "3.9:", - "family": "ppc64", - "name": "pwr7", - "flags": "-march={family} -mcpu={name}" + "flags": "-mcpu={name} -mtune={name}" } } }, @@ -1058,9 +1033,7 @@ ], "clang": { "versions": "3.9:", - "family": "ppc64", - "name": "pwr8", - "flags": "-march={family} -mcpu={name}" + "flags": "-mcpu={name} -mtune={name}" } } }, @@ -1076,9 +1049,7 @@ }, "clang": { "versions": "3.9:", - "family": "ppc64", - "name": "pwr9", - "flags": "-march={family} -mcpu={name}" + "flags": "-mcpu={name} -mtune={name}" } } }, @@ -1094,8 +1065,7 @@ }, "clang": { "versions": ":", - "family": "ppc64le", - "flags": "-march={family} -mcpu=generic" + "flags": "-mcpu={name} -mtune={name}" } } }, @@ -1121,8 +1091,8 @@ "clang": { "versions": "3.9:", "family": "ppc64le", - "name": "pwr8", - "flags": "-march={family} -mcpu={name}" + "name": "power8", + "flags": "-mcpu={name} -mtune={name}" } } }, @@ -1140,8 +1110,8 @@ "clang": { "versions": "3.9:", "family": "ppc64le", - "name": "pwr9", - "flags": "-march={family} -mcpu={name}" + "name": "power9", + "flags": "-mcpu={name} -mtune={name}" } } }, @@ -1156,11 +1126,113 @@ }, "clang": { "versions": ":", - "family": "aarch64", - "flags": "-march={family} -mcpu=generic" + "flags": "-march=armv8-a -mtune=generic" } } }, + "thunderx2": { + "from": "aarch64", + "vendor": "Cavium", + "features": [ + "fp", + "asimd", + "evtstrm", + "aes", + "pmull", + "sha1", + "sha2", + "crc32", + "atomics", + "cpuid", + "asimdrdm" + ], + "compilers": { + "gcc": [ + { + "versions": "4.8:4.8.9", + "flags": "-march=armv8-a" + }, + { + "versions": "4.9:5.9", + "flags": "-march=armv8-a+crc+crypto" + }, + { + "versions": "6:6.9", + "flags": "-march=armv8.1-a+crc+crypto" + }, + { + "versions": "7:", + "flags": "-mcpu=thunderx2t99" + } + ], + "clang": [ + { + "versions": "3.9:4.9", + "flags": "-march=armv8.1-a+crc+crypto" + }, + { + "versions": "5:", + "flags": "-mcpu=thunderx2t99" + } + ] + } + }, + "a64fx": { + "from": "aarch64", + "vendor": "Fujitsu", + "features": [ + "fp", + "asimd", + "evtstrm", + "aes", + "pmull", + "sha1", + "sha2", + "crc32", + "atomics", + "cpuid", + "asimdrdm", + "fphp", + "asimdhp", + "fcma", + "dcpop", + "sve" + ], + "compilers": { + "gcc": [ + { + "versions": "4.8:4.8.9", + "flags": "-march=armv8-a" + }, + { + "versions": "4.9:5.9", + "flags": "-march=armv8-a+crc+crypto" + }, + { + "versions": "6:6.9", + "flags": "-march=armv8.1-a+crc+crypto" + }, + { + "versions": "7:7.9", + "flags": "-march=armv8.2-a+crc+crypto+fp16" + }, + { + "versions": "8:", + "flags": "-march=armv8.2-a+crc+aes+sha2+fp16+sve -msve-vector-bits=512" + } + ], + "clang": [ + { + "versions": "3.9:4.9", + "flags": "-march=armv8.2-a+crc+crypto+fp16" + }, + { + "versions": "5:", + "flags": "-march=armv8.2-a+crc+crypto+fp16+sve" + } + ] + } + }, "arm": { "from": null, "vendor": "generic", @@ -1226,6 +1298,20 @@ "ppc64" ] }, + "vsx": { + "reason": "VSX alitvec extensions are supported by PowerISA from v2.06 (Power7+), but might not be listed in features", + "families": [ + "ppc64le", + "ppc64" + ] + }, + "fma": { + "reason": "FMA has been supported by PowerISA since Power1, but might not be listed in features", + "families": [ + "ppc64le", + "ppc64" + ] + }, "sse4.1": { "reason": "permits to refer to sse4_1 also as sse4.1", "any_of": [ @@ -1244,5 +1330,34 @@ "aarch64" ] } + }, + "conversions": { + "description": "Conversions that map some platform specific values to canonical values", + "arm_vendors": { + "0x41": "ARM", + "0x42": "Broadcom", + "0x43": "Cavium", + "0x44": "DEC", + "0x46": "Fujitsu", + "0x48": "HiSilicon", + "0x49": "Infineon Technologies AG", + "0x4d": "Motorola", + "0x4e": "Nvidia", + "0x50": "APM", + "0x51": "Qualcomm", + "0x53": "Samsung", + "0x56": "Marvell", + "0x61": "Apple", + "0x66": "Faraday", + "0x68": "HXT", + "0x69": "Intel" + }, + "darwin_flags": { + "sse4.1": "sse4_1", + "sse4.2": "sse4_2", + "avx1.0": "avx", + "clfsopt": "clflushopt", + "xsave": "xsavec xsaveopt" + } } } diff --git a/lib/spack/llnl/util/cpu/schema.py b/lib/spack/llnl/util/cpu/schema.py index d13d014c01..2cb456730e 100644 --- a/lib/spack/llnl/util/cpu/schema.py +++ b/lib/spack/llnl/util/cpu/schema.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,7 +6,7 @@ import json import os.path try: - from collections.abc import MutableMapping + from collections.abc import MutableMapping # novm except ImportError: from collections import MutableMapping @@ -72,7 +72,21 @@ properties = { 'additionalProperties': False } }, - + }, + 'conversions': { + 'type': 'object', + 'properties': { + 'description': { + 'type': 'string' + }, + 'arm_vendors': { + 'type': 'object', + }, + 'darwin_flags': { + 'type': 'object' + } + }, + 'additionalProperties': False } } diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index f6c8d161d7..273840a424 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -201,7 +201,6 @@ def filter_file(regex, repl, *filenames, **kwargs): output_file.writelines(input_file.readlines()) except BaseException: - os.remove(tmp_filename) # clean up the original file on failure. shutil.move(backup_filename, filename) raise @@ -1154,7 +1153,9 @@ class HeaderList(FileList): # Make sure to only match complete words, otherwise path components such # as "xinclude" will cause false matches. - include_regex = re.compile(r'(.*)(\binclude\b)(.*)') + # Avoid matching paths such as <prefix>/include/something/detail/include, + # e.g. in the CUDA Toolkit which ships internal libc++ headers. + include_regex = re.compile(r'(.*?)(\binclude\b)(.*)') def __init__(self, files): super(HeaderList, self).__init__(files) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 11820a7f58..55407992a1 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -19,12 +19,6 @@ import sys ignore_modules = [r'^\.#', '~$'] -class classproperty(property): - """classproperty decorator: like property but for classmethods.""" - def __get__(self, cls, owner): - return self.fget.__get__(None, owner)() - - def index_by(objects, *funcs): """Create a hierarchy of dictionaries by splitting the supplied set of objects on unique values of the supplied functions. @@ -612,12 +606,14 @@ def load_module_from_file(module_name, module_path): """ if sys.version_info[0] == 3 and sys.version_info[1] >= 5: import importlib.util - spec = importlib.util.spec_from_file_location(module_name, module_path) - module = importlib.util.module_from_spec(spec) + spec = importlib.util.spec_from_file_location( # novm + module_name, module_path) + module = importlib.util.module_from_spec(spec) # novm spec.loader.exec_module(module) elif sys.version_info[0] == 3 and sys.version_info[1] < 5: import importlib.machinery - loader = importlib.machinery.SourceFileLoader(module_name, module_path) + loader = importlib.machinery.SourceFileLoader( # novm + module_name, module_path) module = loader.load_module() elif sys.version_info[0] == 2: import imp diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index e9d8e72161..d29b1289e3 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 3a58093491..63f970c98b 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/multiproc.py b/lib/spack/llnl/util/multiproc.py index ff21d0f758..f87b32c307 100644 --- a/lib/spack/llnl/util/multiproc.py +++ b/lib/spack/llnl/util/multiproc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index 8581c16045..668e881b2a 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index d5e0aa8def..e0b4a5fa05 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -199,10 +199,16 @@ def colify(elts, **options): def colify_table(table, **options): """Version of ``colify()`` for data expressed in rows, (list of lists). - Same as regular colify but takes a list of lists, where each - sub-list must be the same length, and each is interpreted as a - row in a table. Regular colify displays a sequential list of - values in columns. + Same as regular colify but: + + 1. This takes a list of lists, where each sub-list must be the + same length, and each is interpreted as a row in a table. + Regular colify displays a sequential list of values in columns. + + 2. Regular colify will always print with 1 column when the output + is not a tty. This will always print with same dimensions of + the table argument. + """ if table is None: raise TypeError("Can't call colify_table on NoneType") @@ -220,6 +226,9 @@ def colify_table(table, **options): raise ValueError("Cannot override columsn in colify_table.") options['cols'] = columns + # don't reduce to 1 column for non-tty + options['tty'] = True + colify(transpose(), **options) diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py index 6789ecbdb3..79f62c0040 100644 --- a/lib/spack/llnl/util/tty/color.py +++ b/lib/spack/llnl/util/tty/color.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index b061e13b0e..10516e59d2 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 953ec89f9f..fa061756f0 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index aaccc50461..9e1ef14551 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 378fb5d5d9..38ed5baa7b 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -69,6 +69,7 @@ from llnl.util.lang import memoized, list_modules, key_ordering import spack.compiler import spack.paths import spack.error as serr +import spack.util.executable import spack.version from spack.util.naming import mod_to_class from spack.util.spack_yaml import syaml_dict @@ -214,7 +215,11 @@ class Target(object): import spack.spec if isinstance(compiler, spack.spec.CompilerSpec): compiler = spack.compilers.compilers_for_spec(compiler).pop() - compiler_version = compiler.cc_version(compiler.cc) + try: + compiler_version = compiler.cc_version(compiler.cc) + except spack.util.executable.ProcessError as e: + # log this and just return compiler.version instead + tty.debug(str(e)) return self.microarchitecture.optimization_flags( compiler.name, str(compiler_version) diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 3b10cca180..515a6166d2 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -23,7 +23,7 @@ from llnl.util.filesystem import mkdirp, install_tree import spack.cmd import spack.config as config import spack.fetch_strategy as fs -import spack.util.gpg as gpg_util +import spack.util.gpg import spack.relocate as relocate import spack.util.spack_yaml as syaml import spack.mirror @@ -33,7 +33,7 @@ import spack.util.web as web_util from spack.spec import Spec from spack.stage import Stage from spack.util.gpg import Gpg -from spack.util.executable import ProcessError +import spack.architecture as architecture _build_cache_relative_path = 'build_cache' @@ -110,14 +110,6 @@ class NewLayoutException(spack.error.SpackError): pass -def has_gnupg2(): - try: - gpg_util.Gpg.gpg()('--version', output=os.devnull) - return True - except ProcessError: - return False - - def build_cache_relative_path(): return _build_cache_relative_path @@ -243,27 +235,31 @@ def checksum_tarball(file): def sign_tarball(key, force, specfile_path): # Sign the packages if keys available - if not has_gnupg2(): + if spack.util.gpg.Gpg.gpg() is None: raise NoGpgException( "gpg2 is not available in $PATH .\n" "Use spack install gnupg and spack load gnupg.") - else: - if key is None: - keys = Gpg.signing_keys() - if len(keys) == 1: - key = keys[0] - if len(keys) > 1: - raise PickKeyException(str(keys)) - if len(keys) == 0: - msg = "No default key available for signing.\n" - msg += "Use spack gpg init and spack gpg create" - msg += " to create a default key." - raise NoKeyException(msg) + + if key is None: + keys = Gpg.signing_keys() + if len(keys) == 1: + key = keys[0] + + if len(keys) > 1: + raise PickKeyException(str(keys)) + + if len(keys) == 0: + msg = "No default key available for signing.\n" + msg += "Use spack gpg init and spack gpg create" + msg += " to create a default key." + raise NoKeyException(msg) + if os.path.exists('%s.asc' % specfile_path): if force: os.remove('%s.asc' % specfile_path) else: raise NoOverwriteException('%s.asc' % specfile_path) + Gpg.sign(key, specfile_path, '%s.asc' % specfile_path) @@ -272,7 +268,7 @@ def generate_package_index(cache_prefix): Creates (or replaces) the "index.html" page at the location given in cache_prefix. This page contains a link for each binary package (*.yaml) - and signing key (*.key) under cache_prefix. + and public key (*.key) under cache_prefix. """ tmpdir = tempfile.mkdtemp() try: @@ -664,22 +660,88 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False, shutil.rmtree(tmpdir) -#: Internal cache for get_specs -_cached_specs = None +# Internal cache for downloaded specs +_cached_specs = set() -def get_specs(force=False): +def try_download_specs(urls=None, force=False): + ''' + Try to download the urls and cache them + ''' + global _cached_specs + if urls is None: + return {} + for link in urls: + with Stage(link, name="build_cache", keep=True) as stage: + if force and os.path.exists(stage.save_filename): + os.remove(stage.save_filename) + if not os.path.exists(stage.save_filename): + try: + stage.fetch() + except fs.FetchError: + continue + with open(stage.save_filename, 'r') as f: + # read the spec from the build cache file. All specs + # in build caches are concrete (as they are built) so + # we need to mark this spec concrete on read-in. + spec = Spec.from_yaml(f) + spec._mark_concrete() + _cached_specs.add(spec) + + return _cached_specs + + +def get_spec(spec=None, force=False): """ - Get spec.yaml's for build caches available on mirror + Check if spec.yaml exists on mirrors and return it if it does """ global _cached_specs + urls = set() + if spec is None: + return {} + specfile_name = tarball_name(spec, '.spec.yaml') - if _cached_specs: - tty.debug("Using previously-retrieved specs") + if not spack.mirror.MirrorCollection(): + tty.debug("No Spack mirrors are currently configured") + return {} + + if spec in _cached_specs: return _cached_specs + for mirror in spack.mirror.MirrorCollection().values(): + fetch_url_build_cache = url_util.join( + mirror.fetch_url, _build_cache_relative_path) + + mirror_dir = url_util.local_file_path(fetch_url_build_cache) + if mirror_dir: + tty.msg("Finding buildcaches in %s" % mirror_dir) + link = url_util.join(fetch_url_build_cache, specfile_name) + urls.add(link) + + else: + tty.msg("Finding buildcaches at %s" % + url_util.format(fetch_url_build_cache)) + link = url_util.join(fetch_url_build_cache, specfile_name) + urls.add(link) + + return try_download_specs(urls=urls, force=force) + + +def get_specs(force=False, allarch=False): + """ + Get spec.yaml's for build caches available on mirror + """ + arch = architecture.Arch(architecture.platform(), + 'default_os', 'default_target') + arch_pattern = ('([^-]*-[^-]*-[^-]*)') + if not allarch: + arch_pattern = '(%s-%s-[^-]*)' % (arch.platform, arch.os) + + regex_pattern = '%s(.*)(spec.yaml$)' % (arch_pattern) + arch_re = re.compile(regex_pattern) + if not spack.mirror.MirrorCollection(): - tty.warn("No Spack mirrors are currently configured") + tty.debug("No Spack mirrors are currently configured") return {} urls = set() @@ -693,7 +755,8 @@ def get_specs(force=False): if os.path.exists(mirror_dir): files = os.listdir(mirror_dir) for file in files: - if re.search('spec.yaml', file): + m = arch_re.search(file) + if m: link = url_util.join(fetch_url_build_cache, file) urls.add(link) else: @@ -702,28 +765,11 @@ def get_specs(force=False): p, links = web_util.spider( url_util.join(fetch_url_build_cache, 'index.html')) for link in links: - if re.search("spec.yaml", link): + m = arch_re.search(link) + if m: urls.add(link) - _cached_specs = [] - for link in urls: - with Stage(link, name="build_cache", keep=True) as stage: - if force and os.path.exists(stage.save_filename): - os.remove(stage.save_filename) - if not os.path.exists(stage.save_filename): - try: - stage.fetch() - except fs.FetchError: - continue - with open(stage.save_filename, 'r') as f: - # read the spec from the build cache file. All specs - # in build caches are concrete (as they are built) so - # we need to mark this spec concrete on read-in. - spec = Spec.from_yaml(f) - spec._mark_concrete() - _cached_specs.append(spec) - - return _cached_specs + return try_download_specs(urls=urls, force=force) def get_keys(install=False, trust=False, force=False): @@ -803,10 +849,10 @@ def needs_rebuild(spec, mirror_url, rebuild_on_errors=False): try: _, _, yaml_file = web_util.read_from_url(file_path) yaml_contents = codecs.getreader('utf-8')(yaml_file).read() - except URLError as url_err: + except (URLError, web_util.SpackWebError) as url_err: err_msg = [ 'Unable to determine whether {0} needs rebuilding,', - ' caught URLError attempting to read from {1}.', + ' caught exception attempting to read from {1}.', ] tty.error(''.join(err_msg).format(spec.short_spec, file_path)) tty.debug(url_err) @@ -908,11 +954,16 @@ def _download_buildcache_entry(mirror_root, descriptions): return True -def download_buildcache_entry(file_descriptions): - if not spack.mirror.MirrorCollection(): - tty.die("Please add a spack mirror to allow " + +def download_buildcache_entry(file_descriptions, mirror_url=None): + if not mirror_url and not spack.mirror.MirrorCollection(): + tty.die("Please provide or add a spack mirror to allow " + "download of buildcache entries.") + if mirror_url: + mirror_root = os.path.join( + mirror_url, _build_cache_relative_path) + return _download_buildcache_entry(mirror_root, file_descriptions) + for mirror in spack.mirror.MirrorCollection().values(): mirror_root = os.path.join( mirror.fetch_url, diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 13ee99d177..4809a1010e 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -39,7 +39,6 @@ import shutil import sys import traceback import types -from six import iteritems from six import StringIO import llnl.util.tty as tty @@ -52,6 +51,7 @@ import spack.build_systems.meson import spack.config import spack.main import spack.paths +import spack.schema.environment import spack.store from spack.util.string import plural from spack.util.environment import ( @@ -342,21 +342,7 @@ def set_build_environment_variables(pkg, env, dirty): # Set environment variables if specified for # the given compiler compiler = pkg.compiler - environment = compiler.environment - - for command, variable in iteritems(environment): - if command == 'set': - for name, value in iteritems(variable): - env.set(name, value) - elif command == 'unset': - for name, _ in iteritems(variable): - env.unset(name) - elif command == 'prepend-path': - for name, value in iteritems(variable): - env.prepend_path(name, value) - elif command == 'append-path': - for name, value in iteritems(variable): - env.append_path(name, value) + env.extend(spack.schema.environment.parse(compiler.environment)) if compiler.extra_rpaths: extra_rpaths = ':'.join(compiler.extra_rpaths) diff --git a/lib/spack/spack/build_systems/__init__.py b/lib/spack/spack/build_systems/__init__.py index 94f8ac4d9e..9f87532b85 100644 --- a/lib/spack/spack/build_systems/__init__.py +++ b/lib/spack/spack/build_systems/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/aspell_dict.py b/lib/spack/spack/build_systems/aspell_dict.py index e9019a1948..7da4d2f10c 100644 --- a/lib/spack/spack/build_systems/aspell_dict.py +++ b/lib/spack/spack/build_systems/aspell_dict.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/autotools.py b/lib/spack/spack/build_systems/autotools.py index ce0e40b34c..c21b8dad71 100644 --- a/lib/spack/spack/build_systems/autotools.py +++ b/lib/spack/spack/build_systems/autotools.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/cmake.py b/lib/spack/spack/build_systems/cmake.py index 0c6a166409..14f33e94e6 100644 --- a/lib/spack/spack/build_systems/cmake.py +++ b/lib/spack/spack/build_systems/cmake.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/cuda.py b/lib/spack/spack/build_systems/cuda.py index b75807fd06..fc96cffe60 100644 --- a/lib/spack/spack/build_systems/cuda.py +++ b/lib/spack/spack/build_systems/cuda.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -61,12 +61,13 @@ class CudaPackage(PackageBase): conflicts('%gcc@6:', when='+cuda ^cuda@:8' + arch_platform) conflicts('%gcc@7:', when='+cuda ^cuda@:9.1' + arch_platform) conflicts('%gcc@8:', when='+cuda ^cuda@:10.0.130' + arch_platform) - conflicts('%gcc@9:', when='+cuda ^cuda@:10.1.243' + arch_platform) + conflicts('%gcc@9:', when='+cuda ^cuda@:10.2.89' + arch_platform) conflicts('%pgi@:14.8', when='+cuda ^cuda@:7.0.27' + arch_platform) conflicts('%pgi@:15.3,15.5:', when='+cuda ^cuda@7.5' + arch_platform) conflicts('%pgi@:16.2,16.0:16.3', when='+cuda ^cuda@8' + arch_platform) conflicts('%pgi@:15,18:', when='+cuda ^cuda@9.0:9.1' + arch_platform) conflicts('%pgi@:16', when='+cuda ^cuda@9.2.88:10' + arch_platform) + conflicts('%pgi@:17', when='+cuda ^cuda@10.2.89' + arch_platform) conflicts('%clang@:3.4', when='+cuda ^cuda@:7.5' + arch_platform) conflicts('%clang@:3.7,4:', when='+cuda ^cuda@8.0:9.0' + arch_platform) @@ -74,6 +75,10 @@ class CudaPackage(PackageBase): when='+cuda ^cuda@9.1' + arch_platform) conflicts('%clang@:3.7,5.1:', when='+cuda ^cuda@9.2' + arch_platform) conflicts('%clang@:3.7,6.1:', when='+cuda ^cuda@10.0.130' + arch_platform) + conflicts('%clang@:3.7,7.1:', when='+cuda ^cuda@10.1.105' + arch_platform) + conflicts('%clang@:3.7,8.1:', + when='+cuda ^cuda@10.1.105:10.1.243' + arch_platform) + conflicts('%clang@:3.2,9.0:', when='+cuda ^cuda@10.2.89' + arch_platform) # x86_64 vs. ppc64le differ according to NVidia docs # Linux ppc64le compiler conflicts from Table from the docs below: @@ -95,6 +100,8 @@ class CudaPackage(PackageBase): conflicts('%clang@5:', when='+cuda ^cuda@:9.1' + arch_platform) conflicts('%clang@6:', when='+cuda ^cuda@:9.2' + arch_platform) conflicts('%clang@7:', when='+cuda ^cuda@10.0.130' + arch_platform) + conflicts('%clang@7.1:', when='+cuda ^cuda@:10.1.105' + arch_platform) + conflicts('%clang@8.1:', when='+cuda ^cuda@:10.2.89' + arch_platform) # Intel is mostly relevant for x86_64 Linux, even though it also # exists for Mac OS X. No information prior to CUDA 3.2 or Intel 11.1 @@ -107,12 +114,12 @@ class CudaPackage(PackageBase): conflicts('%intel@16.0:', when='+cuda ^cuda@:8.0.43') conflicts('%intel@17.0:', when='+cuda ^cuda@:8.0.60') conflicts('%intel@18.0:', when='+cuda ^cuda@:9.9') - conflicts('%intel@19.0:', when='+cuda ^cuda@:10.0') + conflicts('%intel@19.0:', when='+cuda ^cuda@:10.2.89') # XL is mostly relevant for ppc64le Linux conflicts('%xl@:12,14:', when='+cuda ^cuda@:9.1') conflicts('%xl@:12,14:15,17:', when='+cuda ^cuda@9.2') - conflicts('%xl@17:', when='+cuda ^cuda@10.0.130') + conflicts('%xl@17:', when='+cuda ^cuda@10.0.130:10.2.89') # Mac OS X # platform = ' platform=darwin' diff --git a/lib/spack/spack/build_systems/gnu.py b/lib/spack/spack/build_systems/gnu.py new file mode 100644 index 0000000000..c5b5e74bb9 --- /dev/null +++ b/lib/spack/spack/build_systems/gnu.py @@ -0,0 +1,37 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.util.url +import spack.package + + +class GNUMirrorPackage(spack.package.PackageBase): + """Mixin that takes care of setting url and mirrors for GNU packages.""" + #: Path of the package in a GNU mirror + gnu_mirror_path = None + + #: List of GNU mirrors used by Spack + base_mirrors = [ + 'https://ftpmirror.gnu.org/', + 'https://ftp.gnu.org/gnu/', + # Fall back to http if https didn't work (for instance because + # Spack is bootstrapping curl) + 'http://ftpmirror.gnu.org/' + ] + + @property + def urls(self): + self._ensure_gnu_mirror_path_is_set_or_raise() + return [ + spack.util.url.join(m, self.gnu_mirror_path, resolve_href=True) + for m in self.base_mirrors + ] + + def _ensure_gnu_mirror_path_is_set_or_raise(self): + if self.gnu_mirror_path is None: + cls_name = type(self).__name__ + msg = ('{0} must define a `gnu_mirror_path` attribute' + ' [none defined]') + raise AttributeError(msg.format(cls_name)) diff --git a/lib/spack/spack/build_systems/intel.py b/lib/spack/spack/build_systems/intel.py index 858b4e37fb..4017f2e526 100644 --- a/lib/spack/spack/build_systems/intel.py +++ b/lib/spack/spack/build_systems/intel.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -89,7 +89,7 @@ class IntelPackage(PackageBase): 2. :py:meth:`~.IntelPackage.install` They both have sensible defaults and for many packages the - only thing necessary will be to override setup_environment + only thing necessary will be to override setup_run_environment to set the appropriate environment variables. """ #: Phases of an Intel package @@ -455,9 +455,7 @@ class IntelPackage(PackageBase): break if not matching_dirs: - # No match -- this *will* happen during pre-build call to - # setup_environment() when the destination dir is still empty. - # Return a sensible value anyway. + # No match -- return a sensible value anyway. d = unversioned_dirname debug_print(d) @@ -786,7 +784,8 @@ class IntelPackage(PackageBase): debug_print(mkl_libs) if len(mkl_libs) < 3: - raise_lib_error('Cannot locate core MKL libraries:', mkl_libnames) + raise_lib_error('Cannot locate core MKL libraries:', mkl_libnames, + 'in:', self.component_lib_dir('mkl')) # The Intel MKL link line advisor recommends these system libraries system_libs = find_system_libraries( @@ -888,15 +887,15 @@ class IntelPackage(PackageBase): # debug_print("wrapper_vars =", wrapper_vars) return wrapper_vars - def mpi_setup_dependent_environment( - self, spack_env, run_env, dependent_spec, compilers_of_client={}): - '''Unified back-end for setup_dependent_environment() of Intel packages - that provide 'mpi'. + def mpi_setup_dependent_build_environment( + self, env, dependent_spec, compilers_of_client={}): + '''Unified back-end for setup_dependent_build_environment() of + Intel packages that provide 'mpi'. Parameters: - spack_env, run_env, dependent_spec: same as in - setup_dependent_environment(). + env, dependent_spec: same as in + setup_dependent_build_environment(). compilers_of_client (dict): Conveys spack_cc, spack_cxx, etc., from the scope of dependent packages; constructed in caller. @@ -938,12 +937,12 @@ class IntelPackage(PackageBase): # Ensure that the directory containing the compiler wrappers is in the # PATH. Spack packages add `prefix.bin` to their dependents' paths, # but because of the intel directory hierarchy that is insufficient. - spack_env.prepend_path('PATH', os.path.dirname(wrapper_vars['MPICC'])) + env.prepend_path('PATH', os.path.dirname(wrapper_vars['MPICC'])) for key, value in wrapper_vars.items(): - spack_env.set(key, value) + env.set(key, value) - debug_print("adding to spack_env:", wrapper_vars) + debug_print("adding to build env:", wrapper_vars) # --------------------------------------------------------------------- # General support for child packages @@ -994,7 +993,7 @@ class IntelPackage(PackageBase): debug_print(result) return result - def setup_environment(self, spack_env, run_env): + def setup_run_environment(self, env): """Adds environment variables to the generated module file. These environment variables come from running: @@ -1004,24 +1003,7 @@ class IntelPackage(PackageBase): $ source parallel_studio_xe_2017/bin/psxevars.sh intel64 [and likewise for MKL, MPI, and other components] """ - # https://spack.readthedocs.io/en/latest/spack.html#spack.package.PackageBase.setup_environment - # - # spack_env -> Applied when dependent is built within Spack. - # Not used here. - # run_env -> Applied to the modulefile of dependent. - # - # NOTE: Spack runs setup_environment twice, once pre-build to set up - # the build environment, and once post-installation to determine - # the environment variables needed at run-time to add to the module - # file. The script we need to source is only present post-installation, - # so check for its existence before sourcing. - # TODO: At some point we should split setup_environment into - # setup_build_environment and setup_run_environment to get around - # this problem. f = self.file_to_source - if not f or not os.path.isfile(f): - return - tty.debug("sourcing " + f) # All Intel packages expect at least the architecture as argument. @@ -1033,15 +1015,9 @@ class IntelPackage(PackageBase): # if sys.platform == 'darwin': # args = () - run_env.extend(EnvironmentModifications.from_sourcing_file(f, *args)) + env.extend(EnvironmentModifications.from_sourcing_file(f, *args)) - def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - # https://spack.readthedocs.io/en/latest/spack.html#spack.package.PackageBase.setup_dependent_environment - # - # spack_env -> Applied when dependent is built within Spack. - # run_env -> Applied to the modulefile of dependent. - # Not used here. - # + def setup_dependent_build_environment(self, env, dependent_spec): # NB: This function is overwritten by 'mpi' provider packages: # # var/spack/repos/builtin/packages/intel-mpi/package.py @@ -1051,18 +1027,20 @@ class IntelPackage(PackageBase): # dictionary kwarg compilers_of_client{} present and populated. # Handle everything in a callback version. - self._setup_dependent_env_callback(spack_env, run_env, dependent_spec) + self._setup_dependent_env_callback(env, dependent_spec) def _setup_dependent_env_callback( - self, spack_env, run_env, dependent_spec, compilers_of_client={}): - # Expected to be called from a client's setup_dependent_environment(), + self, env, dependent_spec, compilers_of_client={}): + # Expected to be called from a client's + # setup_dependent_build_environment(), # with args extended to convey the client's compilers as needed. if '+mkl' in self.spec or self.provides('mkl'): # Spack's env philosophy demands that we replicate some of the # settings normally handled by file_to_source ... # - # TODO: Why is setup_environment() [which uses file_to_source()] + # TODO: Why is setup_run_environment() + # [which uses file_to_source()] # not called as a matter of course upon entering the current # function? (guarding against multiple calls notwithstanding) # @@ -1070,18 +1048,26 @@ class IntelPackage(PackageBase): env_mods = { 'MKLROOT': self.normalize_path('mkl'), 'SPACK_COMPILER_EXTRA_RPATHS': self.component_lib_dir('mkl'), + 'CMAKE_PREFIX_PATH': self.normalize_path('mkl'), + 'CMAKE_LIBRARY_PATH': self.component_lib_dir('mkl'), + 'CMAKE_INCLUDE_PATH': self.component_include_dir('mkl'), } - spack_env.set('MKLROOT', env_mods['MKLROOT']) - spack_env.append_path('SPACK_COMPILER_EXTRA_RPATHS', - env_mods['SPACK_COMPILER_EXTRA_RPATHS']) + env.set('MKLROOT', env_mods['MKLROOT']) + env.append_path('SPACK_COMPILER_EXTRA_RPATHS', + env_mods['SPACK_COMPILER_EXTRA_RPATHS']) + env.append_path('CMAKE_PREFIX_PATH', env_mods['CMAKE_PREFIX_PATH']) + env.append_path('CMAKE_LIBRARY_PATH', + env_mods['CMAKE_LIBRARY_PATH']) + env.append_path('CMAKE_INCLUDE_PATH', + env_mods['CMAKE_INCLUDE_PATH']) - debug_print("adding/modifying spack_env:", env_mods) + debug_print("adding/modifying build env:", env_mods) if '+mpi' in self.spec or self.provides('mpi'): if compilers_of_client: - self.mpi_setup_dependent_environment( - spack_env, run_env, dependent_spec, compilers_of_client) + self.mpi_setup_dependent_build_environment( + env, dependent_spec, compilers_of_client) # We could forego this nonce function and inline its code here, # but (a) it sisters mpi_compiler_wrappers() [needed twice] # which performs dizzyingly similar but necessarily different diff --git a/lib/spack/spack/build_systems/makefile.py b/lib/spack/spack/build_systems/makefile.py index 3f8d5bf1be..6cd05c7ad9 100644 --- a/lib/spack/spack/build_systems/makefile.py +++ b/lib/spack/spack/build_systems/makefile.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/meson.py b/lib/spack/spack/build_systems/meson.py index 646c3a18f1..825b4c98c3 100644 --- a/lib/spack/spack/build_systems/meson.py +++ b/lib/spack/spack/build_systems/meson.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/octave.py b/lib/spack/spack/build_systems/octave.py index 677e623bd6..16d6a0a1b8 100644 --- a/lib/spack/spack/build_systems/octave.py +++ b/lib/spack/spack/build_systems/octave.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -29,12 +29,11 @@ class OctavePackage(PackageBase): extends('octave') depends_on('octave', type=('build', 'run')) - def setup_environment(self, spack_env, run_env): - """Set up the compile and runtime environments for a package.""" + def setup_build_environment(self, env): # octave does not like those environment variables to be set: - spack_env.unset('CC') - spack_env.unset('CXX') - spack_env.unset('FC') + env.unset('CC') + env.unset('CXX') + env.unset('FC') def install(self, spec, prefix): """Install the package from the archive file""" diff --git a/lib/spack/spack/build_systems/perl.py b/lib/spack/spack/build_systems/perl.py index 28f1fc5608..d134dc06f2 100644 --- a/lib/spack/spack/build_systems/perl.py +++ b/lib/spack/spack/build_systems/perl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py index 6dee28e4ed..5a054a77b4 100644 --- a/lib/spack/spack/build_systems/python.py +++ b/lib/spack/spack/build_systems/python.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/qmake.py b/lib/spack/spack/build_systems/qmake.py index 06a5f2c431..22914d4d3a 100644 --- a/lib/spack/spack/build_systems/qmake.py +++ b/lib/spack/spack/build_systems/qmake.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/r.py b/lib/spack/spack/build_systems/r.py index d97bde4374..6ee2668833 100644 --- a/lib/spack/spack/build_systems/r.py +++ b/lib/spack/spack/build_systems/r.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/scons.py b/lib/spack/spack/build_systems/scons.py index be0f06cd28..ffa4390249 100644 --- a/lib/spack/spack/build_systems/scons.py +++ b/lib/spack/spack/build_systems/scons.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/sip.py b/lib/spack/spack/build_systems/sip.py index da2c770ba7..314f91d5d2 100644 --- a/lib/spack/spack/build_systems/sip.py +++ b/lib/spack/spack/build_systems/sip.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/build_systems/waf.py b/lib/spack/spack/build_systems/waf.py index b46b37aba9..6bf9a432e0 100644 --- a/lib/spack/spack/build_systems/waf.py +++ b/lib/spack/spack/build_systems/waf.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index ce2f8a6371..98fa8d5795 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/release_jobs.py b/lib/spack/spack/ci.py index 44fcc8511a..ed06524073 100644 --- a/lib/spack/spack/cmd/release_jobs.py +++ b/lib/spack/spack/ci.py @@ -1,10 +1,14 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import base64 +import datetime import json +import os +import shutil +import tempfile import zlib from six import iteritems @@ -14,31 +18,36 @@ from six.moves.urllib.request import build_opener, HTTPHandler, Request import llnl.util.tty as tty -import spack.environment as ev +import spack +import spack.binary_distribution as bindist +import spack.cmd.buildcache as buildcache import spack.compilers as compilers +import spack.config as cfg +import spack.environment as ev from spack.dependency import all_deptypes from spack.error import SpackError import spack.hash_types as ht +from spack.main import SpackCommand +import spack.repo from spack.spec import Spec import spack.util.spack_yaml as syaml +import spack.util.web as web_util -description = "generate release build set as .gitlab-ci.yml" -section = "build" -level = "long" +spack_gpg = SpackCommand('gpg') +spack_compiler = SpackCommand('compiler') -def setup_parser(subparser): - subparser.add_argument( - '-o', '--output-file', default=".gitlab-ci.yml", - help="path to output file to write") - subparser.add_argument( - '-p', '--print-summary', action='store_true', default=False, - help="Print summary of staged jobs to standard output") +class TemporaryDirectory(object): + def __init__(self): + self.temporary_directory = tempfile.mkdtemp() - subparser.add_argument( - '--cdash-credentials', default=None, - help="Path to file containing CDash authentication token") + def __enter__(self): + return self.temporary_directory + + def __exit__(self, exc_type, exc_value, exc_traceback): + shutil.rmtree(self.temporary_directory) + return False def _create_buildgroup(opener, headers, url, project, group_name, group_type): @@ -131,6 +140,10 @@ def get_job_name(phase, strip_compiler, spec, osarch, build_group): format_args.append(spec.name) item_idx += 1 + format_str += '/{{{0}}}'.format(item_idx) + format_args.append(spec.dag_hash(7)) + item_idx += 1 + format_str += ' {{{0}}}'.format(item_idx) format_args.append(spec.version) item_idx += 1 @@ -368,6 +381,10 @@ def compute_spec_deps(spec_list): rkey, rlabel = spec_deps_key_label(spec) for s in spec.traverse(deptype=deptype): + if s.external: + tty.msg('Will not stage external pkg: {0}'.format(s)) + continue + skey, slabel = spec_deps_key_label(s) spec_labels[slabel] = { 'spec': get_spec_string(s), @@ -377,6 +394,10 @@ def compute_spec_deps(spec_list): for d in s.dependencies(deptype=deptype): dkey, dlabel = spec_deps_key_label(d) + if d.external: + tty.msg('Will not stage external dep: {0}'.format(d)) + continue + append_dep(slabel, dlabel) for l, d in spec_labels.items(): @@ -406,15 +427,28 @@ def find_matching_config(spec, ci_mappings): return None -def release_jobs(parser, args): - env = ev.get_env(args, 'release-jobs', required=True) +def pkg_name_from_spec_label(spec_label): + return spec_label[:spec_label.index('/')] + + +def generate_gitlab_ci_yaml(env, print_summary, output_file, + custom_spack_repo=None, custom_spack_ref=None): + # FIXME: What's the difference between one that opens with 'spack' + # and one that opens with 'env'? This will only handle the former. + with spack.concretize.disable_compiler_existence_check(): + env.concretize() yaml_root = ev.config_dict(env.yaml) if 'gitlab-ci' not in yaml_root: tty.die('Environment yaml does not have "gitlab-ci" section') - ci_mappings = yaml_root['gitlab-ci']['mappings'] + gitlab_ci = yaml_root['gitlab-ci'] + ci_mappings = gitlab_ci['mappings'] + + final_job_config = None + if 'final-stage-rebuild-index' in gitlab_ci: + final_job_config = gitlab_ci['final-stage-rebuild-index'] build_group = None enable_cdash_reporting = False @@ -426,23 +460,40 @@ def release_jobs(parser, args): build_group = ci_cdash['build-group'] cdash_url = ci_cdash['url'] cdash_project = ci_cdash['project'] - proj_enc = urlencode({'project': cdash_project}) - eq_idx = proj_enc.find('=') + 1 - cdash_project_enc = proj_enc[eq_idx:] cdash_site = ci_cdash['site'] - if args.cdash_credentials: - with open(args.cdash_credentials) as fd: - cdash_auth_token = fd.read() - cdash_auth_token = cdash_auth_token.strip() + if 'SPACK_CDASH_AUTH_TOKEN' in os.environ: + tty.verbose("Using CDash auth token from environment") + cdash_auth_token = os.environ.get('SPACK_CDASH_AUTH_TOKEN') + + # Make sure we use a custom spack if necessary + before_script = None + after_script = None + if custom_spack_repo: + if not custom_spack_ref: + custom_spack_ref = 'master' + before_script = [ + ('git clone "{0}" --branch "{1}" --depth 1 ' + '--single-branch'.format(custom_spack_repo, custom_spack_ref)), + # Next line just shows spack version in pipeline output + 'pushd ./spack && git rev-parse HEAD && popd', + '. "./spack/share/spack/setup-env.sh"', + ] + after_script = [ + 'rm -rf "./spack"' + ] ci_mirrors = yaml_root['mirrors'] mirror_urls = [url for url in ci_mirrors.values()] + enable_artifacts_buildcache = False + if 'enable-artifacts-buildcache' in gitlab_ci: + enable_artifacts_buildcache = gitlab_ci['enable-artifacts-buildcache'] + bootstrap_specs = [] phases = [] - if 'bootstrap' in yaml_root['gitlab-ci']: - for phase in yaml_root['gitlab-ci']['bootstrap']: + if 'bootstrap' in gitlab_ci: + for phase in gitlab_ci['bootstrap']: try: phase_name = phase.get('name') strip_compilers = phase.get('compiler-agnostic') @@ -469,9 +520,11 @@ def release_jobs(parser, args): staged_phases = {} for phase in phases: phase_name = phase['name'] - staged_phases[phase_name] = stage_spec_jobs(env.spec_lists[phase_name]) + with spack.concretize.disable_compiler_existence_check(): + staged_phases[phase_name] = stage_spec_jobs( + env.spec_lists[phase_name]) - if args.print_summary: + if print_summary: for phase in phases: phase_name = phase['name'] tty.msg('Stages for phase "{0}"'.format(phase_name)) @@ -498,10 +551,12 @@ def release_jobs(parser, args): stage_id += 1 for spec_label in stage_jobs: - release_spec = spec_labels[spec_label]['spec'] root_spec = spec_labels[spec_label]['rootSpec'] + pkg_name = pkg_name_from_spec_label(spec_label) + release_spec = root_spec[pkg_name] - runner_attribs = find_matching_config(root_spec, ci_mappings) + runner_attribs = find_matching_config( + release_spec, ci_mappings) if not runner_attribs: tty.warn('No match found for {0}, skipping it'.format( @@ -529,7 +584,11 @@ def release_jobs(parser, args): job_name = get_job_name(phase_name, strip_compilers, release_spec, osname, build_group) - job_scripts = ['./bin/rebuild-package.sh'] + debug_flag = '' + if 'enable-debug-messages' in gitlab_ci: + debug_flag = '-d ' + + job_scripts = ['spack {0}ci rebuild'.format(debug_flag)] compiler_action = 'NONE' if len(phases) > 1: @@ -538,7 +597,6 @@ def release_jobs(parser, args): compiler_action = 'INSTALL_MISSING' job_vars = { - 'SPACK_MIRROR_URL': mirror_urls[0], 'SPACK_ROOT_SPEC': format_root_spec( root_spec, main_phase, strip_compilers), 'SPACK_JOB_SPEC_PKG_NAME': release_spec.name, @@ -547,11 +605,13 @@ def release_jobs(parser, args): job_dependencies = [] if spec_label in dependencies: - job_dependencies = ( - [get_job_name(phase_name, strip_compilers, - spec_labels[dep_label]['spec'], - osname, build_group) - for dep_label in dependencies[spec_label]]) + for dep_label in dependencies[spec_label]: + dep_pkg = pkg_name_from_spec_label(dep_label) + dep_spec = spec_labels[dep_label]['rootSpec'][dep_pkg] + dep_job_name = get_job_name( + phase_name, strip_compilers, dep_spec, osname, + build_group) + job_dependencies.append(dep_job_name) # This next section helps gitlab make sure the right # bootstrapped compiler exists in the artifacts buildcache by @@ -585,34 +645,38 @@ def release_jobs(parser, args): [spec_labels[d]['spec'].name for d in dependencies[spec_label]]) - job_vars['SPACK_CDASH_BASE_URL'] = cdash_url - job_vars['SPACK_CDASH_PROJECT'] = cdash_project - job_vars['SPACK_CDASH_PROJECT_ENC'] = cdash_project_enc job_vars['SPACK_CDASH_BUILD_NAME'] = cdash_build_name - job_vars['SPACK_CDASH_SITE'] = cdash_site - job_vars['SPACK_RELATED_BUILDS'] = ';'.join(related_builds) - job_vars['SPACK_JOB_SPEC_BUILDGROUP'] = build_group - - job_vars['SPACK_ENABLE_CDASH'] = str(enable_cdash_reporting) + job_vars['SPACK_RELATED_BUILDS_CDASH'] = ';'.join( + related_builds) variables.update(job_vars) + artifact_paths = [ + 'jobs_scratch_dir', + 'cdash_report', + ] + + if enable_artifacts_buildcache: + artifact_paths.append('local_mirror/build_cache') + job_object = { 'stage': stage_name, 'variables': variables, 'script': job_scripts, 'tags': tags, 'artifacts': { - 'paths': [ - 'jobs_scratch_dir', - 'cdash_report', - 'local_mirror/build_cache', - ], + 'paths': artifact_paths, 'when': 'always', }, 'dependencies': job_dependencies, } + if before_script: + job_object['before_script'] = before_script + + if after_script: + job_object['after_script'] = after_script + if image_name: job_object['image'] = image_name if image_entry is not None: @@ -624,7 +688,7 @@ def release_jobs(parser, args): output_object[job_name] = job_object job_id += 1 - tty.msg('{0} build jobs generated in {1} stages'.format( + tty.debug('{0} build jobs generated in {1} stages'.format( job_id, stage_id)) # Use "all_job_names" to populate the build group for this set @@ -637,20 +701,280 @@ def release_jobs(parser, args): else: tty.warn('Unable to populate buildgroup without CDash credentials') - # Add an extra, final job to regenerate the index - final_stage = 'stage-rebuild-index' - final_job = { - 'stage': final_stage, - 'variables': { - 'MIRROR_URL': mirror_urls[0], - }, - 'script': './bin/rebuild-index.sh', - 'tags': ['spack-post-ci'] # may want a runner to handle this - } - output_object['rebuild-index'] = final_job - stage_names.append(final_stage) + if final_job_config: + # Add an extra, final job to regenerate the index + final_stage = 'stage-rebuild-index' + final_job = { + 'stage': final_stage, + 'script': 'spack buildcache update-index -d {0}'.format( + mirror_urls[0]), + 'tags': final_job_config['tags'] + } + if 'image' in final_job_config: + final_job['image'] = final_job_config['image'] + if before_script: + final_job['before_script'] = before_script + if after_script: + final_job['after_script'] = after_script + output_object['rebuild-index'] = final_job + stage_names.append(final_stage) output_object['stages'] = stage_names - with open(args.output_file, 'w') as outf: + with open(output_file, 'w') as outf: outf.write(syaml.dump_config(output_object, default_flow_style=True)) + + +def url_encode_string(input_string): + encoded_keyval = urlencode({'donotcare': input_string}) + eq_idx = encoded_keyval.find('=') + 1 + encoded_value = encoded_keyval[eq_idx:] + return encoded_value + + +def import_signing_key(base64_signing_key): + if not base64_signing_key: + tty.warn('No key found for signing/verifying packages') + return + + tty.debug('ci.import_signing_key() will attempt to import a key') + + # This command has the side-effect of creating the directory referred + # to as GNUPGHOME in setup_environment() + list_output = spack_gpg('list', output=str) + + tty.debug('spack gpg list:') + tty.debug(list_output) + + decoded_key = base64.b64decode(base64_signing_key) + if isinstance(decoded_key, bytes): + decoded_key = decoded_key.decode('utf8') + + with TemporaryDirectory() as tmpdir: + sign_key_path = os.path.join(tmpdir, 'signing_key') + with open(sign_key_path, 'w') as fd: + fd.write(decoded_key) + + key_import_output = spack_gpg('trust', sign_key_path, output=str) + tty.debug('spack gpg trust {0}'.format(sign_key_path)) + tty.debug(key_import_output) + + # Now print the keys we have for verifying and signing + trusted_keys_output = spack_gpg('list', '--trusted', output=str) + signing_keys_output = spack_gpg('list', '--signing', output=str) + + tty.debug('spack gpg list --trusted') + tty.debug(trusted_keys_output) + tty.debug('spack gpg list --signing') + tty.debug(signing_keys_output) + + +def configure_compilers(compiler_action, scope=None): + if compiler_action == 'INSTALL_MISSING': + tty.debug('Make sure bootstrapped compiler will be installed') + config = cfg.get('config') + config['install_missing_compilers'] = True + cfg.set('config', config) + elif compiler_action == 'FIND_ANY': + tty.debug('Just find any available compiler') + find_args = ['find'] + if scope: + find_args.extend(['--scope', scope]) + output = spack_compiler(*find_args) + tty.debug('spack compiler find') + tty.debug(output) + output = spack_compiler('list') + tty.debug('spack compiler list') + tty.debug(output) + else: + tty.debug('No compiler action to be taken') + + return None + + +def get_concrete_specs(root_spec, job_name, related_builds, compiler_action): + spec_map = { + 'root': None, + 'deps': {}, + } + + if compiler_action == 'FIND_ANY': + # This corresponds to a bootstrapping phase where we need to + # rely on any available compiler to build the package (i.e. the + # compiler needed to be stripped from the spec when we generated + # the job), and thus we need to concretize the root spec again. + tty.debug('About to concretize {0}'.format(root_spec)) + concrete_root = Spec(root_spec).concretized() + tty.debug('Resulting concrete root: {0}'.format(concrete_root)) + else: + # in this case, either we're relying on Spack to install missing + # compiler bootstrapped in a previous phase, or else we only had one + # phase (like a site which already knows what compilers are available + # on it's runners), so we don't want to concretize that root spec + # again. The reason we take this path in the first case (bootstrapped + # compiler), is that we can't concretize a spec at this point if we're + # going to ask spack to "install_missing_compilers". + concrete_root = Spec.from_yaml( + str(zlib.decompress(base64.b64decode(root_spec)).decode('utf-8'))) + + spec_map['root'] = concrete_root + spec_map[job_name] = concrete_root[job_name] + + if related_builds: + for dep_job_name in related_builds.split(';'): + spec_map['deps'][dep_job_name] = concrete_root[dep_job_name] + + return spec_map + + +def register_cdash_build(build_name, base_url, project, site, track): + url = base_url + '/api/v1/addBuild.php' + time_stamp = datetime.datetime.now().strftime('%Y%m%d-%H%M') + build_stamp = '{0}-{1}'.format(time_stamp, track) + payload = { + "project": project, + "site": site, + "name": build_name, + "stamp": build_stamp, + } + + tty.debug('Registering cdash build to {0}, payload:'.format(url)) + tty.debug(payload) + + enc_data = json.dumps(payload).encode('utf-8') + + headers = { + 'Content-Type': 'application/json', + } + + opener = build_opener(HTTPHandler) + + request = Request(url, data=enc_data, headers=headers) + + response = opener.open(request) + response_code = response.getcode() + + if response_code != 200 and response_code != 201: + msg = 'Adding build failed (response code = {0}'.format(response_code) + raise SpackError(msg) + + response_text = response.read() + response_json = json.loads(response_text) + build_id = response_json['buildid'] + + return (build_id, build_stamp) + + +def relate_cdash_builds(spec_map, cdash_base_url, job_build_id, cdash_project, + cdashids_mirror_url): + if not job_build_id: + return + + dep_map = spec_map['deps'] + + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + + cdash_api_url = '{0}/api/v1/relateBuilds.php'.format(cdash_base_url) + + for dep_pkg_name in dep_map: + tty.debug('Fetching cdashid file for {0}'.format(dep_pkg_name)) + dep_spec = dep_map[dep_pkg_name] + dep_build_id = read_cdashid_from_mirror(dep_spec, cdashids_mirror_url) + + payload = { + "project": cdash_project, + "buildid": job_build_id, + "relatedid": dep_build_id, + "relationship": "depends on" + } + + enc_data = json.dumps(payload).encode('utf-8') + + opener = build_opener(HTTPHandler) + + request = Request(cdash_api_url, data=enc_data, headers=headers) + + response = opener.open(request) + response_code = response.getcode() + + if response_code != 200 and response_code != 201: + msg = 'Relate builds ({0} -> {1}) failed (resp code = {2})'.format( + job_build_id, dep_build_id, response_code) + raise SpackError(msg) + + response_text = response.read() + tty.debug('Relate builds response: {0}'.format(response_text)) + + +def write_cdashid_to_mirror(cdashid, spec, mirror_url): + if not spec.concrete: + tty.die('Can only write cdashid for concrete spec to mirror') + + with TemporaryDirectory() as tmpdir: + local_cdash_path = os.path.join(tmpdir, 'job.cdashid') + with open(local_cdash_path, 'w') as fd: + fd.write(cdashid) + + buildcache_name = bindist.tarball_name(spec, '') + cdashid_file_name = '{0}.cdashid'.format(buildcache_name) + remote_url = os.path.join( + mirror_url, bindist.build_cache_relative_path(), cdashid_file_name) + + tty.debug('pushing cdashid to url') + tty.debug(' local file path: {0}'.format(local_cdash_path)) + tty.debug(' remote url: {0}'.format(remote_url)) + web_util.push_to_url(local_cdash_path, remote_url) + + +def read_cdashid_from_mirror(spec, mirror_url): + if not spec.concrete: + tty.die('Can only read cdashid for concrete spec from mirror') + + buildcache_name = bindist.tarball_name(spec, '') + cdashid_file_name = '{0}.cdashid'.format(buildcache_name) + url = os.path.join( + mirror_url, bindist.build_cache_relative_path(), cdashid_file_name) + + resp_url, resp_headers, response = web_util.read_from_url(url) + contents = response.fp.read() + + return int(contents) + + +def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id): + if mirror_url: + tty.debug('Creating buildcache') + buildcache._createtarball(env, yaml_path, None, mirror_url, None, + True, True, False, False, True, False) + if build_id: + tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( + build_id, mirror_url)) + write_cdashid_to_mirror(build_id, spec, mirror_url) + + +def copy_stage_logs_to_artifacts(job_spec, job_log_dir): + try: + job_pkg = spack.repo.get(job_spec) + tty.debug('job package: {0}'.format(job_pkg)) + stage_dir = job_pkg.stage.path + tty.debug('stage dir: {0}'.format(stage_dir)) + build_env_src = os.path.join(stage_dir, 'spack-build-env.txt') + build_out_src = os.path.join(stage_dir, 'spack-build-out.txt') + build_env_dst = os.path.join( + job_log_dir, 'spack-build-env.txt') + build_out_dst = os.path.join( + job_log_dir, 'spack-build-out.txt') + tty.debug('Copying logs to artifacts:') + tty.debug(' 1: {0} -> {1}'.format( + build_env_src, build_env_dst)) + shutil.copyfile(build_env_src, build_env_dst) + tty.debug(' 2: {0} -> {1}'.format( + build_out_src, build_out_dst)) + shutil.copyfile(build_out_src, build_out_dst) + except Exception as inst: + msg = ('Unable to copy build logs from stage to artifacts ' + 'due to exception: {0}').format(inst) + tty.error(msg) diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 0d2e92940b..2a75a87b54 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -19,13 +19,13 @@ from llnl.util.tty.color import colorize from llnl.util.filesystem import working_dir import spack.config +import spack.error import spack.extensions import spack.paths import spack.spec import spack.store import spack.util.spack_json as sjson import spack.util.string -from spack.error import SpackError # cmd has a submodule called "list" so preserve the python list module @@ -150,15 +150,15 @@ def parse_specs(args, **kwargs): except spack.spec.SpecParseError as e: msg = e.message + "\n" + str(e.string) + "\n" msg += (e.pos + 2) * " " + "^" - raise SpackError(msg) + raise spack.error.SpackError(msg) - except spack.spec.SpecError as e: + except spack.error.SpecError as e: msg = e.message if e.long_message: msg += e.long_message - raise SpackError(msg) + raise spack.error.SpackError(msg) def elide_list(line_list, max_num=10): @@ -190,6 +190,20 @@ def disambiguate_spec(spec, env, local=False, installed=True): database query. See ``spack.database.Database._query`` for details. """ hashes = env.all_hashes() if env else None + return disambiguate_spec_from_hashes(spec, hashes, local, installed) + + +def disambiguate_spec_from_hashes(spec, hashes, local=False, installed=True): + """Given a spec and a list of hashes, get concrete spec the spec refers to. + + Arguments: + spec (spack.spec.Spec): a spec to disambiguate + hashes (iterable): a set of hashes of specs among which to disambiguate + local (boolean, default False): do not search chained spack instances + installed (boolean or any, or spack.database.InstallStatus or iterable + of spack.database.InstallStatus): install status argument passed to + database query. See ``spack.database.Database._query`` for details. + """ if local: matching_specs = spack.store.db.query_local(spec, hashes=hashes, installed=installed) @@ -212,6 +226,9 @@ def disambiguate_spec(spec, env, local=False, installed=True): def gray_hash(spec, length): + if not length: + # default to maximum hash length + length = 32 h = spec.dag_hash(length) if spec.concrete else '-' * length return colorize('@K{%s}' % h) diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index 6a7297432d..bfca9f5604 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -1,13 +1,12 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse - import llnl.util.tty as tty import spack.cmd +import spack.cmd.common.arguments as arguments import spack.environment as ev from spack.filesystem_view import YamlFilesystemView @@ -23,9 +22,7 @@ def setup_parser(subparser): subparser.add_argument( '-v', '--view', metavar='VIEW', type=str, help="the view to operate on") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help="spec of package extension to activate") + arguments.add_common_arguments(subparser, ['installed_spec']) def activate(parser, args): diff --git a/lib/spack/spack/cmd/add.py b/lib/spack/spack/cmd/add.py index 894161fe00..94c8620dbb 100644 --- a/lib/spack/spack/cmd/add.py +++ b/lib/spack/spack/cmd/add.py @@ -1,13 +1,12 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse - import llnl.util.tty as tty import spack.cmd +import spack.cmd.common.arguments as arguments import spack.environment as ev @@ -20,17 +19,17 @@ def setup_parser(subparser): subparser.add_argument('-l', '--list-name', dest='list_name', default='specs', help="name of the list to add specs to") - subparser.add_argument( - 'specs', nargs=argparse.REMAINDER, help="specs of packages to add") + arguments.add_common_arguments(subparser, ['specs']) def add(parser, args): env = ev.get_env(args, 'add', required=True) - for spec in spack.cmd.parse_specs(args.specs): - if not env.add(spec, args.list_name): - tty.msg("Package {0} was already added to {1}" - .format(spec.name, env.name)) - else: - tty.msg('Adding %s to environment %s' % (spec, env.name)) - env.write() + with env.write_transaction(): + for spec in spack.cmd.parse_specs(args.specs): + if not env.add(spec, args.list_name): + tty.msg("Package {0} was already added to {1}" + .format(spec.name, env.name)) + else: + tty.msg('Adding %s to environment %s' % (spec, env.name)) + env.write() diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index 081722c05b..6f5dad758f 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/blame.py b/lib/spack/spack/cmd/blame.py index 5e3f4095a4..b806058aec 100644 --- a/lib/spack/spack/cmd/blame.py +++ b/lib/spack/spack/cmd/blame.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -35,7 +35,7 @@ def setup_parser(subparser): help='show git blame output instead of summary') subparser.add_argument( - 'package_name', help='name of package to show contributions for, ' + 'package_or_file', help='name of package to show contributions for, ' 'or path to a file in the spack repo') @@ -47,13 +47,13 @@ def blame(parser, args): # Get name of file to blame blame_file = None - if os.path.isfile(args.package_name): - path = os.path.realpath(args.package_name) + if os.path.isfile(args.package_or_file): + path = os.path.realpath(args.package_or_file) if path.startswith(spack.paths.prefix): blame_file = path if not blame_file: - pkg = spack.repo.get(args.package_name) + pkg = spack.repo.get(args.package_or_file) blame_file = pkg.module.__file__.rstrip('c') # .pyc -> .py # get git blame for the package diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 96cb8ccaa9..53c284ab6d 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/build.py b/lib/spack/spack/cmd/build.py index 8687e9a741..a56c37fdd4 100644 --- a/lib/spack/spack/cmd/build.py +++ b/lib/spack/spack/cmd/build.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/build_env.py b/lib/spack/spack/cmd/build_env.py index 87961fcb9c..128d167a29 100644 --- a/lib/spack/spack/cmd/build_env.py +++ b/lib/spack/spack/cmd/build_env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -33,7 +33,7 @@ def setup_parser(subparser): subparser.add_argument( 'spec', nargs=argparse.REMAINDER, metavar='spec [--] [cmd]...', - help="specs of package environment to emulate") + help="spec of package environment to emulate") subparser.epilog\ = 'If a command is not specified, the environment will be printed ' \ 'to standard output (cf /usr/bin/env) unless --dump and/or --pickle ' \ diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index ee09a33f39..392984f852 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -1,9 +1,8 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse import os import shutil import sys @@ -61,11 +60,9 @@ def setup_parser(subparser): "building package(s)") create.add_argument('-y', '--spec-yaml', default=None, help='Create buildcache entry for spec from yaml file') - create.add_argument( - 'packages', nargs=argparse.REMAINDER, - help="specs of packages to create buildcache for") create.add_argument('--no-deps', action='store_true', default='false', help='Create buildcache entry wo/ dependencies') + arguments.add_common_arguments(create, ['specs']) create.set_defaults(func=createtarball) install = subparsers.add_parser('install', help=installtarball.__doc__) @@ -79,9 +76,7 @@ def setup_parser(subparser): install.add_argument('-u', '--unsigned', action='store_true', help="install unsigned buildcache" + " tarballs for testing") - install.add_argument( - 'packages', nargs=argparse.REMAINDER, - help="specs of packages to install buildcache for") + arguments.add_common_arguments(install, ['specs']) install.set_defaults(func=installtarball) listcache = subparsers.add_parser('list', help=listspecs.__doc__) @@ -92,9 +87,10 @@ def setup_parser(subparser): help='show variants in output (can be long)') listcache.add_argument('-f', '--force', action='store_true', help="force new download of specs") - listcache.add_argument( - 'packages', nargs=argparse.REMAINDER, - help="specs of packages to search for") + listcache.add_argument('-a', '--allarch', action='store_true', + help="list specs for all available architectures" + + " instead of default platform and OS") + arguments.add_common_arguments(listcache, ['specs']) listcache.set_defaults(func=listspecs) dlkeys = subparsers.add_parser('keys', help=getkeys.__doc__) @@ -113,10 +109,9 @@ def setup_parser(subparser): help='analyzes an installed spec and reports whether ' 'executables and libraries are relocatable' ) - preview_parser.add_argument( - 'packages', nargs='+', help='list of installed packages' - ) + arguments.add_common_arguments(preview_parser, ['installed_specs']) preview_parser.set_defaults(func=preview) + # Check if binaries need to be rebuilt on remote mirror check = subparsers.add_parser('check', help=check_binaries.__doc__) check.add_argument( @@ -271,7 +266,8 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False): # List of specs that match expressions given via command line specs_from_cli = [] has_errors = False - specs = bindist.get_specs(force) + allarch = False + specs = bindist.get_specs(force, allarch) for pkg in pkgs: matches = [] tty.msg("buildcache spec(s) matching %s \n" % pkg) @@ -303,19 +299,20 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False): return specs_from_cli -def createtarball(args): - """create a binary package from an existing install""" - if args.spec_yaml: +def _createtarball(env, spec_yaml, packages, directory, key, no_deps, force, + rel, unsigned, allow_root, no_rebuild_index): + if spec_yaml: packages = set() - tty.msg('createtarball, reading spec from {0}'.format(args.spec_yaml)) - with open(args.spec_yaml, 'r') as fd: + with open(spec_yaml, 'r') as fd: yaml_text = fd.read() tty.debug('createtarball read spec yaml:') tty.debug(yaml_text) s = Spec.from_yaml(yaml_text) packages.add('/{0}'.format(s.dag_hash())) - elif args.packages: - packages = args.packages + + elif packages: + packages = packages + else: tty.die("build cache file creation requires at least one" + " installed package argument or else path to a" + @@ -324,18 +321,15 @@ def createtarball(args): specs = set() outdir = '.' - if args.directory: - outdir = args.directory + if directory: + outdir = directory mirror = spack.mirror.MirrorCollection().lookup(outdir) outdir = url_util.format(mirror.push_url) signkey = None - if args.key: - signkey = args.key - - # restrict matching to current environment if one is active - env = ev.get_env(args, 'buildcache create') + if key: + signkey = key matches = find_matching_specs(pkgs, env=env) @@ -350,7 +344,7 @@ def createtarball(args): else: tty.debug('adding matching spec %s' % match.format()) specs.add(match) - if args.no_deps is True: + if no_deps is True: continue tty.debug('recursing dependencies') for d, node in match.traverse(order='post', @@ -368,20 +362,31 @@ def createtarball(args): for spec in specs: tty.msg('creating binary cache file for package %s ' % spec.format()) try: - bindist.build_tarball(spec, outdir, args.force, args.rel, - args.unsigned, args.allow_root, signkey, - not args.no_rebuild_index) + bindist.build_tarball(spec, outdir, force, rel, + unsigned, allow_root, signkey, + not no_rebuild_index) except Exception as e: tty.warn('%s' % e) pass +def createtarball(args): + """create a binary package from an existing install""" + + # restrict matching to current environment if one is active + env = ev.get_env(args, 'buildcache create') + + _createtarball(env, args.spec_yaml, args.specs, args.directory, + args.key, args.no_deps, args.force, args.rel, args.unsigned, + args.allow_root, args.no_rebuild_index) + + def installtarball(args): """install from a binary package""" - if not args.packages: + if not args.specs: tty.die("build cache file installation requires" + " at least one package spec argument") - pkgs = set(args.packages) + pkgs = set(args.specs) matches = match_downloaded_specs(pkgs, args.multiple, args.force) for match in matches: @@ -414,13 +419,11 @@ def install_tarball(spec, args): def listspecs(args): """list binary packages available from mirrors""" - specs = bindist.get_specs(args.force) - if args.packages: - pkgs = set(args.packages) - specs = [s for s in specs for p in pkgs if s.satisfies(p)] - display_specs(specs, args, all_headers=True) - else: - display_specs(specs, args, all_headers=True) + specs = bindist.get_specs(args.force, args.allarch) + if args.specs: + constraints = set(args.specs) + specs = [s for s in specs if any(s.satisfies(c) for c in constraints)] + display_specs(specs, args, all_headers=True) def getkeys(args): @@ -435,7 +438,7 @@ def preview(args): Args: args: command line arguments """ - specs = find_matching_specs(args.packages, allow_multiple_matches=True) + specs = find_matching_specs(args.specs, allow_multiple_matches=True) # Cycle over the specs that match for spec in specs: @@ -479,6 +482,32 @@ def check_binaries(args): configured_mirrors, specs, args.output_file, args.rebuild_on_error)) +def download_buildcache_files(concrete_spec, local_dest, require_cdashid, + mirror_url=None): + tarfile_name = bindist.tarball_name(concrete_spec, '.spack') + tarball_dir_name = bindist.tarball_directory_name(concrete_spec) + tarball_path_name = os.path.join(tarball_dir_name, tarfile_name) + local_tarball_path = os.path.join(local_dest, tarball_dir_name) + + files_to_fetch = [ + { + 'url': tarball_path_name, + 'path': local_tarball_path, + 'required': True, + }, { + 'url': bindist.tarball_name(concrete_spec, '.spec.yaml'), + 'path': local_dest, + 'required': True, + }, { + 'url': bindist.tarball_name(concrete_spec, '.cdashid'), + 'path': local_dest, + 'required': require_cdashid, + }, + ] + + return bindist.download_buildcache_entry(files_to_fetch, mirror_url) + + def get_tarball(args): """Download buildcache entry from a remote mirror to local folder. This command uses the process exit code to indicate its result, specifically, @@ -495,34 +524,10 @@ def get_tarball(args): sys.exit(0) spec = get_concrete_spec(args) + result = download_buildcache_files(spec, args.path, args.require_cdashid) - tarfile_name = bindist.tarball_name(spec, '.spack') - tarball_dir_name = bindist.tarball_directory_name(spec) - tarball_path_name = os.path.join(tarball_dir_name, tarfile_name) - local_tarball_path = os.path.join(args.path, tarball_dir_name) - - files_to_fetch = [ - { - 'url': tarball_path_name, - 'path': local_tarball_path, - 'required': True, - }, { - 'url': bindist.tarball_name(spec, '.spec.yaml'), - 'path': args.path, - 'required': True, - }, { - 'url': bindist.tarball_name(spec, '.cdashid'), - 'path': args.path, - 'required': args.require_cdashid, - }, - ] - - result = bindist.download_buildcache_entry(files_to_fetch) - - if result: - sys.exit(0) - - sys.exit(1) + if not result: + sys.exit(1) def get_concrete_spec(args): diff --git a/lib/spack/spack/cmd/cd.py b/lib/spack/spack/cmd/cd.py index e293b9f220..a810e36ef3 100644 --- a/lib/spack/spack/cmd/cd.py +++ b/lib/spack/spack/cmd/cd.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 2518dfef9f..343915868c 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -10,6 +10,7 @@ import argparse import llnl.util.tty as tty import spack.cmd +import spack.cmd.common.arguments as arguments import spack.repo import spack.stage import spack.util.crypto @@ -23,11 +24,9 @@ level = "long" def setup_parser(subparser): subparser.add_argument( - 'package', - help='package to checksum versions for') - subparser.add_argument( '--keep-stage', action='store_true', help="don't clean up staging area when command completes") + arguments.add_common_arguments(subparser, ['package']) subparser.add_argument( 'versions', nargs=argparse.REMAINDER, help='versions to generate checksums for') @@ -61,3 +60,4 @@ def checksum(parser, args): print() print(version_lines) + print() diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py new file mode 100644 index 0000000000..22c555d85a --- /dev/null +++ b/lib/spack/spack/cmd/ci.py @@ -0,0 +1,482 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import shutil +import sys + +from six.moves.urllib.parse import urlencode + +import llnl.util.tty as tty + +import spack.binary_distribution as bindist +import spack.ci as spack_ci +import spack.cmd.buildcache as buildcache +import spack.environment as ev +import spack.hash_types as ht +import spack.util.executable as exe + + +description = "manage continuous integration pipelines" +section = "build" +level = "long" + + +def get_env_var(variable_name): + if variable_name in os.environ: + return os.environ.get(variable_name) + return None + + +def setup_parser(subparser): + setup_parser.parser = subparser + subparsers = subparser.add_subparsers(help='CI sub-commands') + + start = subparsers.add_parser('start', help=ci_start.__doc__) + start.add_argument( + '--output-file', default=None, + help="Absolute path to file where generated jobs file should be " + + "written. The default is .gitlab-ci.yml in the root of the " + + "repository.") + start.add_argument( + '--copy-to', default=None, + help="Absolute path of additional location where generated jobs " + + "yaml file should be copied. Default is not to copy.") + start.add_argument( + '--spack-repo', default=None, + help="Provide a url for this argument if a custom spack repo " + + "should be cloned as a step in each generated job.") + start.add_argument( + '--spack-ref', default=None, + help="Provide a git branch or tag if a custom spack branch " + + "should be checked out as a step in each generated job. " + + "This argument is ignored if no --spack-repo is provided.") + start.add_argument( + '--downstream-repo', default=None, + help="Url to repository where commit containing jobs yaml file " + + "should be pushed.") + start.add_argument( + '--branch-name', default='default-branch', + help="Name of current branch, used in generation of pushed commit.") + start.add_argument( + '--commit-sha', default='none', + help="SHA of current commit, used in generation of pushed commit.") + start.set_defaults(func=ci_start) + + # Dynamic generation of the jobs yaml from a spack environment + generate = subparsers.add_parser('generate', help=ci_generate.__doc__) + generate.add_argument( + '--output-file', default=None, + help="Absolute path to file where generated jobs file should be " + + "written. The default is .gitlab-ci.yml in the root of the " + + "repository.") + generate.add_argument( + '--copy-to', default=None, + help="Absolute path of additional location where generated jobs " + + "yaml file should be copied. Default is not to copy.") + generate.add_argument( + '--spack-repo', default=None, + help="Provide a url for this argument if a custom spack repo " + + "should be cloned as a step in each generated job.") + generate.add_argument( + '--spack-ref', default=None, + help="Provide a git branch or tag if a custom spack branch " + + "should be checked out as a step in each generated job. " + + "This argument is ignored if no --spack-repo is provided.") + generate.set_defaults(func=ci_generate) + + # Commit and push jobs yaml to a downstream CI repo + pushyaml = subparsers.add_parser('pushyaml', help=ci_pushyaml.__doc__) + pushyaml.add_argument( + '--downstream-repo', default=None, + help="Url to repository where commit containing jobs yaml file " + + "should be pushed.") + pushyaml.add_argument( + '--branch-name', default='default-branch', + help="Name of current branch, used in generation of pushed commit.") + pushyaml.add_argument( + '--commit-sha', default='none', + help="SHA of current commit, used in generation of pushed commit.") + pushyaml.set_defaults(func=ci_pushyaml) + + # Check a spec against mirror. Rebuild, create buildcache and push to + # mirror (if necessary). + rebuild = subparsers.add_parser('rebuild', help=ci_rebuild.__doc__) + rebuild.set_defaults(func=ci_rebuild) + + +def ci_generate(args): + """Generate jobs file from a spack environment file containing CI info. + Before invoking this command, you can set the environment variable + SPACK_CDASH_AUTH_TOKEN to contain the CDash authorization token + for creating a build group for the generated workload and registering + all generated jobs under that build group. If this environment + variable is not set, no build group will be created on CDash.""" + env = ev.get_env(args, 'ci generate', required=True) + + output_file = args.output_file + copy_yaml_to = args.copy_to + spack_repo = args.spack_repo + spack_ref = args.spack_ref + + if not output_file: + gen_ci_dir = os.getcwd() + output_file = os.path.join(gen_ci_dir, '.gitlab-ci.yml') + else: + gen_ci_dir = os.path.dirname(output_file) + if not os.path.exists(gen_ci_dir): + os.makedirs(gen_ci_dir) + + # Generate the jobs + spack_ci.generate_gitlab_ci_yaml( + env, True, output_file, spack_repo, spack_ref) + + if copy_yaml_to: + copy_to_dir = os.path.dirname(copy_yaml_to) + if not os.path.exists(copy_to_dir): + os.makedirs(copy_to_dir) + shutil.copyfile(output_file, copy_yaml_to) + + +def ci_pushyaml(args): + """Push the generated jobs yaml file to a remote repository. The file + (.gitlab-ci.yaml) is expected to be in the current directory, which + should be the root of the repository.""" + downstream_repo = args.downstream_repo + branch_name = args.branch_name + commit_sha = args.commit_sha + + if not downstream_repo: + tty.die('No downstream repo to push to, exiting') + + working_dir = os.getcwd() + jobs_yaml = os.path.join(working_dir, '.gitlab-ci.yml') + git_dir = os.path.join(working_dir, '.git') + + if not os.path.exists(jobs_yaml): + tty.die('.gitlab-ci.yml must exist in current directory') + + if not os.path.exists(git_dir): + tty.die('.git directory must exist in current directory') + + # Create a temporary working directory + with spack_ci.TemporaryDirectory() as temp_dir: + git = exe.which('git', required=True) + + # Push a commit with the generated file to the downstream ci repo + saved_git_dir = os.path.join(temp_dir, 'original-git-dir') + + shutil.move('.git', saved_git_dir) + + git('init', '.') + + git('config', 'user.email', 'robot@spack.io') + git('config', 'user.name', 'Spack Build Bot') + + git('add', '.') + + # If the environment contains a spack directory, do not commit + # or push it with any other generated products + if os.path.exists('./spack') and os.path.isdir('./spack'): + git('rm', '-rf', '--cached', 'spack') + + tty.msg('git commit') + commit_message = '{0} {1} ({2})'.format( + 'Auto-generated commit testing', branch_name, commit_sha) + + git('commit', '-m', '{0}'.format(commit_message)) + + tty.msg('git push') + git('remote', 'add', 'downstream', downstream_repo) + push_to_branch = 'master:multi-ci-{0}'.format(branch_name) + git('push', '--force', 'downstream', push_to_branch) + + shutil.rmtree('.git') + shutil.move(saved_git_dir, '.git') + git('reset', '--hard', 'HEAD') + + +def ci_rebuild(args): + """This command represents a gitlab-ci job, corresponding to a single + release spec. As such it must first decide whether or not the spec it + has been assigned to build is up to date on the remote binary mirror. + If it is not (i.e. the full_hash of the spec as computed locally does + not match the one stored in the metadata on the mirror), this script + will build the package, create a binary cache for it, and then push all + related files to the remote binary mirror. This script also + communicates with a remote CDash instance to share status on the package + build process. + + The spec to be built by this job is represented by essentially two + pieces of information: 1) a root spec (possibly already concrete, but + maybe still needing to be concretized) and 2) a package name used to + index that root spec (once the root is, for certain, concrete).""" + env = ev.get_env(args, 'ci rebuild', required=True) + yaml_root = ev.config_dict(env.yaml) + + # The following environment variables should defined in the CI + # infrastructre (or some other external source) in the case that the + # remote mirror is an S3 bucket. The AWS keys are used to upload + # buildcache entries to S3 using the boto3 api. + # + # AWS_ACCESS_KEY_ID + # AWS_SECRET_ACCESS_KEY + # S3_ENDPOINT_URL (only needed for non-AWS S3 implementations) + # + # If present, we will import the SPACK_SIGNING_KEY using the + # "spack gpg trust" command, so it can be used both for verifying + # dependency buildcache entries and signing the buildcache entry we create + # for our target pkg. + # + # SPACK_SIGNING_KEY + + ci_artifact_dir = get_env_var('CI_PROJECT_DIR') + signing_key = get_env_var('SPACK_SIGNING_KEY') + root_spec = get_env_var('SPACK_ROOT_SPEC') + job_spec_pkg_name = get_env_var('SPACK_JOB_SPEC_PKG_NAME') + compiler_action = get_env_var('SPACK_COMPILER_ACTION') + cdash_build_name = get_env_var('SPACK_CDASH_BUILD_NAME') + related_builds = get_env_var('SPACK_RELATED_BUILDS_CDASH') + + gitlab_ci = None + if 'gitlab-ci' in yaml_root: + gitlab_ci = yaml_root['gitlab-ci'] + + if not gitlab_ci: + tty.die('spack ci rebuild requires an env containing gitlab-ci cfg') + + enable_cdash = False + if 'cdash' in yaml_root: + enable_cdash = True + ci_cdash = yaml_root['cdash'] + job_spec_buildgroup = ci_cdash['build-group'] + cdash_base_url = ci_cdash['url'] + cdash_project = ci_cdash['project'] + proj_enc = urlencode({'project': cdash_project}) + eq_idx = proj_enc.find('=') + 1 + cdash_project_enc = proj_enc[eq_idx:] + cdash_site = ci_cdash['site'] + tty.debug('cdash_base_url = {0}'.format(cdash_base_url)) + tty.debug('cdash_project = {0}'.format(cdash_project)) + tty.debug('cdash_project_enc = {0}'.format(cdash_project_enc)) + tty.debug('cdash_build_name = {0}'.format(cdash_build_name)) + tty.debug('cdash_site = {0}'.format(cdash_site)) + tty.debug('related_builds = {0}'.format(related_builds)) + tty.debug('job_spec_buildgroup = {0}'.format(job_spec_buildgroup)) + + remote_mirror_url = None + if 'mirrors' in yaml_root: + ci_mirrors = yaml_root['mirrors'] + mirror_urls = [url for url in ci_mirrors.values()] + remote_mirror_url = mirror_urls[0] + + if not remote_mirror_url: + tty.die('spack ci rebuild requires an env containing a mirror') + + tty.debug('ci_artifact_dir = {0}'.format(ci_artifact_dir)) + tty.debug('root_spec = {0}'.format(root_spec)) + tty.debug('remote_mirror_url = {0}'.format(remote_mirror_url)) + tty.debug('job_spec_pkg_name = {0}'.format(job_spec_pkg_name)) + tty.debug('compiler_action = {0}'.format(compiler_action)) + + spack_cmd = exe.which('spack') + + os.environ['FORCE_UNSAFE_CONFIGURE'] = '1' + + cdash_report_dir = os.path.join(ci_artifact_dir, 'cdash_report') + temp_dir = os.path.join(ci_artifact_dir, 'jobs_scratch_dir') + job_log_dir = os.path.join(temp_dir, 'logs') + spec_dir = os.path.join(temp_dir, 'specs') + + local_mirror_dir = os.path.join(ci_artifact_dir, 'local_mirror') + build_cache_dir = os.path.join(local_mirror_dir, 'build_cache') + + enable_artifacts_mirror = False + artifact_mirror_url = None + if 'enable-artifacts-buildcache' in gitlab_ci: + enable_artifacts_mirror = gitlab_ci['enable-artifacts-buildcache'] + if enable_artifacts_mirror: + artifact_mirror_url = 'file://' + local_mirror_dir + mirror_msg = 'artifact buildcache enabled, mirror url: {0}'.format( + artifact_mirror_url) + tty.debug(mirror_msg) + + # Clean out scratch directory from last stage + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + + if os.path.exists(cdash_report_dir): + shutil.rmtree(cdash_report_dir) + + os.makedirs(job_log_dir) + os.makedirs(spec_dir) + + job_spec_yaml_path = os.path.join( + spec_dir, '{0}.yaml'.format(job_spec_pkg_name)) + job_log_file = os.path.join(job_log_dir, 'pipeline_log.txt') + + cdash_build_id = None + cdash_build_stamp = None + + with open(job_log_file, 'w') as log_fd: + os.dup2(log_fd.fileno(), sys.stdout.fileno()) + os.dup2(log_fd.fileno(), sys.stderr.fileno()) + + current_directory = os.getcwd() + tty.debug('Current working directory: {0}, Contents:'.format( + current_directory)) + directory_list = os.listdir(current_directory) + for next_entry in directory_list: + tty.debug(' {0}'.format(next_entry)) + + # Make a copy of the environment file, so we can overwrite the changed + # version in between the two invocations of "spack install" + env_src_path = os.path.join(current_directory, 'spack.yaml') + env_dst_path = os.path.join(current_directory, 'spack.yaml_BACKUP') + shutil.copyfile(env_src_path, env_dst_path) + + tty.debug('job concrete spec path: {0}'.format(job_spec_yaml_path)) + + if signing_key: + spack_ci.import_signing_key(signing_key) + + spack_ci.configure_compilers(compiler_action) + + spec_map = spack_ci.get_concrete_specs( + root_spec, job_spec_pkg_name, related_builds, compiler_action) + + job_spec = spec_map[job_spec_pkg_name] + + tty.debug('Here is the concrete spec: {0}'.format(job_spec)) + + with open(job_spec_yaml_path, 'w') as fd: + fd.write(job_spec.to_yaml(hash=ht.build_hash)) + + tty.debug('Done writing concrete spec') + + # DEBUG + with open(job_spec_yaml_path) as fd: + tty.debug('Wrote spec file, read it back. Contents:') + tty.debug(fd.read()) + + # DEBUG the root spec + root_spec_yaml_path = os.path.join(spec_dir, 'root.yaml') + with open(root_spec_yaml_path, 'w') as fd: + fd.write(spec_map['root'].to_yaml(hash=ht.build_hash)) + + if bindist.needs_rebuild(job_spec, remote_mirror_url, True): + # Binary on remote mirror is not up to date, we need to rebuild + # it. + # + # FIXME: ensure mirror precedence causes this local mirror to + # be chosen ahead of the remote one when installing deps + if enable_artifacts_mirror: + mirror_add_output = spack_cmd( + 'mirror', 'add', 'local_mirror', artifact_mirror_url) + tty.debug('spack mirror add:') + tty.debug(mirror_add_output) + + mirror_list_output = spack_cmd('mirror', 'list') + tty.debug('listing spack mirrors:') + tty.debug(mirror_list_output) + + # 2) build up install arguments + install_args = ['-d', '-v', '-k', 'install', '--keep-stage'] + + # 3) create/register a new build on CDash (if enabled) + cdash_args = [] + if enable_cdash: + tty.debug('Registering build with CDash') + (cdash_build_id, + cdash_build_stamp) = spack_ci.register_cdash_build( + cdash_build_name, cdash_base_url, cdash_project, + cdash_site, job_spec_buildgroup) + + cdash_upload_url = '{0}/submit.php?project={1}'.format( + cdash_base_url, cdash_project_enc) + + cdash_args = [ + '--cdash-upload-url', cdash_upload_url, + '--cdash-build', cdash_build_name, + '--cdash-site', cdash_site, + '--cdash-buildstamp', cdash_build_stamp, + ] + + spec_cli_arg = [job_spec_yaml_path] + + tty.debug('Installing package') + + try: + # Two-pass install is intended to avoid spack trying to + # install from buildcache even though the locally computed + # full hash is different than the one stored in the spec.yaml + # file on the remote mirror. + first_pass_args = install_args + [ + '--cache-only', + '--only', + 'dependencies', + ] + first_pass_args.extend(spec_cli_arg) + tty.debug('First pass install arguments: {0}'.format( + first_pass_args)) + spack_cmd(*first_pass_args) + + # Overwrite the changed environment file so it doesn't + # the next install invocation. + shutil.copyfile(env_dst_path, env_src_path) + + second_pass_args = install_args + [ + '--no-cache', + '--only', + 'package', + ] + second_pass_args.extend(cdash_args) + second_pass_args.extend(spec_cli_arg) + tty.debug('Second pass install arguments: {0}'.format( + second_pass_args)) + spack_cmd(*second_pass_args) + except Exception as inst: + tty.error('Caught exception during install:') + tty.error(inst) + + spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) + + # 4) create buildcache on remote mirror + spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, + remote_mirror_url, cdash_build_id) + + # 5) create another copy of that buildcache on "local artifact + # mirror" (only done if cash reporting is enabled) + spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, + artifact_mirror_url, cdash_build_id) + + # 6) relate this build to its dependencies on CDash (if enabled) + if enable_cdash: + spack_ci.relate_cdash_builds( + spec_map, cdash_base_url, cdash_build_id, cdash_project, + artifact_mirror_url or remote_mirror_url) + else: + # There is nothing to do here unless "local artifact mirror" is + # enabled, in which case, we need to download the buildcache to + # the local artifacts directory to be used by dependent jobs in + # subsequent stages + tty.debug('No need to rebuild {0}'.format(job_spec_pkg_name)) + if enable_artifacts_mirror: + tty.debug('Getting {0} buildcache'.format(job_spec_pkg_name)) + tty.debug('Downloading to {0}'.format(build_cache_dir)) + buildcache.download_buildcache_files( + job_spec, build_cache_dir, True, remote_mirror_url) + + +def ci_start(args): + """Kicks of the CI process (currently just calls ci_generate() then + ci_push())""" + ci_generate(args) + ci_pushyaml(args) + + +def ci(parser, args): + if args.func: + args.func(args) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 5dc7396ed4..791a1b7dc3 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,6 +11,7 @@ import llnl.util.tty as tty import spack.caches import spack.cmd +import spack.cmd.common.arguments as arguments import spack.repo import spack.stage from spack.paths import lib_path, var_path @@ -43,11 +44,7 @@ def setup_parser(subparser): subparser.add_argument( '-a', '--all', action=AllClean, help="equivalent to -sdmp", nargs=0 ) - subparser.add_argument( - 'specs', - nargs=argparse.REMAINDER, - help="removes the build stages and tarballs for specs" - ) + arguments.add_common_arguments(subparser, ['specs']) def clean(parser, args): diff --git a/lib/spack/spack/cmd/clone.py b/lib/spack/spack/cmd/clone.py index 0385adf100..ecf5855dab 100644 --- a/lib/spack/spack/cmd/clone.py +++ b/lib/spack/spack/cmd/clone.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py index 8c1a6b1e77..3664fce477 100644 --- a/lib/spack/spack/cmd/commands.py +++ b/lib/spack/spack/cmd/commands.py @@ -1,21 +1,26 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function -import sys +import argparse +import copy import os import re -import argparse +import sys +import llnl.util.filesystem as fs import llnl.util.tty as tty -from llnl.util.argparsewriter import ArgparseWriter, ArgparseRstWriter +from llnl.util.argparsewriter import ( + ArgparseWriter, ArgparseRstWriter, ArgparseCompletionWriter +) from llnl.util.tty.colify import colify import spack.cmd import spack.main +import spack.paths from spack.main import section_descriptions @@ -28,6 +33,20 @@ level = "long" formatters = {} +#: standard arguments for updating completion scripts +#: we iterate through these when called with --update-completion +update_completion_args = { + "bash": { + "aliases": True, + "format": "bash", + "header": os.path.join( + spack.paths.share_path, "bash", "spack-completion.in"), + "update": os.path.join( + spack.paths.share_path, "spack-completion.bash"), + }, +} + + def formatter(func): """Decorator used to register formatters""" formatters[func.__name__] = func @@ -36,6 +55,13 @@ def formatter(func): def setup_parser(subparser): subparser.add_argument( + "--update-completion", action='store_true', default=False, + help="regenerate spack's tab completion scripts") + + subparser.add_argument( + '-a', '--aliases', action='store_true', default=False, + help='include command aliases') + subparser.add_argument( '--format', default='names', choices=formatters, help='format to be used to print the output (default: names)') subparser.add_argument( @@ -52,29 +78,97 @@ def setup_parser(subparser): class SpackArgparseRstWriter(ArgparseRstWriter): """RST writer tailored for spack documentation.""" - def __init__(self, documented_commands, out=sys.stdout): - super(SpackArgparseRstWriter, self).__init__(out) - self.documented = documented_commands if documented_commands else [] + def __init__(self, prog, out=sys.stdout, aliases=False, + documented_commands=[], + rst_levels=['-', '-', '^', '~', ':', '`']): + super(SpackArgparseRstWriter, self).__init__( + prog, out, aliases, rst_levels) + self.documented = documented_commands def usage(self, *args): - super(SpackArgparseRstWriter, self).usage(*args) - cmd = re.sub(' ', '-', self.parser.prog) + string = super(SpackArgparseRstWriter, self).usage(*args) + + cmd = self.parser.prog.replace(' ', '-') if cmd in self.documented: - self.line() - self.line(':ref:`More documentation <cmd-%s>`' % cmd) + string += '\n:ref:`More documentation <cmd-{0}>`\n'.format(cmd) + + return string class SubcommandWriter(ArgparseWriter): - def begin_command(self, prog): - self.out.write(' ' * self.level + prog) - self.out.write('\n') + def format(self, cmd): + return ' ' * self.level + cmd.prog + '\n' + + +_positional_to_subroutine = { + 'package': '_all_packages', + 'spec': '_all_packages', + 'filter': '_all_packages', + 'installed': '_installed_packages', + 'compiler': '_installed_compilers', + 'section': '_config_sections', + 'env': '_environments', + 'extendable': '_extensions', + 'keys': '_keys', + 'help_command': '_subcommands', + 'mirror': '_mirrors', + 'virtual': '_providers', + 'namespace': '_repos', + 'hash': '_all_resource_hashes', + 'pytest': '_tests', +} + + +class BashCompletionWriter(ArgparseCompletionWriter): + """Write argparse output as bash programmable tab completion.""" + + def body(self, positionals, optionals, subcommands): + if positionals: + return """ + if $list_options + then + {0} + else + {1} + fi +""".format(self.optionals(optionals), self.positionals(positionals)) + elif subcommands: + return """ + if $list_options + then + {0} + else + {1} + fi +""".format(self.optionals(optionals), self.subcommands(subcommands)) + else: + return """ + {0} +""".format(self.optionals(optionals)) + + def positionals(self, positionals): + # If match found, return function name + for positional in positionals: + for key, value in _positional_to_subroutine.items(): + if positional.startswith(key): + return value + + # If no matches found, return empty list + return 'SPACK_COMPREPLY=""' + + def optionals(self, optionals): + return 'SPACK_COMPREPLY="{0}"'.format(' '.join(optionals)) + + def subcommands(self, subcommands): + return 'SPACK_COMPREPLY="{0}"'.format(' '.join(subcommands)) @formatter def subcommands(args, out): parser = spack.main.make_argument_parser() spack.main.add_all_commands(parser) - SubcommandWriter(out).write(parser) + writer = SubcommandWriter(parser.prog, out, args.aliases) + writer.write(parser) def rst_index(out): @@ -124,12 +218,28 @@ def rst(args, out): out.write('\n') # print sections for each command and subcommand - SpackArgparseRstWriter(documented_commands, out).write(parser, root=1) + writer = SpackArgparseRstWriter( + parser.prog, out, args.aliases, documented_commands) + writer.write(parser) @formatter def names(args, out): - colify(spack.cmd.all_commands(), output=out) + commands = copy.copy(spack.cmd.all_commands()) + + if args.aliases: + commands.extend(spack.main.aliases.keys()) + + colify(commands, output=out) + + +@formatter +def bash(args, out): + parser = spack.main.make_argument_parser() + spack.main.add_all_commands(parser) + + writer = BashCompletionWriter(parser.prog, out, args.aliases) + writer.write(parser) def prepend_header(args, out): @@ -140,7 +250,11 @@ def prepend_header(args, out): out.write(header.read()) -def commands(parser, args): +def _commands(parser, args): + """This is the 'regular' command, which can be called multiple times. + + See ``commands()`` below for ``--update-completion`` handling. + """ formatter = formatters[args.format] # check header first so we don't open out files unnecessarily @@ -148,12 +262,14 @@ def commands(parser, args): tty.die("No such file: '%s'" % args.header) # if we're updating an existing file, only write output if a command - # is newer than the file. + # or the header is newer than the file. if args.update: if os.path.exists(args.update): files = [ spack.cmd.get_module(command).__file__.rstrip('c') # pyc -> py for command in spack.cmd.all_commands()] + if args.header: + files.append(args.header) last_update = os.path.getmtime(args.update) if not any(os.path.getmtime(f) > last_update for f in files): tty.msg('File is up to date: %s' % args.update) @@ -164,6 +280,37 @@ def commands(parser, args): prepend_header(args, f) formatter(args, f) + if args.update_completion: + fs.set_executable(args.update) + else: prepend_header(args, sys.stdout) formatter(args, sys.stdout) + + +def update_completion(parser, args): + """Iterate through the shells and update the standard completion files. + + This is a convenience method to avoid calling this command many + times, and to simplify completion update for developers. + + """ + for shell, shell_args in update_completion_args.items(): + for attr, value in shell_args.items(): + setattr(args, attr, value) + _commands(parser, args) + + +def commands(parser, args): + if args.update_completion: + if args.format != 'names' or any([ + args.aliases, args.update, args.header + ]): + tty.die("--update-completion can only be specified alone.") + + # this runs the command multiple times with different arguments + return update_completion(parser, args) + + else: + # run commands normally + return _commands(parser, args) diff --git a/lib/spack/spack/cmd/common/__init__.py b/lib/spack/spack/cmd/common/__init__.py index ea7eced004..00804493cd 100644 --- a/lib/spack/spack/cmd/common/__init__.py +++ b/lib/spack/spack/cmd/common/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/common/arguments.py b/lib/spack/spack/cmd/common/arguments.py index 8a38d09f03..b93f265c7a 100644 --- a/lib/spack/spack/cmd/common/arguments.py +++ b/lib/spack/spack/cmd/common/arguments.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -18,9 +18,22 @@ from spack.util.pattern import Args __all__ = ['add_common_arguments'] +#: dictionary of argument-generating functions, keyed by name _arguments = {} +def arg(fn): + """Decorator for a function that generates a common argument. + + This ensures that argument bunches are created lazily. Decorate + argument-generating functions below with @arg so that + ``add_common_arguments()`` can find them. + + """ + _arguments[fn.__name__] = fn + return fn + + def add_common_arguments(parser, list_of_arguments): """Extend a parser with extra arguments @@ -32,7 +45,8 @@ def add_common_arguments(parser, list_of_arguments): if argument not in _arguments: message = 'Trying to add non existing argument "{0}" to a command' raise KeyError(message.format(argument)) - x = _arguments[argument] + + x = _arguments[argument]() parser.add_argument(*x.flags, **x.kwargs) @@ -106,7 +120,7 @@ class SetParallelJobs(argparse.Action): class DeptypeAction(argparse.Action): - """Creates a tuple of valid dependency tpyes from a deptype argument.""" + """Creates a tuple of valid dependency types from a deptype argument.""" def __call__(self, parser, namespace, values, option_string=None): deptype = dep.all_deptypes if values: @@ -118,64 +132,146 @@ class DeptypeAction(argparse.Action): setattr(namespace, self.dest, deptype) -_arguments['constraint'] = Args( - 'constraint', nargs=argparse.REMAINDER, action=ConstraintAction, - help='constraint to select a subset of installed packages') - -_arguments['yes_to_all'] = Args( - '-y', '--yes-to-all', action='store_true', dest='yes_to_all', - help='assume "yes" is the answer to every confirmation request') - -_arguments['recurse_dependencies'] = Args( - '-r', '--dependencies', action='store_true', dest='recurse_dependencies', - help='recursively traverse spec dependencies') - -_arguments['recurse_dependents'] = Args( - '-R', '--dependents', action='store_true', dest='dependents', - help='also uninstall any packages that depend on the ones given ' - 'via command line') - -_arguments['clean'] = Args( - '--clean', - action='store_false', - default=spack.config.get('config:dirty'), - dest='dirty', - help='unset harmful variables in the build environment (default)') - -_arguments['deptype'] = Args( - '--deptype', action=DeptypeAction, default=dep.all_deptypes, - help="comma-separated list of deptypes to traverse\ndefault=%s" - % ','.join(dep.all_deptypes)) - -_arguments['dirty'] = Args( - '--dirty', - action='store_true', - default=spack.config.get('config:dirty'), - dest='dirty', - help='preserve user environment in the spack build environment (danger!)') - -_arguments['long'] = Args( - '-l', '--long', action='store_true', - help='show dependency hashes as well as versions') - -_arguments['very_long'] = Args( - '-L', '--very-long', action='store_true', - help='show full dependency hashes as well as versions') - -_arguments['tags'] = Args( - '-t', '--tags', action='append', - help='filter a package query by tags') - -_arguments['jobs'] = Args( - '-j', '--jobs', action=SetParallelJobs, type=int, dest='jobs', - help='explicitly set number of parallel jobs') - -_arguments['install_status'] = Args( - '-I', '--install-status', action='store_true', default=False, - help='show install status of packages. packages can be: ' - 'installed [+], missing and needed by an installed package [-], ' - 'or not installed (no annotation)') - -_arguments['no_checksum'] = Args( - '-n', '--no-checksum', action='store_true', default=False, - help="do not use checksums to verify downloaded files (unsafe)") +# TODO: merge constraint and installed_specs +@arg +def constraint(): + return Args( + 'constraint', nargs=argparse.REMAINDER, action=ConstraintAction, + help='constraint to select a subset of installed packages', + metavar='installed_specs') + + +@arg +def package(): + return Args('package', help='package name') + + +@arg +def packages(): + return Args( + 'packages', nargs='+', help='one or more package names', + metavar='package') + + +# Specs must use `nargs=argparse.REMAINDER` because a single spec can +# contain spaces, and contain variants like '-mpi' that argparse thinks +# are a collection of optional flags. +@arg +def spec(): + return Args('spec', nargs=argparse.REMAINDER, help='package spec') + + +@arg +def specs(): + return Args( + 'specs', nargs=argparse.REMAINDER, help='one or more package specs') + + +@arg +def installed_spec(): + return Args( + 'spec', nargs=argparse.REMAINDER, help='installed package spec', + metavar='installed_spec') + + +@arg +def installed_specs(): + return Args( + 'specs', nargs=argparse.REMAINDER, + help='one or more installed package specs', metavar='installed_specs') + + +@arg +def yes_to_all(): + return Args( + '-y', '--yes-to-all', action='store_true', dest='yes_to_all', + help='assume "yes" is the answer to every confirmation request') + + +@arg +def recurse_dependencies(): + return Args( + '-r', '--dependencies', action='store_true', + dest='recurse_dependencies', + help='recursively traverse spec dependencies') + + +@arg +def recurse_dependents(): + return Args( + '-R', '--dependents', action='store_true', dest='dependents', + help='also uninstall any packages that depend on the ones given ' + 'via command line') + + +@arg +def clean(): + return Args( + '--clean', + action='store_false', + default=spack.config.get('config:dirty'), + dest='dirty', + help='unset harmful variables in the build environment (default)') + + +@arg +def deptype(): + return Args( + '--deptype', action=DeptypeAction, default=dep.all_deptypes, + help="comma-separated list of deptypes to traverse\ndefault=%s" + % ','.join(dep.all_deptypes)) + + +@arg +def dirty(): + return Args( + '--dirty', + action='store_true', + default=spack.config.get('config:dirty'), + dest='dirty', + help="preserve user environment in spack's build environment (danger!)" + ) + + +@arg +def long(): + return Args( + '-l', '--long', action='store_true', + help='show dependency hashes as well as versions') + + +@arg +def very_long(): + return Args( + '-L', '--very-long', action='store_true', + help='show full dependency hashes as well as versions') + + +@arg +def tags(): + return Args( + '-t', '--tags', action='append', + help='filter a package query by tags') + + +@arg +def jobs(): + return Args( + '-j', '--jobs', action=SetParallelJobs, type=int, dest='jobs', + help='explicitly set number of parallel jobs') + + +@arg +def install_status(): + return Args( + '-I', '--install-status', action='store_true', default=False, + help='show install status of packages. packages can be: ' + 'installed [+], missing and needed by an installed package [-], ' + 'or not installed (no annotation)') + + +@arg +def no_checksum(): + return Args( + '-n', '--no-checksum', action='store_true', default=False, + help="do not use checksums to verify downloaded files (unsafe)") diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index c52a3c7081..c9647898ef 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index a67735fd8d..c29e2257b7 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/concretize.py b/lib/spack/spack/cmd/concretize.py index dd88fe8534..d28f7b4a5d 100644 --- a/lib/spack/spack/cmd/concretize.py +++ b/lib/spack/spack/cmd/concretize.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -18,6 +18,7 @@ def setup_parser(subparser): def concretize(parser, args): env = ev.get_env(args, 'concretize', required=True) - concretized_specs = env.concretize(force=args.force) - ev.display_specs(concretized_specs) - env.write() + with env.write_transaction(): + concretized_specs = env.concretize(force=args.force) + ev.display_specs(concretized_specs) + env.write() diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 2a11af9f51..b1a6454555 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -34,7 +34,7 @@ def setup_parser(subparser): help="configuration section to print. " "options: %(choices)s", nargs='?', - metavar='SECTION', + metavar='section', choices=spack.config.section_schemas) blame_parser = sp.add_parser( @@ -42,20 +42,22 @@ def setup_parser(subparser): blame_parser.add_argument('section', help="configuration section to print. " "options: %(choices)s", - metavar='SECTION', + metavar='section', choices=spack.config.section_schemas) edit_parser = sp.add_parser('edit', help='edit configuration file') edit_parser.add_argument('section', help="configuration section to edit. " "options: %(choices)s", - metavar='SECTION', + metavar='section', nargs='?', choices=spack.config.section_schemas) edit_parser.add_argument( '--print-file', action='store_true', help="print the file name that would be edited") + sp.add_parser('list', help='list configuration sections') + def _get_scope_and_section(args): """Extract config scope and section from arguments.""" @@ -83,7 +85,6 @@ def config_get(args): With no arguments and an active environment, print the contents of the environment's manifest file (spack.yaml). - """ scope, section = _get_scope_and_section(args) @@ -113,7 +114,6 @@ def config_edit(args): With no arguments and an active environment, edit the spack.yaml for the active environment. - """ scope, section = _get_scope_and_section(args) if not scope and not section: @@ -127,8 +127,19 @@ def config_edit(args): editor(config_file) +def config_list(args): + """List the possible configuration sections. + + Used primarily for shell tab completion scripts. + """ + print(' '.join(list(spack.config.section_schemas))) + + def config(parser, args): - action = {'get': config_get, - 'blame': config_blame, - 'edit': config_edit} + action = { + 'get': config_get, + 'blame': config_blame, + 'edit': config_edit, + 'list': config_list, + } action[args.config_command](args) diff --git a/lib/spack/spack/cmd/configure.py b/lib/spack/spack/cmd/configure.py index 10a1294a41..3df3c87413 100644 --- a/lib/spack/spack/cmd/configure.py +++ b/lib/spack/spack/cmd/configure.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -7,6 +7,7 @@ import argparse import llnl.util.tty as tty import spack.cmd +import spack.cmd.common.arguments as arguments import spack.cmd.install as inst from spack.build_systems.autotools import AutotoolsPackage @@ -37,15 +38,11 @@ build_system_to_phase = { def setup_parser(subparser): subparser.add_argument( - 'package', - nargs=argparse.REMAINDER, - help="spec of the package to install" - ) - subparser.add_argument( '-v', '--verbose', action='store_true', help="print additional output during builds" ) + arguments.add_common_arguments(subparser, ['spec']) def _stop_at_phase_during_install(args, calling_fn, phase_mapping): @@ -64,15 +61,15 @@ def _stop_at_phase_during_install(args, calling_fn, phase_mapping): # Install package dependencies if needed parser = argparse.ArgumentParser() inst.setup_parser(parser) - tty.msg('Checking dependencies for {0}'.format(args.package[0])) + tty.msg('Checking dependencies for {0}'.format(args.spec[0])) cli_args = ['-v'] if args.verbose else [] install_args = parser.parse_args(cli_args + ['--only=dependencies']) - install_args.package = args.package + install_args.spec = args.spec inst.install(parser, install_args) # Install package and stop at the given phase cli_args = ['-v'] if args.verbose else [] install_args = parser.parse_args(cli_args + ['--only=package']) - install_args.package = args.package + install_args.spec = args.spec inst.install(parser, install_args, stop_at=phase) except IndexError: tty.error( diff --git a/lib/spack/spack/cmd/containerize.py b/lib/spack/spack/cmd/containerize.py new file mode 100644 index 0000000000..cc2c001560 --- /dev/null +++ b/lib/spack/spack/cmd/containerize.py @@ -0,0 +1,25 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os +import os.path +import spack.container + +description = ("creates recipes to build images for different" + " container runtimes") +section = "container" +level = "long" + + +def containerize(parser, args): + config_dir = args.env_dir or os.getcwd() + config_file = os.path.abspath(os.path.join(config_dir, 'spack.yaml')) + if not os.path.exists(config_file): + msg = 'file not found: {0}' + raise ValueError(msg.format(config_file)) + + config = spack.container.validate(config_file) + + recipe = spack.container.recipe(config) + print(recipe) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index ee0fb3a347..f9b7a382ea 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -28,7 +28,7 @@ level = "short" package_template = '''\ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -246,6 +246,7 @@ class PythonPackageTemplate(PackageTemplate): dependencies = """\ # FIXME: Add dependencies if required. + # depends_on('python@2.X:2.Y,3.Z:', type=('build', 'run')) # depends_on('py-setuptools', type='build') # depends_on('py-foo', type=('build', 'run'))""" @@ -426,7 +427,8 @@ def setup_parser(subparser): '-n', '--name', help="name of the package to create") subparser.add_argument( - '-t', '--template', metavar='TEMPLATE', choices=templates.keys(), + '-t', '--template', metavar='TEMPLATE', + choices=sorted(templates.keys()), help="build system template to use. options: %(choices)s") subparser.add_argument( '-r', '--repo', @@ -458,7 +460,7 @@ class BuildSystemGuesser: the contents of its archive or the URL it was downloaded from.""" # Most octave extensions are hosted on Octave-Forge: - # http://octave.sourceforge.net/index.html + # https://octave.sourceforge.net/index.html # They all have the same base URL. if url is not None and 'downloads.sourceforge.net/octave/' in url: self.build_system = 'octave' @@ -471,13 +473,13 @@ class BuildSystemGuesser: # build systems, we choose the first match in this list. clues = [ (r'/CMakeLists\.txt$', 'cmake'), + (r'/NAMESPACE$', 'r'), (r'/configure$', 'autotools'), (r'/configure\.(in|ac)$', 'autoreconf'), (r'/Makefile\.am$', 'autoreconf'), (r'/SConstruct$', 'scons'), (r'/waf$', 'waf'), (r'/setup\.py$', 'python'), - (r'/NAMESPACE$', 'r'), (r'/WORKSPACE$', 'bazel'), (r'/Build\.PL$', 'perlbuild'), (r'/Makefile\.PL$', 'perlmake'), @@ -571,7 +573,7 @@ def get_url(args): """ # Default URL - url = 'http://www.example.com/example-1.2.3.tar.gz' + url = 'https://www.example.com/example-1.2.3.tar.gz' if args.url: # Use a user-supplied URL if one is present diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 722112ead2..3c72531a9c 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -1,12 +1,12 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse import llnl.util.tty as tty import spack.cmd +import spack.cmd.common.arguments as arguments import spack.environment as ev import spack.store from spack.filesystem_view import YamlFilesystemView @@ -28,9 +28,7 @@ def setup_parser(subparser): '-a', '--all', action='store_true', help="deactivate all extensions of an extendable package, or " "deactivate an extension AND its dependencies") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help="spec of package extension to deactivate") + arguments.add_common_arguments(subparser, ['installed_spec']) def deactivate(parser, args): diff --git a/lib/spack/spack/cmd/debug.py b/lib/spack/spack/cmd/debug.py index e729373194..4fc39251af 100644 --- a/lib/spack/spack/cmd/debug.py +++ b/lib/spack/spack/cmd/debug.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/dependencies.py b/lib/spack/spack/cmd/dependencies.py index 671224edb4..e65e050bfa 100644 --- a/lib/spack/spack/cmd/dependencies.py +++ b/lib/spack/spack/cmd/dependencies.py @@ -1,10 +1,8 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse - import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -31,8 +29,7 @@ def setup_parser(subparser): subparser.add_argument( '-V', '--no-expand-virtuals', action='store_false', default=True, dest="expand_virtuals", help="do not expand virtual dependencies") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, help="spec or package name") + arguments.add_common_arguments(subparser, ['spec']) def dependencies(parser, args): diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index 368dae8f55..e60733f589 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -1,17 +1,16 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse - import llnl.util.tty as tty from llnl.util.tty.colify import colify +import spack.cmd +import spack.cmd.common.arguments as arguments import spack.environment as ev import spack.repo import spack.store -import spack.cmd description = "show packages that depend on another" section = "basic" @@ -26,8 +25,7 @@ def setup_parser(subparser): subparser.add_argument( '-t', '--transitive', action='store_true', default=False, help="Show all transitive dependents.") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, help="spec or package name") + arguments.add_common_arguments(subparser, ['spec']) def inverted_dependencies(): diff --git a/lib/spack/spack/cmd/deprecate.py b/lib/spack/spack/cmd/deprecate.py index 286d2e995a..6172033fa7 100644 --- a/lib/spack/spack/cmd/deprecate.py +++ b/lib/spack/spack/cmd/deprecate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/dev_build.py b/lib/spack/spack/cmd/dev_build.py index 7c92b004ca..c1004f24b3 100644 --- a/lib/spack/spack/cmd/dev_build.py +++ b/lib/spack/spack/cmd/dev_build.py @@ -1,18 +1,17 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import sys import os -import argparse import llnl.util.tty as tty import spack.config import spack.cmd -import spack.repo import spack.cmd.common.arguments as arguments +import spack.repo from spack.stage import DIYStage description = "developer build: build from code in current working directory" @@ -41,9 +40,7 @@ def setup_parser(subparser): subparser.add_argument( '-u', '--until', type=str, dest='until', default=None, help="phase to stop after when installing (default None)") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help="specs to use for install. must contain package AND version") + arguments.add_common_arguments(subparser, ['spec']) cd_group = subparser.add_mutually_exclusive_group() arguments.add_common_arguments(cd_group, ['clean', 'dirty']) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 127a6bbed1..186745d80c 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/docs.py b/lib/spack/spack/cmd/docs.py index 696bfb3a13..a7f2a641ee 100644 --- a/lib/spack/spack/cmd/docs.py +++ b/lib/spack/spack/cmd/docs.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index cca52618e8..6cdc3b788d 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -84,12 +84,11 @@ def setup_parser(subparser): help="namespace of package to edit") subparser.add_argument( - 'name', nargs='?', default=None, - help="name of package to edit") + 'package', nargs='?', default=None, help="package name") def edit(parser, args): - name = args.name + name = args.package # By default, edit package files path = spack.paths.packages_path diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index 64ba9d53a4..a8bc1e5bbe 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -157,7 +157,7 @@ def env_deactivate(args): def env_create_setup_parser(subparser): """create a new environment""" subparser.add_argument( - 'create_env', metavar='ENV', help='name of environment to create') + 'create_env', metavar='env', help='name of environment to create') subparser.add_argument( '-d', '--dir', action='store_true', help='create an environment in a specific directory') @@ -221,7 +221,7 @@ def _env_create(name_or_path, init_file=None, dir=False, with_view=None): def env_remove_setup_parser(subparser): """remove an existing environment""" subparser.add_argument( - 'rm_env', metavar='ENV', nargs='+', + 'rm_env', metavar='env', nargs='+', help='environment(s) to remove') arguments.add_common_arguments(subparser, ['yes_to_all']) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index c9dcf6c4eb..e834d7fd18 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -37,7 +37,7 @@ def setup_parser(subparser): subparser.add_argument( 'spec', nargs=argparse.REMAINDER, - help='spec of package to list extensions for') + help='spec of package to list extensions for', metavar='extendable') def extensions(parser, args): diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 68ec635d26..b91eb52ab8 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -1,16 +1,14 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse - import llnl.util.tty as tty import spack.cmd +import spack.cmd.common.arguments as arguments import spack.config import spack.repo -import spack.cmd.common.arguments as arguments description = "fetch archives for packages" section = "build" @@ -25,19 +23,17 @@ def setup_parser(subparser): subparser.add_argument( '-D', '--dependencies', action='store_true', help="also fetch all dependencies") - subparser.add_argument( - 'packages', nargs=argparse.REMAINDER, - help="specs of packages to fetch") + arguments.add_common_arguments(subparser, ['specs']) def fetch(parser, args): - if not args.packages: + if not args.specs: tty.die("fetch requires at least one package argument") if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') - specs = spack.cmd.parse_specs(args.packages, concretize=True) + specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: if args.missing or args.dependencies: for s in spec.traverse(): diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 37cfcd30d5..fa36e1cd26 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -1,10 +1,13 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function +import copy +import os + import llnl.util.tty as tty import llnl.util.tty.color as color import llnl.util.lang @@ -13,6 +16,7 @@ import spack.environment as ev import spack.repo import spack.cmd as cmd import spack.cmd.common.arguments as arguments +import spack.user_environment as uenv from spack.util.string import plural from spack.database import InstallStatuses @@ -80,6 +84,9 @@ def setup_parser(subparser): action='store_true', dest='variants', help='show variants in output (can be long)') + subparser.add_argument( + '--loaded', action='store_true', + help='show only packages loaded in the user environment') subparser.add_argument('-M', '--only-missing', action='store_true', dest='only_missing', @@ -166,11 +173,16 @@ def display_env(env, args, decorator): else: tty.msg('Root specs') + # Root specs cannot be displayed with prefixes, since those are not + # set for abstract specs. Same for hashes + root_args = copy.copy(args) + root_args.paths = False + # Roots are displayed with variants, etc. so that we can see # specifically what the user asked for. cmd.display_specs( env.user_specs, - args, + root_args, decorator=lambda s, f: color.colorize('@*{%s}' % f), namespace=True, show_flags=True, @@ -214,6 +226,10 @@ def find(parser, args): packages_with_tags = spack.repo.path.packages_with_tags(*args.tags) results = [x for x in results if x.name in packages_with_tags] + if args.loaded: + hashes = os.environ.get(uenv.spack_loaded_hashes_var, '').split(':') + results = [x for x in results if x.dag_hash() in hashes] + # Display the result if args.json: cmd.display_specs_as_json(results, deps=args.deps) diff --git a/lib/spack/spack/cmd/flake8.py b/lib/spack/spack/cmd/flake8.py index cc5b168c4b..cdfce2cab2 100644 --- a/lib/spack/spack/cmd/flake8.py +++ b/lib/spack/spack/cmd/flake8.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/gc.py b/lib/spack/spack/cmd/gc.py new file mode 100644 index 0000000000..ca9d88169a --- /dev/null +++ b/lib/spack/spack/cmd/gc.py @@ -0,0 +1,47 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import llnl.util.tty as tty + +import spack.cmd.common.arguments +import spack.cmd.uninstall +import spack.environment +import spack.store + +description = "remove specs that are now no longer needed" +section = "build" +level = "short" + + +def setup_parser(subparser): + spack.cmd.common.arguments.add_common_arguments(subparser, ['yes_to_all']) + + +def gc(parser, args): + specs = spack.store.db.unused_specs + + # Restrict garbage collection to the active environment + # speculating over roots that are yet to be installed + env = spack.environment.get_env(args=None, cmd_name='gc') + if env: + msg = 'Restricting the garbage collection to the "{0}" environment' + tty.msg(msg.format(env.name)) + env.concretize() + roots = [s for s in env.roots()] + all_hashes = set([s.dag_hash() for r in roots for s in r.traverse()]) + lr_hashes = set([s.dag_hash() for r in roots + for s in r.traverse(deptype=('link', 'run'))]) + maybe_to_be_removed = all_hashes - lr_hashes + specs = [s for s in specs if s.dag_hash() in maybe_to_be_removed] + + if not specs: + msg = "There are no unused specs. Spack's store is clean." + tty.msg(msg) + return + + if not args.yes_to_all: + spack.cmd.uninstall.confirm_removal(specs) + + spack.cmd.uninstall.do_uninstall(None, specs, force=False) diff --git a/lib/spack/spack/cmd/gpg.py b/lib/spack/spack/cmd/gpg.py index dcd4f798a2..0a77812c12 100644 --- a/lib/spack/spack/cmd/gpg.py +++ b/lib/spack/spack/cmd/gpg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,6 +6,7 @@ import os import argparse +import spack.cmd.common.arguments as arguments import spack.paths from spack.util.gpg import Gpg @@ -19,8 +20,7 @@ def setup_parser(subparser): subparsers = subparser.add_subparsers(help='GPG sub-commands') verify = subparsers.add_parser('verify', help=gpg_verify.__doc__) - verify.add_argument('package', type=str, - help='the package to verify') + arguments.add_common_arguments(verify, ['installed_spec']) verify.add_argument('signature', type=str, nargs='?', help='the signature file') verify.set_defaults(func=gpg_verify) @@ -44,8 +44,7 @@ def setup_parser(subparser): help='the key to use for signing') sign.add_argument('--clearsign', action='store_true', help='if specified, create a clearsign signature') - sign.add_argument('package', type=str, - help='the package to sign') + arguments.add_common_arguments(sign, ['installed_spec']) sign.set_defaults(func=gpg_sign) create = subparsers.add_parser('create', help=gpg_create.__doc__) @@ -122,9 +121,9 @@ def gpg_sign(args): 'please choose one') output = args.output if not output: - output = args.package + '.asc' + output = args.spec[0] + '.asc' # TODO: Support the package format Spack creates. - Gpg.sign(key, args.package, output, args.clearsign) + Gpg.sign(key, ' '.join(args.spec), output, args.clearsign) def gpg_trust(args): @@ -155,8 +154,8 @@ def gpg_verify(args): # TODO: Support the package format Spack creates. signature = args.signature if signature is None: - signature = args.package + '.asc' - Gpg.verify(signature, args.package) + signature = args.spec[0] + '.asc' + Gpg.verify(signature, ' '.join(args.spec)) def gpg(parser, args): diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py index c07b009286..d0fbf8e6c6 100644 --- a/lib/spack/spack/cmd/graph.py +++ b/lib/spack/spack/cmd/graph.py @@ -1,11 +1,10 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function -import argparse import llnl.util.tty as tty import spack.cmd @@ -38,11 +37,7 @@ def setup_parser(subparser): '-i', '--installed', action='store_true', help="graph all installed specs in dot format (implies --dot)") - arguments.add_common_arguments(subparser, ['deptype']) - - subparser.add_argument( - 'specs', nargs=argparse.REMAINDER, - help="specs of packages to graph") + arguments.add_common_arguments(subparser, ['deptype', 'specs']) def graph(parser, args): diff --git a/lib/spack/spack/cmd/help.py b/lib/spack/spack/cmd/help.py index 8178ef79ed..3af82312df 100644 --- a/lib/spack/spack/cmd/help.py +++ b/lib/spack/spack/cmd/help.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 37a1aeb0a4..81a68dae96 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,6 +11,7 @@ from six.moves import zip_longest import llnl.util.tty.color as color from llnl.util.tty.colify import colify +import spack.cmd.common.arguments as arguments import spack.repo import spack.spec import spack.fetch_strategy as fs @@ -36,8 +37,7 @@ def padder(str_list, extra=0): def setup_parser(subparser): - subparser.add_argument( - 'name', metavar='PACKAGE', help='name of package to get info for') + arguments.add_common_arguments(subparser, ['package']) def section_title(s): @@ -106,7 +106,9 @@ class VariantFormatter(object): yield ' None' else: yield ' ' + self.fmt % self.headers - yield '\n' + underline = tuple([l * "=" for l in self.column_widths]) + yield ' ' + self.fmt % underline + yield '' for k, v in sorted(self.variants.items()): name = textwrap.wrap( '{0} [{1}]'.format(k, self.default(v)), @@ -235,5 +237,5 @@ def print_text_info(pkg): def info(parser, args): - pkg = spack.repo.get(args.name) + pkg = spack.repo.get(args.package) print_text_info(pkg) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index ab012eaead..9e13643920 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -7,6 +7,7 @@ import argparse import os import shutil import sys +import textwrap import llnl.util.filesystem as fs import llnl.util.tty as tty @@ -41,7 +42,8 @@ def update_kwargs_from_args(args, kwargs): 'use_cache': args.use_cache, 'cache_only': args.cache_only, 'explicit': True, # Always true for install command - 'stop_at': args.until + 'stop_at': args.until, + 'unsigned': args.unsigned, }) kwargs.update({ @@ -72,7 +74,7 @@ the dependencies""" subparser.add_argument( '-u', '--until', type=str, dest='until', default=None, help="phase to stop after when installing (default None)") - arguments.add_common_arguments(subparser, ['jobs', 'install_status']) + arguments.add_common_arguments(subparser, ['jobs']) subparser.add_argument( '--overwrite', action='store_true', help="reinstall an existing spec, even if it has dependents") @@ -98,6 +100,10 @@ the dependencies""" help="only install package from binary mirrors") subparser.add_argument( + '--no-check-signature', action='store_true', + dest='unsigned', default=False, + help="do not check signatures of binary packages") + subparser.add_argument( '--show-log-on-error', action='store_true', help="print full build log to stderr if build fails") subparser.add_argument( @@ -121,11 +127,6 @@ the dependencies""" cd_group = subparser.add_mutually_exclusive_group() arguments.add_common_arguments(cd_group, ['clean', 'dirty']) - subparser.add_argument( - 'package', - nargs=argparse.REMAINDER, - help="spec of the package to install" - ) testing = subparser.add_mutually_exclusive_group() testing.add_argument( '--test', default=None, @@ -151,38 +152,62 @@ packages. If neither are chosen, don't run tests for any packages.""" help="filename for the log file. if not passed a default will be used" ) subparser.add_argument( + '--help-cdash', + action='store_true', + help="Show usage instructions for CDash reporting" + ) + add_cdash_args(subparser, False) + arguments.add_common_arguments(subparser, ['yes_to_all', 'spec']) + + +def add_cdash_args(subparser, add_help): + cdash_help = {} + if add_help: + cdash_help['upload-url'] = "CDash URL where reports will be uploaded" + cdash_help['build'] = """The name of the build that will be reported to CDash. +Defaults to spec of the package to install.""" + cdash_help['site'] = """The site name that will be reported to CDash. +Defaults to current system hostname.""" + cdash_help['track'] = """Results will be reported to this group on CDash. +Defaults to Experimental.""" + cdash_help['buildstamp'] = """Instead of letting the CDash reporter prepare the +buildstamp which, when combined with build name, site and project, +uniquely identifies the build, provide this argument to identify +the build yourself. Format: %%Y%%m%%d-%%H%%M-[cdash-track]""" + else: + cdash_help['upload-url'] = argparse.SUPPRESS + cdash_help['build'] = argparse.SUPPRESS + cdash_help['site'] = argparse.SUPPRESS + cdash_help['track'] = argparse.SUPPRESS + cdash_help['buildstamp'] = argparse.SUPPRESS + + subparser.add_argument( '--cdash-upload-url', default=None, - help="CDash URL where reports will be uploaded" + help=cdash_help['upload-url'] ) subparser.add_argument( '--cdash-build', default=None, - help="""The name of the build that will be reported to CDash. -Defaults to spec of the package to install.""" + help=cdash_help['build'] ) subparser.add_argument( '--cdash-site', default=None, - help="""The site name that will be reported to CDash. -Defaults to current system hostname.""" + help=cdash_help['site'] ) + cdash_subgroup = subparser.add_mutually_exclusive_group() cdash_subgroup.add_argument( '--cdash-track', default='Experimental', - help="""Results will be reported to this group on CDash. -Defaults to Experimental.""" + help=cdash_help['track'] ) cdash_subgroup.add_argument( '--cdash-buildstamp', default=None, - help="""Instead of letting the CDash reporter prepare the -buildstamp which, when combined with build name, site and project, -uniquely identifies the build, provide this argument to identify -the build yourself. Format: %%Y%%m%%d-%%H%%M-[cdash-track]""" + help=cdash_help['buildstamp'] ) - arguments.add_common_arguments(subparser, ['yes_to_all']) def default_log_file(spec): @@ -203,8 +228,13 @@ def install_spec(cli_args, kwargs, abstract_spec, spec): # handle active environment, if any env = ev.get_env(cli_args, 'install') if env: - env.install(abstract_spec, spec, **kwargs) - env.write() + with env.write_transaction(): + concrete = env.concretize_and_add( + abstract_spec, spec) + env.write(regenerate_views=False) + env._install(concrete, **kwargs) + with env.write_transaction(): + env.regenerate_views() else: spec.package.do_install(**kwargs) @@ -221,22 +251,38 @@ def install_spec(cli_args, kwargs, abstract_spec, spec): def install(parser, args, **kwargs): - if not args.package and not args.specfiles: + if args.help_cdash: + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent('''\ +environment variables: + SPACK_CDASH_AUTH_TOKEN + authentication token to present to CDash + ''')) + add_cdash_args(parser, True) + parser.print_help() + return + + if not args.spec and not args.specfiles: # if there are no args but an active environment or spack.yaml file # then install the packages from it. env = ev.get_env(args, 'install') if env: if not args.only_concrete: - concretized_specs = env.concretize() - ev.display_specs(concretized_specs) + with env.write_transaction(): + concretized_specs = env.concretize() + ev.display_specs(concretized_specs) - # save view regeneration for later, so that we only do it - # once, as it can be slow. - env.write(regenerate_views=False) + # save view regeneration for later, so that we only do it + # once, as it can be slow. + env.write(regenerate_views=False) tty.msg("Installing environment %s" % env.name) env.install_all(args) - env.regenerate_views() + with env.write_transaction(): + # It is not strictly required to synchronize view regeneration + # but doing so can prevent redundant work in the filesystem. + env.regenerate_views() return else: tty.die("install requires a package argument or a spack.yaml file") @@ -256,7 +302,7 @@ def install(parser, args, **kwargs): if args.log_file: reporter.filename = args.log_file - abstract_specs = spack.cmd.parse_specs(args.package) + abstract_specs = spack.cmd.parse_specs(args.spec) tests = False if args.test == 'all' or args.run_tests: tests = True @@ -266,7 +312,7 @@ def install(parser, args, **kwargs): try: specs = spack.cmd.parse_specs( - args.package, concretize=True, tests=tests) + args.spec, concretize=True, tests=tests) except SpackError as e: tty.debug(e) reporter.concretization_report(e.message) diff --git a/lib/spack/spack/cmd/license.py b/lib/spack/spack/cmd/license.py index 4c69dba0e7..5fd57c39c9 100644 --- a/lib/spack/spack/cmd/license.py +++ b/lib/spack/spack/cmd/license.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -124,7 +124,7 @@ def _check_license(lines, path): r'SPDX-License-Identifier: \(Apache-2\.0 OR MIT\)' ] - strict_date = r'Copyright 2013-2019' + strict_date = r'Copyright 2013-2020' found = [] diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index fe6983ce74..656282599d 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -23,7 +23,7 @@ import spack.cmd.common.arguments as arguments from spack.version import VersionList if sys.version_info > (3, 1): - from html import escape + from html import escape # novm else: from cgi import escape diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index ef8e29816a..09f3fd31ee 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -1,25 +1,80 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse -from spack.cmd.common import print_module_placeholder_help, arguments +import sys -description = "add package to environment using `module load`" -section = "modules" +import llnl.util.tty as tty + +import spack.cmd +import spack.cmd.common.arguments as arguments +import spack.environment as ev +import spack.util.environment +import spack.user_environment as uenv +import spack.store + +description = "add package to the user environment" +section = "user environment" level = "short" def setup_parser(subparser): """Parser is only constructed so that this prints a nice help message with -h. """ + arguments.add_common_arguments( + subparser, ['recurse_dependencies', 'installed_specs']) + + shells = subparser.add_mutually_exclusive_group() + shells.add_argument( + '--sh', action='store_const', dest='shell', const='sh', + help="print sh commands to load the package") + shells.add_argument( + '--csh', action='store_const', dest='shell', const='csh', + help="print csh commands to load the package") + subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help="spec of package to load with modules " + '--only', + default='package,dependencies', + dest='things_to_load', + choices=['package', 'dependencies'], + help="""select whether to load the package and its dependencies +the default is to load the package and all dependencies +alternatively one can decide to load only the package or only +the dependencies""" ) - arguments.add_common_arguments(subparser, ['recurse_dependencies']) def load(parser, args): - print_module_placeholder_help() + env = ev.get_env(args, 'load') + specs = [spack.cmd.disambiguate_spec(spec, env) + for spec in spack.cmd.parse_specs(args.specs)] + + if not args.shell: + msg = [ + "This command works best with Spack's shell support", + "" + ] + spack.cmd.common.shell_init_instructions + [ + 'Or, if you want to use `spack load` without initializing', + 'shell support, you can run one of these:', + '', + ' eval `spack load --sh %s` # for bash/sh' % args.specs, + ' eval `spack load --csh %s` # for csh/tcsh' % args.specs, + ] + tty.msg(*msg) + return 1 + + with spack.store.db.read_transaction(): + if 'dependencies' in args.things_to_load: + include_roots = 'package' in args.things_to_load + specs = [dep for spec in specs + for dep in + spec.traverse(root=include_roots, order='post')] + + env_mod = spack.util.environment.EnvironmentModifications() + for spec in specs: + env_mod.extend(uenv.environment_modifications_for_spec(spec)) + env_mod.prepend_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) + cmds = env_mod.shell_modifications(args.shell) + + sys.stdout.write(cmds) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index d6716233f9..60978fe404 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,11 +6,11 @@ from __future__ import print_function import os -import argparse import llnl.util.tty as tty import spack.environment as ev import spack.cmd +import spack.cmd.common.arguments as arguments import spack.environment import spack.paths import spack.repo @@ -55,9 +55,7 @@ def setup_parser(subparser): '-e', '--env', action='store', help="location of an environment managed by spack") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help="spec of package to fetch directory for") + arguments.add_common_arguments(subparser, ['spec']) def location(parser, args): diff --git a/lib/spack/spack/cmd/log_parse.py b/lib/spack/spack/cmd/log_parse.py index 6932047511..faa44a4ef0 100644 --- a/lib/spack/spack/cmd/log_parse.py +++ b/lib/spack/spack/cmd/log_parse.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/maintainers.py b/lib/spack/spack/cmd/maintainers.py index 361437ece5..a1cf477146 100644 --- a/lib/spack/spack/cmd/maintainers.py +++ b/lib/spack/spack/cmd/maintainers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -40,7 +40,7 @@ def setup_parser(subparser): # options for commands that take package arguments subparser.add_argument( - 'pkg_or_user', nargs=argparse.REMAINDER, + 'package_or_user', nargs=argparse.REMAINDER, help='names of packages or users to get info for') @@ -104,31 +104,31 @@ def maintainers(parser, args): if args.all: if args.by_user: - maintainers = maintainers_to_packages(args.pkg_or_user) + maintainers = maintainers_to_packages(args.package_or_user) for user, packages in sorted(maintainers.items()): color.cprint('@c{%s}: %s' % (user, ', '.join(sorted(packages)))) return 0 if maintainers else 1 else: - packages = packages_to_maintainers(args.pkg_or_user) + packages = packages_to_maintainers(args.package_or_user) for pkg, maintainers in sorted(packages.items()): color.cprint('@c{%s}: %s' % (pkg, ', '.join(sorted(maintainers)))) return 0 if packages else 1 if args.by_user: - if not args.pkg_or_user: + if not args.package_or_user: tty.die('spack maintainers --by-user requires a user or --all') - packages = union_values(maintainers_to_packages(args.pkg_or_user)) + packages = union_values(maintainers_to_packages(args.package_or_user)) colify(packages) return 0 if packages else 1 else: - if not args.pkg_or_user: + if not args.package_or_user: tty.die('spack maintainers requires a package or --all') - users = union_values(packages_to_maintainers(args.pkg_or_user)) + users = union_values(packages_to_maintainers(args.package_or_user)) colify(users) return 0 if users else 1 diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 21723e4965..5206927895 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -1,11 +1,10 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import sys -import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -40,9 +39,6 @@ def setup_parser(subparser): help="directory in which to create mirror") create_parser.add_argument( - 'specs', nargs=argparse.REMAINDER, - help="specs of packages to put in mirror") - create_parser.add_argument( '-a', '--all', action='store_true', help="mirror all versions of all packages in Spack, or all packages" " in the current environment if there is an active environment" @@ -57,6 +53,7 @@ def setup_parser(subparser): '-n', '--versions-per-spec', help="the number of versions to fetch for each spec, choose 'all' to" " retrieve all versions of each package") + arguments.add_common_arguments(create_parser, ['specs']) # used to construct scope arguments below scopes = spack.config.scopes() @@ -64,7 +61,8 @@ def setup_parser(subparser): # Add add_parser = sp.add_parser('add', help=mirror_add.__doc__) - add_parser.add_argument('name', help="mnemonic name for mirror") + add_parser.add_argument( + 'name', help="mnemonic name for mirror", metavar="mirror") add_parser.add_argument( 'url', help="url of mirror directory from 'spack mirror create'") add_parser.add_argument( @@ -75,7 +73,8 @@ def setup_parser(subparser): # Remove remove_parser = sp.add_parser('remove', aliases=['rm'], help=mirror_remove.__doc__) - remove_parser.add_argument('name') + remove_parser.add_argument( + 'name', help="mnemonic name for mirror", metavar="mirror") remove_parser.add_argument( '--scope', choices=scopes, metavar=scopes_metavar, default=spack.config.default_modify_scope(), @@ -83,7 +82,8 @@ def setup_parser(subparser): # Set-Url set_url_parser = sp.add_parser('set-url', help=mirror_set_url.__doc__) - set_url_parser.add_argument('name', help="mnemonic name for mirror") + set_url_parser.add_argument( + 'name', help="mnemonic name for mirror", metavar="mirror") set_url_parser.add_argument( 'url', help="url of mirror directory from 'spack mirror create'") set_url_parser.add_argument( diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 86a27786da..f86f3e5f25 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,7 +11,7 @@ import spack.cmd.modules.lmod import spack.cmd.modules.tcl description = "manipulate module files" -section = "modules" +section = "user environment" level = "short" diff --git a/lib/spack/spack/cmd/modules/__init__.py b/lib/spack/spack/cmd/modules/__init__.py index fbf93e9b2b..6555b8c8a7 100644 --- a/lib/spack/spack/cmd/modules/__init__.py +++ b/lib/spack/spack/cmd/modules/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/modules/lmod.py b/lib/spack/spack/cmd/modules/lmod.py index 2725ec4892..d3610bf333 100644 --- a/lib/spack/spack/cmd/modules/lmod.py +++ b/lib/spack/spack/cmd/modules/lmod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/modules/tcl.py b/lib/spack/spack/cmd/modules/tcl.py index 441ea326e0..cdde21e821 100644 --- a/lib/spack/spack/cmd/modules/tcl.py +++ b/lib/spack/spack/cmd/modules/tcl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 258c9035df..8f91edb8f1 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -1,10 +1,8 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse - import llnl.util.tty as tty import spack.repo @@ -18,20 +16,17 @@ level = "long" def setup_parser(subparser): - arguments.add_common_arguments(subparser, ['no_checksum']) - subparser.add_argument( - 'packages', nargs=argparse.REMAINDER, - help="specs of packages to stage") + arguments.add_common_arguments(subparser, ['no_checksum', 'specs']) def patch(parser, args): - if not args.packages: - tty.die("patch requires at least one package argument") + if not args.specs: + tty.die("patch requires at least one spec argument") if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') - specs = spack.cmd.parse_specs(args.packages, concretize=True) + specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) package.do_patch() diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 534e7b9c69..b988d6a848 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,7 +6,6 @@ from __future__ import print_function import os -import argparse import re import llnl.util.tty as tty @@ -14,6 +13,7 @@ from llnl.util.tty.colify import colify from llnl.util.filesystem import working_dir import spack.cmd +import spack.cmd.common.arguments as arguments import spack.paths import spack.repo from spack.util.executable import which @@ -28,8 +28,7 @@ def setup_parser(subparser): metavar='SUBCOMMAND', dest='pkg_command') add_parser = sp.add_parser('add', help=pkg_add.__doc__) - add_parser.add_argument('packages', nargs=argparse.REMAINDER, - help="names of packages to add to git repo") + arguments.add_common_arguments(add_parser, ['packages']) list_parser = sp.add_parser('list', help=pkg_list.__doc__) list_parser.add_argument('rev', default='HEAD', nargs='?', diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 74dc8f4c60..8ca34542eb 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/pydoc.py b/lib/spack/spack/cmd/pydoc.py index 422129ab35..1f81718e38 100644 --- a/lib/spack/spack/cmd/pydoc.py +++ b/lib/spack/spack/cmd/pydoc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 9c0e683fe2..492c8f98e0 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index a309798f7e..b9f3a2ac3b 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/remove.py b/lib/spack/spack/cmd/remove.py index 1d07830f03..ef01052c29 100644 --- a/lib/spack/spack/cmd/remove.py +++ b/lib/spack/spack/cmd/remove.py @@ -1,13 +1,12 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse - import llnl.util.tty as tty import spack.cmd +import spack.cmd.common.arguments as arguments import spack.environment as ev @@ -26,17 +25,17 @@ def setup_parser(subparser): subparser.add_argument( '-f', '--force', action='store_true', help="remove concretized spec (if any) immediately") - subparser.add_argument( - 'specs', nargs=argparse.REMAINDER, help="specs to be removed") + arguments.add_common_arguments(subparser, ['specs']) def remove(parser, args): env = ev.get_env(args, 'remove', required=True) - if args.all: - env.clear() - else: - for spec in spack.cmd.parse_specs(args.specs): - tty.msg('Removing %s from environment %s' % (spec, env.name)) - env.remove(spec, args.list_name, force=args.force) - env.write() + with env.write_transaction(): + if args.all: + env.clear() + else: + for spec in spack.cmd.parse_specs(args.specs): + tty.msg('Removing %s from environment %s' % (spec, env.name)) + env.remove(spec, args.list_name, force=args.force) + env.write() diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index ea5a34fc0d..83acf796a2 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -51,8 +51,8 @@ def setup_parser(subparser): remove_parser = sp.add_parser( 'remove', help=repo_remove.__doc__, aliases=['rm']) remove_parser.add_argument( - 'path_or_namespace', - help="path or namespace of a Spack package repository") + 'namespace_or_path', + help="namespace or path of a Spack package repository") remove_parser.add_argument( '--scope', choices=scopes, metavar=scopes_metavar, default=spack.config.default_modify_scope(), @@ -101,10 +101,10 @@ def repo_add(args): def repo_remove(args): """Remove a repository from Spack's configuration.""" repos = spack.config.get('repos', scope=args.scope) - path_or_namespace = args.path_or_namespace + namespace_or_path = args.namespace_or_path # If the argument is a path, remove that repository from config. - canon_path = canonicalize_path(path_or_namespace) + canon_path = canonicalize_path(namespace_or_path) for repo_path in repos: repo_canon_path = canonicalize_path(repo_path) if canon_path == repo_canon_path: @@ -117,7 +117,7 @@ def repo_remove(args): for path in repos: try: repo = Repo(path) - if repo.namespace == path_or_namespace: + if repo.namespace == namespace_or_path: repos.remove(path) spack.config.set('repos', repos, args.scope) tty.msg("Removed repository %s with namespace '%s'." @@ -127,7 +127,7 @@ def repo_remove(args): continue tty.die("No repository with path or namespace: %s" - % path_or_namespace) + % namespace_or_path) def repo_list(args): diff --git a/lib/spack/spack/cmd/resource.py b/lib/spack/spack/cmd/resource.py index 3e5080cd7a..0d26c39ab8 100644 --- a/lib/spack/spack/cmd/resource.py +++ b/lib/spack/spack/cmd/resource.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index 5d04890794..0f55884bfe 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -1,13 +1,12 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse - import llnl.util.tty as tty import spack.cmd +import spack.cmd.common.arguments as arguments import spack.repo description = "revert checked out package source code" @@ -16,15 +15,14 @@ level = "long" def setup_parser(subparser): - subparser.add_argument('packages', nargs=argparse.REMAINDER, - help="specs of packages to restage") + arguments.add_common_arguments(subparser, ['specs']) def restage(parser, args): - if not args.packages: + if not args.specs: tty.die("spack restage requires at least one package spec.") - specs = spack.cmd.parse_specs(args.packages, concretize=True) + specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) package.do_restage() diff --git a/lib/spack/spack/cmd/setup.py b/lib/spack/spack/cmd/setup.py index aa391b2b35..246e3b4275 100644 --- a/lib/spack/spack/cmd/setup.py +++ b/lib/spack/spack/cmd/setup.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -30,13 +30,10 @@ def setup_parser(subparser): subparser.add_argument( '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', help="do not try to install dependencies of requested packages") - arguments.add_common_arguments(subparser, ['no_checksum']) + arguments.add_common_arguments(subparser, ['no_checksum', 'spec']) subparser.add_argument( '-v', '--verbose', action='store_true', dest='verbose', help="display verbose build output while installing") - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help="specs to use for install. must contain package AND version") cd_group = subparser.add_mutually_exclusive_group() arguments.add_common_arguments(cd_group, ['clean', 'dirty']) diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index f10eef9c73..fd03f09e57 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -1,11 +1,10 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function -import argparse import contextlib import sys @@ -24,6 +23,10 @@ level = "short" def setup_parser(subparser): + subparser.epilog = """\ +for further documentation regarding the spec syntax, see: + spack help --spec +""" arguments.add_common_arguments( subparser, ['long', 'very_long', 'install_status']) subparser.add_argument( @@ -43,8 +46,7 @@ def setup_parser(subparser): subparser.add_argument( '-t', '--types', action='store_true', default=False, help='show dependency types') - subparser.add_argument( - 'specs', nargs=argparse.REMAINDER, help="specs of packages") + arguments.add_common_arguments(subparser, ['specs']) @contextlib.contextmanager diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index e4daf1778b..1acefb723c 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -1,10 +1,8 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse - import llnl.util.tty as tty import spack.environment as ev @@ -18,14 +16,11 @@ level = "long" def setup_parser(subparser): - arguments.add_common_arguments(subparser, ['no_checksum']) + arguments.add_common_arguments(subparser, ['no_checksum', 'specs']) subparser.add_argument( '-p', '--path', dest='path', help="path to stage package, does not add to spack tree") - subparser.add_argument( - 'specs', nargs=argparse.REMAINDER, help="specs of packages to stage") - def stage(parser, args): if not args.specs: diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 63e3778a6b..f2ca8fc93b 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -1,23 +1,25 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function +from __future__ import division +import collections import sys -import os import re import argparse import pytest from six import StringIO +import llnl.util.tty.color as color from llnl.util.filesystem import working_dir from llnl.util.tty.colify import colify import spack.paths -description = "run spack's unit tests" +description = "run spack's unit tests (wrapper around pytest)" section = "developer" level = "long" @@ -25,61 +27,130 @@ level = "long" def setup_parser(subparser): subparser.add_argument( '-H', '--pytest-help', action='store_true', default=False, - help="print full pytest help message, showing advanced options") - - list_group = subparser.add_mutually_exclusive_group() - list_group.add_argument( - '-l', '--list', action='store_true', default=False, - help="list basic test names") - list_group.add_argument( - '-L', '--long-list', action='store_true', default=False, - help="list the entire hierarchy of tests") + help="show full pytest help, with advanced options") + + # extra spack arguments to list tests + list_group = subparser.add_argument_group("listing tests") + list_mutex = list_group.add_mutually_exclusive_group() + list_mutex.add_argument( + '-l', '--list', action='store_const', default=None, + dest='list', const='list', help="list test filenames") + list_mutex.add_argument( + '-L', '--list-long', action='store_const', default=None, + dest='list', const='long', help="list all test functions") + list_mutex.add_argument( + '-N', '--list-names', action='store_const', default=None, + dest='list', const='names', help="list full names of all tests") + + # use tests for extension subparser.add_argument( '--extension', default=None, - help="run test for a given Spack extension" - ) + help="run test for a given spack extension") + + # spell out some common pytest arguments, so they'll show up in help + pytest_group = subparser.add_argument_group( + "common pytest arguments (spack test --pytest-help for more details)") + pytest_group.add_argument( + "-s", action='append_const', dest='parsed_args', const='-s', + help="print output while tests run (disable capture)") + pytest_group.add_argument( + "-k", action='store', metavar="EXPRESSION", dest='expression', + help="filter tests by keyword (can also use w/list options)") + pytest_group.add_argument( + "--showlocals", action='append_const', dest='parsed_args', + const='--showlocals', help="show local variable values in tracebacks") + + # remainder is just passed to pytest subparser.add_argument( - 'tests', nargs=argparse.REMAINDER, - help="list of tests to run (will be passed to pytest -k)") + 'pytest_args', nargs=argparse.REMAINDER, help="arguments for pytest") -def do_list(args, unknown_args): +def do_list(args, extra_args): """Print a lists of tests than what pytest offers.""" # Run test collection and get the tree out. old_output = sys.stdout try: sys.stdout = output = StringIO() - pytest.main(['--collect-only']) + pytest.main(['--collect-only'] + extra_args) finally: sys.stdout = old_output - # put the output in a more readable tree format. lines = output.getvalue().split('\n') - output_lines = [] + tests = collections.defaultdict(lambda: set()) + prefix = [] + + # collect tests into sections for line in lines: match = re.match(r"(\s*)<([^ ]*) '([^']*)'", line) if not match: continue indent, nodetype, name = match.groups() - # only print top-level for short list - if args.list: - if not indent: - output_lines.append( - os.path.basename(name).replace('.py', '')) - else: - print(indent + name) + # strip parametrized tests + if "[" in name: + name = name[:name.index("[")] + + depth = len(indent) // 2 - if args.list: - colify(output_lines) + if nodetype.endswith("Function"): + key = tuple(prefix) + tests[key].add(name) + else: + prefix = prefix[:depth] + prefix.append(name) + + def colorize(c, prefix): + if isinstance(prefix, tuple): + return "::".join( + color.colorize("@%s{%s}" % (c, p)) + for p in prefix if p != "()" + ) + return color.colorize("@%s{%s}" % (c, prefix)) + + if args.list == "list": + files = set(prefix[0] for prefix in tests) + color_files = [colorize("B", file) for file in sorted(files)] + colify(color_files) + + elif args.list == "long": + for prefix, functions in sorted(tests.items()): + path = colorize("*B", prefix) + "::" + functions = [colorize("c", f) for f in sorted(functions)] + color.cprint(path) + colify(functions, indent=4) + print() + + else: # args.list == "names" + all_functions = [ + colorize("*B", prefix) + "::" + colorize("c", f) + for prefix, functions in sorted(tests.items()) + for f in sorted(functions) + ] + colify(all_functions) + + +def add_back_pytest_args(args, unknown_args): + """Add parsed pytest args, unknown args, and remainder together. + + We add some basic pytest arguments to the Spack parser to ensure that + they show up in the short help, so we have to reassemble things here. + """ + result = args.parsed_args or [] + result += unknown_args or [] + result += args.pytest_args or [] + if args.expression: + result += ["-k", args.expression] + return result def test(parser, args, unknown_args): if args.pytest_help: # make the pytest.main help output more accurate sys.argv[0] = 'spack test' - pytest.main(['-h']) - return + return pytest.main(['-h']) + + # add back any parsed pytest args we need to pass to pytest + pytest_args = add_back_pytest_args(args, unknown_args) # The default is to test the core of Spack. If the option `--extension` # has been used, then test that extension. @@ -91,15 +162,8 @@ def test(parser, args, unknown_args): # pytest.ini lives in the root of the spack repository. with working_dir(pytest_root): - # --list and --long-list print the test output better. - if args.list or args.long_list: - do_list(args, unknown_args) + if args.list: + do_list(args, pytest_args) return - # Allow keyword search without -k if no options are specified - if (args.tests and not unknown_args and - not any(arg.startswith('-') for arg in args.tests)): - return pytest.main(['-k'] + args.tests) - - # Just run the pytest command - return pytest.main(unknown_args + args.tests) + return pytest.main(pytest_args) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 2d903fdd47..2757d5d232 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -1,11 +1,12 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function -import argparse +import sys +import itertools import spack.cmd import spack.environment as ev @@ -31,23 +32,19 @@ error_message = """You can either: # Arguments for display_specs when we find ambiguity display_args = { 'long': True, - 'show_flags': True, - 'variants': True, + 'show_flags': False, + 'variants': False, 'indent': 4, } -def add_common_arguments(subparser): +def setup_parser(subparser): subparser.add_argument( '-f', '--force', action='store_true', dest='force', help="remove regardless of whether other packages or environments " "depend on this one") arguments.add_common_arguments( - subparser, ['recurse_dependents', 'yes_to_all']) - - -def setup_parser(subparser): - add_common_arguments(subparser) + subparser, ['recurse_dependents', 'yes_to_all', 'installed_specs']) subparser.add_argument( '-a', '--all', action='store_true', dest='all', help="USE CAREFULLY. Remove ALL installed packages that match each " @@ -57,11 +54,6 @@ def setup_parser(subparser): "If used in an environment, all packages in the environment " "will be uninstalled.") - subparser.add_argument( - 'packages', - nargs=argparse.REMAINDER, - help="specs of packages to uninstall") - def find_matching_specs(env, specs, allow_multiple_matches=False, force=False): """Returns a list of specs matching the not necessarily @@ -214,9 +206,6 @@ def do_uninstall(env, specs, force): # want to uninstall. spack.package.Package.uninstall_by_spec(item, force=True) - if env: - _remove_from_env(item, env) - # A package is ready to be uninstalled when nothing else references it, # unless we are requested to force uninstall it. is_ready = lambda x: not spack.store.db.query_by_spec_hash(x)[1].ref_count @@ -235,10 +224,6 @@ def do_uninstall(env, specs, force): for item in ready: item.do_uninstall(force=force) - # write any changes made to the active environment - if env: - env.write() - def get_uninstall_list(args, specs, env): # Gets the list of installed specs that match the ones give via cli @@ -324,25 +309,40 @@ def uninstall_specs(args, specs): return if not args.yes_to_all: - tty.msg('The following packages will be uninstalled:\n') - spack.cmd.display_specs(anything_to_do, **display_args) - answer = tty.get_yes_or_no('Do you want to proceed?', default=False) - if not answer: - tty.die('Will not uninstall any packages.') + confirm_removal(anything_to_do) - # just force-remove things in the remove list - for spec in remove_list: - _remove_from_env(spec, env) + if env: + # Remove all the specs that are supposed to be uninstalled or just + # removed. + with env.write_transaction(): + for spec in itertools.chain(remove_list, uninstall_list): + _remove_from_env(spec, env) + env.write() # Uninstall everything on the list do_uninstall(env, uninstall_list, args.force) +def confirm_removal(specs): + """Display the list of specs to be removed and ask for confirmation. + + Args: + specs (list): specs to be removed + """ + tty.msg('The following packages will be uninstalled:\n') + spack.cmd.display_specs(specs, **display_args) + print('') + answer = tty.get_yes_or_no('Do you want to proceed?', default=False) + if not answer: + tty.msg('Aborting uninstallation') + sys.exit(0) + + def uninstall(parser, args): - if not args.packages and not args.all: + if not args.specs and not args.all: tty.die('uninstall requires at least one package argument.', ' Use `spack uninstall --all` to uninstall ALL packages.') # [any] here handles the --all case by forcing all specs to be returned - specs = spack.cmd.parse_specs(args.packages) if args.packages else [any] + specs = spack.cmd.parse_specs(args.specs) if args.specs else [any] uninstall_specs(args, specs) diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index b12fbfa454..d19a33102f 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -1,23 +1,73 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse -from spack.cmd.common import print_module_placeholder_help +import sys +import os -description = "remove package from environment using `module unload`" -section = "modules" +import llnl.util.tty as tty + +import spack.cmd +import spack.cmd.common.arguments as arguments +import spack.util.environment +import spack.user_environment as uenv +import spack.error + +description = "remove package from the user environment" +section = "user environment" level = "short" def setup_parser(subparser): """Parser is only constructed so that this prints a nice help message with -h. """ - subparser.add_argument( - 'spec', nargs=argparse.REMAINDER, - help='spec of package to unload with modules') + arguments.add_common_arguments(subparser, ['installed_specs']) + + shells = subparser.add_mutually_exclusive_group() + shells.add_argument( + '--sh', action='store_const', dest='shell', const='sh', + help="print sh commands to activate the environment") + shells.add_argument( + '--csh', action='store_const', dest='shell', const='csh', + help="print csh commands to activate the environment") + + subparser.add_argument('-a', '--all', action='store_true', + help='unload all loaded Spack packages.') def unload(parser, args): - print_module_placeholder_help() + """Unload spack packages from the user environment.""" + if args.specs and args.all: + raise spack.error.SpackError("Cannot specify specs on command line" + " when unloading all specs with '--all'") + + hashes = os.environ.get(uenv.spack_loaded_hashes_var, '').split(':') + if args.specs: + specs = [spack.cmd.disambiguate_spec_from_hashes(spec, hashes) + for spec in spack.cmd.parse_specs(args.specs)] + else: + specs = spack.store.db.query(hashes=hashes) + + if not args.shell: + msg = [ + "This command works best with Spack's shell support", + "" + ] + spack.cmd.common.shell_init_instructions + [ + 'Or, if you want to use `spack unload` without initializing', + 'shell support, you can run one of these:', + '', + ' eval `spack unload --sh %s` # for bash/sh' % args.specs, + ' eval `spack unload --csh %s` # for csh/tcsh' % args.specs, + ] + tty.msg(*msg) + return 1 + + env_mod = spack.util.environment.EnvironmentModifications() + for spec in specs: + env_mod.extend( + uenv.environment_modifications_for_spec(spec).reversed()) + env_mod.remove_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) + cmds = env_mod.shell_modifications(args.shell) + + sys.stdout.write(cmds) diff --git a/lib/spack/spack/cmd/upload_s3.py b/lib/spack/spack/cmd/upload_s3.py index f3be69c10d..56d03758cf 100644 --- a/lib/spack/spack/cmd/upload_s3.py +++ b/lib/spack/spack/cmd/upload_s3.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/cmd/url.py b/lib/spack/spack/cmd/url.py index a24fcc575c..db8f358887 100644 --- a/lib/spack/spack/cmd/url.py +++ b/lib/spack/spack/cmd/url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -135,7 +135,7 @@ def url_list(args): # Gather set of URLs from all packages for pkg in spack.repo.path.all_packages(): - url = getattr(pkg.__class__, 'url', None) + url = getattr(pkg, 'url', None) urls = url_list_parsing(args, urls, url, pkg) for params in pkg.versions.values(): @@ -174,7 +174,7 @@ def url_summary(args): for pkg in spack.repo.path.all_packages(): urls = set() - url = getattr(pkg.__class__, 'url', None) + url = getattr(pkg, 'url', None) if url: urls.add(url) @@ -434,23 +434,10 @@ def name_parsed_correctly(pkg, name): Returns: bool: True if the name was correctly parsed, else False """ - pkg_name = pkg.name + pkg_name = remove_prefix(pkg.name) name = simplify_name(name) - # After determining a name, `spack create` determines a build system. - # Some build systems prepend a special string to the front of the name. - # Since this can't be guessed from the URL, it would be unfair to say - # that these names are incorrectly parsed, so we remove them. - if pkg_name.startswith('r-'): - pkg_name = pkg_name[2:] - elif pkg_name.startswith('py-'): - pkg_name = pkg_name[3:] - elif pkg_name.startswith('perl-'): - pkg_name = pkg_name[5:] - elif pkg_name.startswith('octave-'): - pkg_name = pkg_name[7:] - return name == pkg_name @@ -475,8 +462,32 @@ def version_parsed_correctly(pkg, version): return False +def remove_prefix(pkg_name): + """Remove build system prefix ('py-', 'perl-', etc.) from a package name. + + After determining a name, `spack create` determines a build system. + Some build systems prepend a special string to the front of the name. + Since this can't be guessed from the URL, it would be unfair to say + that these names are incorrectly parsed, so we remove them. + + Args: + pkg_name (str): the name of the package + + Returns: + str: the name of the package with any build system prefix removed + """ + prefixes = [ + 'r-', 'py-', 'tcl-', 'lua-', 'perl-', 'ruby-', 'llvm-', + 'intel-', 'votca-', 'octave-', 'gtkorvo-' + ] + + prefix = next((p for p in prefixes if pkg_name.startswith(p)), '') + + return pkg_name[len(prefix):] + + def remove_separators(version): - """Removes separator characters ('.', '_', and '-') from a version. + """Remove separator characters ('.', '_', and '-') from a version. A version like 1.2.3 may be displayed as 1_2_3 in the URL. Make sure 1.2.3, 1-2-3, 1_2_3, and 123 are considered equal. diff --git a/lib/spack/spack/cmd/verify.py b/lib/spack/spack/cmd/verify.py index 4bd7b39fea..b20d795ce5 100644 --- a/lib/spack/spack/cmd/verify.py +++ b/lib/spack/spack/cmd/verify.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -25,8 +25,8 @@ def setup_parser(subparser): help="Ouptut json-formatted errors") subparser.add_argument('-a', '--all', action='store_true', help="Verify all packages") - subparser.add_argument('files_or_specs', nargs=argparse.REMAINDER, - help="Files or specs to verify") + subparser.add_argument('specs_or_files', nargs=argparse.REMAINDER, + help="Specs or files to verify") type = subparser.add_mutually_exclusive_group() type.add_argument( @@ -47,7 +47,7 @@ def verify(parser, args): setup_parser.parser.print_help() return 1 - for file in args.files_or_specs: + for file in args.specs_or_files: results = spack.verify.check_file_manifest(file) if results.has_errors(): if args.json: @@ -57,21 +57,21 @@ def verify(parser, args): return 0 else: - spec_args = spack.cmd.parse_specs(args.files_or_specs) + spec_args = spack.cmd.parse_specs(args.specs_or_files) if args.all: query = spack.store.db.query_local if local else spack.store.db.query # construct spec list if spec_args: - spec_list = spack.cmd.parse_specs(args.files_or_specs) + spec_list = spack.cmd.parse_specs(args.specs_or_files) specs = [] for spec in spec_list: specs += query(spec, installed=True) else: specs = query(installed=True) - elif args.files_or_specs: + elif args.specs_or_files: # construct disambiguated spec list env = ev.get_env(args, 'verify') specs = list(map(lambda x: spack.cmd.disambiguate_spec(x, env, diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index b264c1aaf0..723f89ce08 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -1,15 +1,17 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from __future__ import print_function +import sys + from llnl.util.tty.colify import colify import llnl.util.tty as tty +import spack.cmd.common.arguments as arguments import spack.repo -import sys description = "list available versions of a package" section = "packaging" @@ -17,10 +19,9 @@ level = "long" def setup_parser(subparser): - subparser.add_argument('package', metavar='PACKAGE', - help='package to list versions for') subparser.add_argument('-s', '--safe-only', action='store_true', help='only list safe versions of the package') + arguments.add_common_arguments(subparser, ['package']) def versions(parser, args): diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 5480c28c6d..18c836736e 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index f2b62dc3f9..36a2704eac 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -32,7 +32,7 @@ def _verify_executables(*paths): @llnl.util.lang.memoized -def get_compiler_version_output(compiler_path, version_arg): +def get_compiler_version_output(compiler_path, version_arg, ignore_errors=()): """Invokes the compiler at a given path passing a single version argument and returns the output. @@ -41,7 +41,8 @@ def get_compiler_version_output(compiler_path, version_arg): version_arg (str): the argument used to extract version information """ compiler = spack.util.executable.Executable(compiler_path) - output = compiler(version_arg, output=str, error=str) + output = compiler( + version_arg, output=str, error=str, ignore_errors=ignore_errors) return output @@ -199,6 +200,9 @@ class Compiler(object): #: Compiler argument that produces version information version_argument = '-dumpversion' + #: Return values to ignore when invoking the compiler to get its version + ignore_version_errors = () + #: Regex used to extract version from compiler's output version_regex = '(.*)' @@ -412,7 +416,8 @@ class Compiler(object): @classmethod def default_version(cls, cc): """Override just this to override all compiler version functions.""" - output = get_compiler_version_output(cc, cls.version_argument) + output = get_compiler_version_output( + cc, cls.version_argument, tuple(cls.ignore_version_errors)) return cls.extract_version_from_output(output) @classmethod diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index bc2e4df391..a092c930d7 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/arm.py b/lib/spack/spack/compilers/arm.py index ff4c208531..ffce1e2b01 100644 --- a/lib/spack/spack/compilers/arm.py +++ b/lib/spack/spack/compilers/arm.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/cce.py b/lib/spack/spack/compilers/cce.py index 50b9694188..23a1771e9c 100644 --- a/lib/spack/spack/compilers/cce.py +++ b/lib/spack/spack/compilers/cce.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index ea4ddaf401..c8df63e533 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -208,7 +208,9 @@ class Clang(Compiler): r'^Apple (?:LLVM|clang) version ([^ )]+)|' # Normal clang compiler versions are left as-is r'clang version ([^ )]+)-svn[~.\w\d-]*|' - r'clang version ([^ )]+)-[~.\w\d-]*|' + # Don't include hyphenated patch numbers in the version + # (see https://github.com/spack/spack/pull/14365 for details) + r'clang version ([^ )]+?)-[~.\w\d-]*|' r'clang version ([^ )]+)', output ) diff --git a/lib/spack/spack/compilers/fj.py b/lib/spack/spack/compilers/fj.py index 1868972e96..083fe35b0d 100644 --- a/lib/spack/spack/compilers/fj.py +++ b/lib/spack/spack/compilers/fj.py @@ -1,4 +1,4 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -61,3 +61,7 @@ class Fj(spack.compiler.Compiler): @property def pic_flag(self): return "-KPIC" + + def setup_custom_environment(self, pkg, env): + env.append_flags('fcc_ENV', '-Nclang') + env.append_flags('FCC_ENV', '-Nclang') diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 3ac78ccccf..da559d554c 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index f06f5b07cb..bc0e44b274 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index bd0040c9a9..b7ef2297b8 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 90560f7c63..7da991754f 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -30,6 +30,7 @@ class Pgi(Compiler): PrgEnv_compiler = 'pgi' version_argument = '-V' + ignore_version_errors = [2] # `pgcc -V` on PowerPC annoyingly returns 2 version_regex = r'pg[^ ]* ([0-9.]+)-[0-9]+ (LLVM )?[^ ]+ target on ' @classmethod diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index fcfaa29d12..36aa4dae7d 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/compilers/xl_r.py b/lib/spack/spack/compilers/xl_r.py index 18c4e32564..c9a785d5d8 100644 --- a/lib/spack/spack/compilers/xl_r.py +++ b/lib/spack/spack/compilers/xl_r.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 417afed35b..8814d03d06 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -74,7 +74,7 @@ class Concretizer(object): if spec.virtual: candidates = spack.repo.path.providers_for(spec) if not candidates: - raise spack.spec.UnsatisfiableProviderSpecError( + raise spack.error.UnsatisfiableProviderSpecError( candidates[0], spec) # Find nearest spec in the DAG (up then down) that has prefs. diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3f536f3793..b1c0ad73c7 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/container/__init__.py b/lib/spack/spack/container/__init__.py new file mode 100644 index 0000000000..fc3750355a --- /dev/null +++ b/lib/spack/spack/container/__init__.py @@ -0,0 +1,81 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Package that provides functions and classes to +generate container recipes from a Spack environment +""" +import warnings + +import spack.environment +import spack.schema.env as env +import spack.util.spack_yaml as syaml +from .writers import recipe + +__all__ = ['validate', 'recipe'] + + +def validate(configuration_file): + """Validate a Spack environment YAML file that is being used to generate a + recipe for a container. + + Since a few attributes of the configuration must have specific values for + the container recipe, this function returns a sanitized copy of the + configuration in the input file. If any modification is needed, a warning + will be issued. + + Args: + configuration_file (str): path to the Spack environment YAML file + + Returns: + A sanitized copy of the configuration stored in the input file + """ + import jsonschema + with open(configuration_file) as f: + config = syaml.load(f) + + # Ensure we have a "container" attribute with sensible defaults set + env_dict = spack.environment.config_dict(config) + env_dict.setdefault('container', { + 'format': 'docker', + 'base': {'image': 'ubuntu:18.04', 'spack': 'develop'} + }) + env_dict['container'].setdefault('format', 'docker') + env_dict['container'].setdefault( + 'base', {'image': 'ubuntu:18.04', 'spack': 'develop'} + ) + + # Remove attributes that are not needed / allowed in the + # container recipe + for subsection in ('cdash', 'gitlab_ci', 'modules'): + if subsection in env_dict: + msg = ('the subsection "{0}" in "{1}" is not used when generating' + ' container recipes and will be discarded') + warnings.warn(msg.format(subsection, configuration_file)) + env_dict.pop(subsection) + + # Set the default value of the concretization strategy to "together" and + # warn if the user explicitly set another value + env_dict.setdefault('concretization', 'together') + if env_dict['concretization'] != 'together': + msg = ('the "concretization" attribute of the environment is set ' + 'to "{0}" [the advised value is instead "together"]') + warnings.warn(msg.format(env_dict['concretization'])) + + # Check if the install tree was explicitly set to a custom value and warn + # that it will be overridden + environment_config = env_dict.get('config', {}) + if environment_config.get('install_tree', None): + msg = ('the "config:install_tree" attribute has been set explicitly ' + 'and will be overridden in the container image') + warnings.warn(msg) + + # Likewise for the view + environment_view = env_dict.get('view', None) + if environment_view: + msg = ('the "view" attribute has been set explicitly ' + 'and will be overridden in the container image') + warnings.warn(msg) + + jsonschema.validate(config, schema=env.schema) + return config diff --git a/lib/spack/spack/container/images.json b/lib/spack/spack/container/images.json new file mode 100644 index 0000000000..ecd911815d --- /dev/null +++ b/lib/spack/spack/container/images.json @@ -0,0 +1,50 @@ +{ + "ubuntu:18.04": { + "update": "apt-get -yqq update && apt-get -yqq upgrade", + "install": "apt-get -yqq install", + "clean": "rm -rf /var/lib/apt/lists/*", + "environment": [], + "build": "spack/ubuntu-bionic", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + }, + "ubuntu:16.04": { + "update": "apt-get -yqq update && apt-get -yqq upgrade", + "install": "apt-get -yqq install", + "clean": "rm -rf /var/lib/apt/lists/*", + "environment": [], + "build": "spack/ubuntu-xenial", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + }, + "centos:7": { + "update": "yum update -y && yum install -y epel-release && yum update -y", + "install": "yum install -y", + "clean": "rm -rf /var/cache/yum && yum clean all", + "environment": [], + "build": "spack/centos7", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + }, + "centos:6": { + "update": "yum update -y && yum install -y epel-release && yum update -y", + "install": "yum install -y", + "clean": "rm -rf /var/cache/yum && yum clean all", + "environment": [], + "build": "spack/centos6", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + } +}
\ No newline at end of file diff --git a/lib/spack/spack/container/images.py b/lib/spack/spack/container/images.py new file mode 100644 index 0000000000..421fc24425 --- /dev/null +++ b/lib/spack/spack/container/images.py @@ -0,0 +1,72 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Manages the details on the images used in the build and the run stage.""" +import json +import os.path + +#: Global variable used to cache in memory the content of images.json +_data = None + + +def data(): + """Returns a dictionary with the static data on the images. + + The dictionary is read from a JSON file lazily the first time + this function is called. + """ + global _data + if not _data: + json_dir = os.path.abspath(os.path.dirname(__file__)) + json_file = os.path.join(json_dir, 'images.json') + with open(json_file) as f: + _data = json.load(f) + return _data + + +def build_info(image, spack_version): + """Returns the name of the build image and its tag. + + Args: + image (str): image to be used at run-time. Should be of the form + <image_name>:<image_tag> e.g. "ubuntu:18.04" + spack_version (str): version of Spack that we want to use to build + + Returns: + A tuple with (image_name, image_tag) for the build image + """ + # Don't handle error here, as a wrong image should have been + # caught by the JSON schema + image_data = data()[image] + build_image = image_data['build'] + + # Try to check if we have a tag for this Spack version + try: + build_tag = image_data['build_tags'][spack_version] + except KeyError: + msg = ('the image "{0}" has no tag for Spack version "{1}" ' + '[valid versions are {2}]') + msg = msg.format(build_image, spack_version, + ', '.join(image_data['build_tags'].keys())) + raise ValueError(msg) + + return build_image, build_tag + + +def package_info(image): + """Returns the commands used to update system repositories, install + system packages and clean afterwards. + + Args: + image (str): image to be used at run-time. Should be of the form + <image_name>:<image_tag> e.g. "ubuntu:18.04" + + Returns: + A tuple of (update, install, clean) commands. + """ + image_data = data()[image] + update = image_data['update'] + install = image_data['install'] + clean = image_data['clean'] + return update, install, clean diff --git a/lib/spack/spack/container/writers/__init__.py b/lib/spack/spack/container/writers/__init__.py new file mode 100644 index 0000000000..a1d2fa3102 --- /dev/null +++ b/lib/spack/spack/container/writers/__init__.py @@ -0,0 +1,154 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Writers for different kind of recipes and related +convenience functions. +""" +import collections +import copy + +import spack.environment +import spack.schema.env +import spack.tengine as tengine +import spack.util.spack_yaml as syaml + +from spack.container.images import build_info, package_info + +#: Caches all the writers that are currently supported +_writer_factory = {} + + +def writer(name): + """Decorator to register a factory for a recipe writer. + + Each factory should take a configuration dictionary and return a + properly configured writer that, when called, prints the + corresponding recipe. + """ + def _decorator(factory): + _writer_factory[name] = factory + return factory + return _decorator + + +def create(configuration): + """Returns a writer that conforms to the configuration passed as input. + + Args: + configuration: how to generate the current recipe + """ + name = spack.environment.config_dict(configuration)['container']['format'] + return _writer_factory[name](configuration) + + +def recipe(configuration): + """Returns a recipe that conforms to the configuration passed as input. + + Args: + configuration: how to generate the current recipe + """ + return create(configuration)() + + +class PathContext(tengine.Context): + """Generic context used to instantiate templates of recipes that + install software in a common location and make it available + directly via PATH. + """ + def __init__(self, config): + self.config = spack.environment.config_dict(config) + self.container_config = self.config['container'] + + @tengine.context_property + def run(self): + """Information related to the run image.""" + image = self.container_config['base']['image'] + Run = collections.namedtuple('Run', ['image']) + return Run(image=image) + + @tengine.context_property + def build(self): + """Information related to the build image.""" + + # Map the final image to the correct build image + run_image = self.container_config['base']['image'] + spack_version = self.container_config['base']['spack'] + image, tag = build_info(run_image, spack_version) + + Build = collections.namedtuple('Build', ['image', 'tag']) + return Build(image=image, tag=tag) + + @tengine.context_property + def strip(self): + """Whether or not to strip binaries in the image""" + return self.container_config.get('strip', True) + + @tengine.context_property + def paths(self): + """Important paths in the image""" + Paths = collections.namedtuple('Paths', [ + 'environment', 'store', 'view' + ]) + return Paths( + environment='/opt/spack-environment', + store='/opt/software', + view='/opt/view' + ) + + @tengine.context_property + def manifest(self): + """The spack.yaml file that should be used in the image""" + import jsonschema + # Copy in the part of spack.yaml prescribed in the configuration file + manifest = copy.deepcopy(self.config) + manifest.pop('container') + + # Ensure that a few paths are where they need to be + manifest.setdefault('config', syaml.syaml_dict()) + manifest['config']['install_tree'] = self.paths.store + manifest['view'] = self.paths.view + manifest = {'spack': manifest} + + # Validate the manifest file + jsonschema.validate(manifest, schema=spack.schema.env.schema) + + return syaml.dump(manifest, default_flow_style=False).strip() + + @tengine.context_property + def os_packages(self): + """Additional system packages that are needed at run-time.""" + package_list = self.container_config.get('os_packages', None) + if not package_list: + return package_list + + image = self.container_config['base']['image'] + update, install, clean = package_info(image) + Packages = collections.namedtuple( + 'Packages', ['update', 'install', 'list', 'clean'] + ) + return Packages(update=update, install=install, + list=package_list, clean=clean) + + @tengine.context_property + def extra_instructions(self): + Extras = collections.namedtuple('Extra', ['build', 'final']) + extras = self.container_config.get('extra_instructions', {}) + build, final = extras.get('build', None), extras.get('final', None) + return Extras(build=build, final=final) + + @tengine.context_property + def labels(self): + return self.container_config.get('labels', {}) + + def __call__(self): + """Returns the recipe as a string""" + env = tengine.make_environment() + t = env.get_template(self.template_name) + return t.render(**self.to_dict()) + + +# Import after function definition all the modules in this package, +# so that registration of writers will happen automatically +import spack.container.writers.singularity # noqa +import spack.container.writers.docker # noqa diff --git a/lib/spack/spack/container/writers/docker.py b/lib/spack/spack/container/writers/docker.py new file mode 100644 index 0000000000..557d22c803 --- /dev/null +++ b/lib/spack/spack/container/writers/docker.py @@ -0,0 +1,30 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import spack.tengine as tengine + +from . import writer, PathContext + + +@writer('docker') +class DockerContext(PathContext): + """Context used to instantiate a Dockerfile""" + #: Name of the template used for Dockerfiles + template_name = 'container/Dockerfile' + + @tengine.context_property + def manifest(self): + manifest_str = super(DockerContext, self).manifest + # Docker doesn't support HEREDOC so we need to resort to + # a horrible echo trick to have the manifest in the Dockerfile + echoed_lines = [] + for idx, line in enumerate(manifest_str.split('\n')): + if idx == 0: + echoed_lines.append('&& (echo "' + line + '" \\') + continue + echoed_lines.append('&& echo "' + line + '" \\') + + echoed_lines[-1] = echoed_lines[-1].replace(' \\', ')') + + return '\n'.join(echoed_lines) diff --git a/lib/spack/spack/container/writers/singularity.py b/lib/spack/spack/container/writers/singularity.py new file mode 100644 index 0000000000..32f29eb83d --- /dev/null +++ b/lib/spack/spack/container/writers/singularity.py @@ -0,0 +1,33 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import spack.tengine as tengine +from . import writer, PathContext + + +@writer('singularity') +class SingularityContext(PathContext): + """Context used to instantiate a Singularity definition file""" + #: Name of the template used for Singularity definition files + template_name = 'container/singularity.def' + + @property + def singularity_config(self): + return self.container_config.get('singularity', {}) + + @tengine.context_property + def runscript(self): + return self.singularity_config.get('runscript', '') + + @tengine.context_property + def startscript(self): + return self.singularity_config.get('startscript', '') + + @tengine.context_property + def test(self): + return self.singularity_config.get('test', '') + + @tengine.context_property + def help(self): + return self.singularity_config.get('help', '') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 243f1a20d5..c06d1ae546 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -1118,7 +1118,27 @@ class Database(object): continue # TODO: conditional way to do this instead of catching exceptions - def get_by_hash_local(self, dag_hash, default=None, installed=any): + def _get_by_hash_local(self, dag_hash, default=None, installed=any): + # hash is a full hash and is in the data somewhere + if dag_hash in self._data: + rec = self._data[dag_hash] + if rec.install_type_matches(installed): + return [rec.spec] + else: + return default + + # check if hash is a prefix of some installed (or previously + # installed) spec. + matches = [record.spec for h, record in self._data.items() + if h.startswith(dag_hash) and + record.install_type_matches(installed)] + if matches: + return matches + + # nothing found + return default + + def get_by_hash_local(self, *args, **kwargs): """Look up a spec in *this DB* by DAG hash, or by a DAG hash prefix. Arguments: @@ -1142,24 +1162,7 @@ class Database(object): """ with self.read_transaction(): - # hash is a full hash and is in the data somewhere - if dag_hash in self._data: - rec = self._data[dag_hash] - if rec.install_type_matches(installed): - return [rec.spec] - else: - return default - - # check if hash is a prefix of some installed (or previously - # installed) spec. - matches = [record.spec for h, record in self._data.items() - if h.startswith(dag_hash) and - record.install_type_matches(installed)] - if matches: - return matches - - # nothing found - return default + return self._get_by_hash_local(*args, **kwargs) def get_by_hash(self, dag_hash, default=None, installed=any): """Look up a spec by DAG hash, or by a DAG hash prefix. @@ -1184,9 +1187,14 @@ class Database(object): (list): a list of specs matching the hash or hash prefix """ - search_path = [self] + self.upstream_dbs - for db in search_path: - spec = db.get_by_hash_local( + + spec = self.get_by_hash_local( + dag_hash, default=default, installed=installed) + if spec is not None: + return spec + + for upstream_db in self.upstream_dbs: + spec = upstream_db._get_by_hash_local( dag_hash, default=default, installed=installed) if spec is not None: return spec @@ -1295,6 +1303,30 @@ class Database(object): upstream, record = self.query_by_spec_hash(key) return record and not record.installed + @property + def unused_specs(self): + """Return all the specs that are currently installed but not needed + at runtime to satisfy user's requests. + + Specs in the return list are those which are not either: + 1. Installed on an explicit user request + 2. Installed as a "run" or "link" dependency (even transitive) of + a spec at point 1. + """ + needed, visited = set(), set() + with self.read_transaction(): + for key, rec in self._data.items(): + if rec.explicit: + # recycle `visited` across calls to avoid + # redundantly traversing + for spec in rec.spec.traverse(visited=visited): + needed.add(spec.dag_hash()) + + unused = [rec.spec for key, rec in self._data.items() + if key not in needed and rec.installed] + + return unused + class UpstreamDatabaseLockingError(SpackError): """Raised when an operation would need to lock an upstream database""" diff --git a/lib/spack/spack/dependency.py b/lib/spack/spack/dependency.py index 3e427196b5..e6b6c9cedc 100644 --- a/lib/spack/spack/dependency.py +++ b/lib/spack/spack/dependency.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index d877d4a193..f6496bb62a 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index f8a42bf42b..a2e871ee1a 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index bf9af075ce..fff1485e3c 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -29,15 +29,14 @@ import spack.store import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.config -import spack.build_environment as build_env - -from spack.util.prefix import Prefix +import spack.user_environment as uenv from spack.filesystem_view import YamlFilesystemView import spack.util.environment import spack.architecture as architecture from spack.spec import Spec from spack.spec_list import SpecList, InvalidSpecConstraintError from spack.variant import UnknownVariantError +import spack.util.lock as lk #: environment variable used to indicate the active environment spack_env_var = 'SPACK_ENV' @@ -559,12 +558,18 @@ class Environment(object): path to the view. """ self.path = os.path.abspath(path) + + self.txlock = lk.Lock(self._transaction_lock_path) + # This attribute will be set properly from configuration # during concretization self.concretization = None self.clear() if init_file: + # If we are creating the environment from an init file, we don't + # need to lock, because there are no Spack operations that alter + # the init file. with fs.open_if_filename(init_file) as f: if hasattr(f, 'name') and f.name.endswith('.lock'): self._read_manifest(default_manifest_yaml) @@ -573,26 +578,8 @@ class Environment(object): else: self._read_manifest(f, raw_yaml=default_manifest_yaml) else: - default_manifest = not os.path.exists(self.manifest_path) - if default_manifest: - # No manifest, use default yaml - self._read_manifest(default_manifest_yaml) - else: - with open(self.manifest_path) as f: - self._read_manifest(f) - - if os.path.exists(self.lock_path): - with open(self.lock_path) as f: - read_lock_version = self._read_lockfile(f) - if default_manifest: - # No manifest, set user specs from lockfile - self._set_user_specs_from_lockfile() - - if read_lock_version == 1: - tty.debug( - "Storing backup of old lockfile {0} at {1}".format( - self.lock_path, self._lock_backup_v1_path)) - shutil.copy(self.lock_path, self._lock_backup_v1_path) + with lk.ReadTransaction(self.txlock): + self._read() if with_view is False: self.views = {} @@ -604,6 +591,42 @@ class Environment(object): # If with_view is None, then defer to the view settings determined by # the manifest file + def _re_read(self): + """Reinitialize the environment object if it has been written (this + may not be true if the environment was just created in this running + instance of Spack).""" + if not os.path.exists(self.manifest_path): + return + + self.clear() + self._read() + + def _read(self): + default_manifest = not os.path.exists(self.manifest_path) + if default_manifest: + # No manifest, use default yaml + self._read_manifest(default_manifest_yaml) + else: + with open(self.manifest_path) as f: + self._read_manifest(f) + + if os.path.exists(self.lock_path): + with open(self.lock_path) as f: + read_lock_version = self._read_lockfile(f) + if default_manifest: + # No manifest, set user specs from lockfile + self._set_user_specs_from_lockfile() + + if read_lock_version == 1: + tty.debug( + "Storing backup of old lockfile {0} at {1}".format( + self.lock_path, self._lock_backup_v1_path)) + shutil.copy(self.lock_path, self._lock_backup_v1_path) + + def write_transaction(self): + """Get a write lock context manager for use in a `with` block.""" + return lk.WriteTransaction(self.txlock, acquire=self._re_read) + def _read_manifest(self, f, raw_yaml=None): """Read manifest file and set up user specs.""" if raw_yaml: @@ -697,6 +720,13 @@ class Environment(object): return os.path.join(self.path, manifest_name) @property + def _transaction_lock_path(self): + """The location of the lock file used to synchronize multiple + processes updating the same environment. + """ + return os.path.join(self.env_subdir_path, 'transaction_lock') + + @property def lock_path(self): """Path to spack.lock file in this environment.""" return os.path.join(self.path, lockfile_name) @@ -825,7 +855,10 @@ class Environment(object): raise SpackEnvironmentError( 'cannot add anonymous specs to an environment!') elif not spack.repo.path.exists(spec.name): - raise SpackEnvironmentError('no such package: %s' % spec.name) + virtuals = spack.repo.path.provider_index.providers.keys() + if spec.name not in virtuals: + msg = 'no such package: %s' % spec.name + raise SpackEnvironmentError(msg) list_to_change = self.spec_lists[list_name] existing = str(spec) in list_to_change.yaml_list @@ -985,11 +1018,18 @@ class Environment(object): concretized_specs.append((uspec, concrete)) return concretized_specs - def install(self, user_spec, concrete_spec=None, **install_args): - """Install a single spec into an environment. + def concretize_and_add(self, user_spec, concrete_spec=None): + """Concretize and add a single spec to the environment. - This will automatically concretize the single spec, but it won't - affect other as-yet unconcretized specs. + Concretize the provided ``user_spec`` and add it along with the + concretized result to the environment. If the given ``user_spec`` was + already present in the environment, this does not add a duplicate. + The concretized spec will be added unless the ``user_spec`` was + already present and an associated concrete spec was already present. + + Args: + concrete_spec: if provided, then it is assumed that it is the + result of concretizing the provided ``user_spec`` """ if self.concretization == 'together': msg = 'cannot install a single spec in an environment that is ' \ @@ -1000,37 +1040,21 @@ class Environment(object): spec = Spec(user_spec) - with spack.store.db.read_transaction(): - if self.add(spec): - concrete = concrete_spec or spec.concretized() + if self.add(spec): + concrete = concrete_spec or spec.concretized() + self._add_concrete_spec(spec, concrete) + else: + # spec might be in the user_specs, but not installed. + # TODO: Redo name-based comparison for old style envs + spec = next( + s for s in self.user_specs if s.satisfies(user_spec) + ) + concrete = self.specs_by_hash.get(spec.build_hash()) + if not concrete: + concrete = spec.concretized() self._add_concrete_spec(spec, concrete) - else: - # spec might be in the user_specs, but not installed. - # TODO: Redo name-based comparison for old style envs - spec = next( - s for s in self.user_specs if s.satisfies(user_spec) - ) - concrete = self.specs_by_hash.get(spec.build_hash()) - if not concrete: - concrete = spec.concretized() - self._add_concrete_spec(spec, concrete) - - self._install(concrete, **install_args) - def _install(self, spec, **install_args): - spec.package.do_install(**install_args) - - # Make sure log directory exists - log_path = self.log_path - fs.mkdirp(log_path) - - with fs.working_dir(self.path): - # Link the resulting log file into logs dir - build_log_link = os.path.join( - log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7))) - if os.path.lexists(build_log_link): - os.remove(build_log_link) - os.symlink(spec.package.build_log_path, build_log_link) + return concrete @property def default_view(self): @@ -1067,62 +1091,6 @@ class Environment(object): for view in self.views.values(): view.regenerate(specs, self.roots()) - prefix_inspections = { - 'bin': ['PATH'], - 'lib': ['LD_LIBRARY_PATH', 'LIBRARY_PATH', 'DYLD_LIBRARY_PATH'], - 'lib64': ['LD_LIBRARY_PATH', 'LIBRARY_PATH', 'DYLD_LIBRARY_PATH'], - 'man': ['MANPATH'], - 'share/man': ['MANPATH'], - 'share/aclocal': ['ACLOCAL_PATH'], - 'include': ['CPATH'], - 'lib/pkgconfig': ['PKG_CONFIG_PATH'], - 'lib64/pkgconfig': ['PKG_CONFIG_PATH'], - '': ['CMAKE_PREFIX_PATH'] - } - - def unconditional_environment_modifications(self, view): - """List of environment (shell) modifications to be processed for view. - - This list does not depend on the specs in this environment""" - env = spack.util.environment.EnvironmentModifications() - - for subdir, vars in self.prefix_inspections.items(): - full_subdir = os.path.join(view.root, subdir) - for var in vars: - env.prepend_path(var, full_subdir) - - return env - - def environment_modifications_for_spec(self, spec, view=None): - """List of environment (shell) modifications to be processed for spec. - - This list is specific to the location of the spec or its projection in - the view.""" - spec = spec.copy() - if view: - spec.prefix = Prefix(view.view().get_projection_for_spec(spec)) - - # generic environment modifications determined by inspecting the spec - # prefix - env = spack.util.environment.inspect_path( - spec.prefix, - self.prefix_inspections, - exclude=spack.util.environment.is_system_path - ) - - # Let the extendee/dependency modify their extensions/dependents - # before asking for package-specific modifications - env.extend( - build_env.modifications_from_dependencies( - spec, context='run' - ) - ) - # Package specific modifications - build_env.set_module_variables_for_package(spec.package) - spec.package.setup_run_environment(env) - - return env - def add_default_view_to_shell(self, shell): env_mod = spack.util.environment.EnvironmentModifications() @@ -1130,12 +1098,12 @@ class Environment(object): # No default view to add to shell return env_mod.shell_modifications(shell) - env_mod.extend(self.unconditional_environment_modifications( + env_mod.extend(uenv.unconditional_environment_modifications( self.default_view)) for _, spec in self.concretized_specs(): if spec in self.default_view and spec.package.installed: - env_mod.extend(self.environment_modifications_for_spec( + env_mod.extend(uenv.environment_modifications_for_spec( spec, self.default_view)) # deduplicate paths from specs mapped to the same location @@ -1151,13 +1119,13 @@ class Environment(object): # No default view to add to shell return env_mod.shell_modifications(shell) - env_mod.extend(self.unconditional_environment_modifications( + env_mod.extend(uenv.unconditional_environment_modifications( self.default_view).reversed()) for _, spec in self.concretized_specs(): if spec in self.default_view and spec.package.installed: env_mod.extend( - self.environment_modifications_for_spec( + uenv.environment_modifications_for_spec( spec, self.default_view).reversed()) return env_mod.shell_modifications(shell) @@ -1186,6 +1154,33 @@ class Environment(object): self.concretized_order.append(h) self.specs_by_hash[h] = concrete + def install(self, user_spec, concrete_spec=None, **install_args): + """Install a single spec into an environment. + + This will automatically concretize the single spec, but it won't + affect other as-yet unconcretized specs. + """ + concrete = self.concretize_and_add(user_spec, concrete_spec) + + self._install(concrete, **install_args) + + def _install(self, spec, **install_args): + # "spec" must be concrete + spec.package.do_install(**install_args) + + if not spec.external: + # Make sure log directory exists + log_path = self.log_path + fs.mkdirp(log_path) + + with fs.working_dir(self.path): + # Link the resulting log file into logs dir + build_log_link = os.path.join( + log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7))) + if os.path.lexists(build_log_link): + os.remove(build_log_link) + os.symlink(spec.package.build_log_path, build_log_link) + def install_all(self, args=None): """Install all concretized specs in an environment. @@ -1193,25 +1188,27 @@ class Environment(object): that needs to be done separately with a call to write(). """ + + # If "spack install" is invoked repeatedly for a large environment + # where all specs are already installed, the operation can take + # a large amount of time due to repeatedly acquiring and releasing + # locks, this does an initial check across all specs within a single + # DB read transaction to reduce time spent in this case. + uninstalled_specs = [] with spack.store.db.read_transaction(): for concretized_hash in self.concretized_order: spec = self.specs_by_hash[concretized_hash] + if not spec.package.installed: + uninstalled_specs.append(spec) - # Parse cli arguments and construct a dictionary - # that will be passed to Package.do_install API - kwargs = dict() - if args: - spack.cmd.install.update_kwargs_from_args(args, kwargs) - - self._install(spec, **kwargs) + for spec in uninstalled_specs: + # Parse cli arguments and construct a dictionary + # that will be passed to Package.do_install API + kwargs = dict() + if args: + spack.cmd.install.update_kwargs_from_args(args, kwargs) - if not spec.external: - # Link the resulting log file into logs dir - log_name = '%s-%s' % (spec.name, spec.dag_hash(7)) - build_log_link = os.path.join(self.log_path, log_name) - if os.path.lexists(build_log_link): - os.remove(build_log_link) - os.symlink(spec.package.build_log_path, build_log_link) + self._install(spec, **kwargs) def all_specs_by_hash(self): """Map of hashes to spec for all specs in this environment.""" @@ -1479,13 +1476,13 @@ class Environment(object): # Remove yaml sections that are shadowing defaults # construct garbage path to ensure we don't find a manifest by accident - bare_env = Environment(os.path.join(self.manifest_path, 'garbage'), - with_view=self.view_path_default) - keys_present = list(yaml_dict.keys()) - for key in keys_present: - if yaml_dict[key] == config_dict(bare_env.yaml).get(key, None): - if key not in raw_yaml_dict: - del yaml_dict[key] + with fs.temp_cwd() as env_dir: + bare_env = Environment(env_dir, with_view=self.view_path_default) + keys_present = list(yaml_dict.keys()) + for key in keys_present: + if yaml_dict[key] == config_dict(bare_env.yaml).get(key, None): + if key not in raw_yaml_dict: + del yaml_dict[key] # if all that worked, write out the manifest file at the top level # Only actually write if it has changed or was never written diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index 81968ac569..f79744f6a5 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py index 995464ea73..b8fde7d7fa 100644 --- a/lib/spack/spack/extensions.py +++ b/lib/spack/spack/extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 5a57703d27..5ae01286c4 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -22,33 +22,30 @@ in order to build it. They need to define the following methods: * archive() Archive a source directory, e.g. for creating a mirror. """ +import copy +import functools import os import os.path -import sys import re import shutil -import copy +import sys import xml.etree.ElementTree -from functools import wraps -from six import string_types, with_metaclass -import six.moves.urllib.parse as urllib_parse import llnl.util.tty as tty -from llnl.util.filesystem import ( - working_dir, mkdirp, temp_rename, temp_cwd, get_single_file) - +import six +import six.moves.urllib.parse as urllib_parse import spack.config import spack.error import spack.util.crypto as crypto import spack.util.pattern as pattern -import spack.util.web as web_util import spack.util.url as url_util - +import spack.util.web as web_util +from llnl.util.filesystem import ( + working_dir, mkdirp, temp_rename, temp_cwd, get_single_file) +from spack.util.compression import decompressor_for, extension from spack.util.executable import which from spack.util.string import comma_and, quote from spack.version import Version, ver -from spack.util.compression import decompressor_for, extension - #: List of all fetch strategies, created by FetchStrategy metaclass. all_strategies = [] @@ -69,7 +66,7 @@ def _needs_stage(fun): """Many methods on fetch strategies require a stage to be set using set_stage(). This decorator adds a check for self.stage.""" - @wraps(fun) + @functools.wraps(fun) def wrapper(self, *args, **kwargs): if not self.stage: raise NoStageError(fun) @@ -85,18 +82,14 @@ def _ensure_one_stage_entry(stage_path): return os.path.join(stage_path, stage_entries[0]) -class FSMeta(type): - """This metaclass registers all fetch strategies in a list.""" - def __init__(cls, name, bases, dict): - type.__init__(cls, name, bases, dict) - if cls.enabled: - all_strategies.append(cls) +def fetcher(cls): + """Decorator used to register fetch strategies.""" + all_strategies.append(cls) + return cls -class FetchStrategy(with_metaclass(FSMeta, object)): +class FetchStrategy(object): """Superclass of all fetch strategies.""" - enabled = False # Non-abstract subclasses should be enabled. - #: The URL attribute must be specified either at the package class #: level, or as a keyword argument to ``version()``. It is used to #: distinguish fetchers for different versions in the package DSL. @@ -113,16 +106,7 @@ class FetchStrategy(with_metaclass(FSMeta, object)): self.stage = None # Enable or disable caching for this strategy based on # 'no_cache' option from version directive. - self._cache_enabled = not kwargs.pop('no_cache', False) - - def set_stage(self, stage): - """This is called by Stage before any of the fetching - methods are called on the stage.""" - self.stage = stage - - @property - def cache_enabled(self): - return self._cache_enabled + self.cache_enabled = not kwargs.pop('no_cache', False) # Subclasses need to implement these methods def fetch(self): @@ -186,13 +170,18 @@ class FetchStrategy(with_metaclass(FSMeta, object)): def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" - # This method is used to match fetch strategies to version() - # arguments in packages. @classmethod def matches(cls, args): + """Predicate that matches fetch strategies to arguments of + the version directive. + + Args: + args: arguments of the version directive + """ return cls.url_attr in args +@fetcher class BundleFetchStrategy(FetchStrategy): """ Fetch strategy associated with bundle, or no-code, packages. @@ -204,9 +193,6 @@ class BundleFetchStrategy(FetchStrategy): TODO: Remove this class by refactoring resource handling and the link between composite stages and composite fetch strategies (see #11981). """ - #: This is a concrete fetch strategy for no-code packages. - enabled = True - #: There is no associated URL keyword in ``version()`` for no-code #: packages but this property is required for some strategy-related #: functions (e.g., check_pkg_attributes). @@ -225,6 +211,9 @@ class BundleFetchStrategy(FetchStrategy): """BundlePackages don't have a source id.""" return '' + def mirror_id(self): + """BundlePackages don't have a mirror id.""" + @pattern.composite(interface=FetchStrategy) class FetchStrategyComposite(object): @@ -233,7 +222,6 @@ class FetchStrategyComposite(object): Implements the GoF composite pattern. """ matches = FetchStrategy.matches - set_stage = FetchStrategy.set_stage def source_id(self): component_ids = tuple(i.source_id() for i in self) @@ -241,13 +229,13 @@ class FetchStrategyComposite(object): return component_ids +@fetcher class URLFetchStrategy(FetchStrategy): + """URLFetchStrategy pulls source code from a URL for an archive, check the + archive against a checksum, and decompresses the archive. + + The destination for the resulting file(s) is the standard stage path. """ - FetchStrategy that pulls source code from a URL for an archive, check the - archive against a checksum, and decompresses the archive. The destination - for the resulting file(s) is the standard stage source path. - """ - enabled = True url_attr = 'url' # these are checksum types. The generic 'checksum' is deprecated for @@ -259,6 +247,7 @@ class URLFetchStrategy(FetchStrategy): # Prefer values in kwargs to the positionals. self.url = kwargs.get('url', url) + self.mirrors = kwargs.get('mirrors', []) # digest can be set as the first argument, or from an explicit # kwarg by the hash name. @@ -294,20 +283,36 @@ class URLFetchStrategy(FetchStrategy): return os.path.sep.join( ['archive', self.digest[:2], self.digest]) + @property + def candidate_urls(self): + return [self.url] + (self.mirrors or []) + @_needs_stage def fetch(self): if self.archive_file: tty.msg("Already downloaded %s" % self.archive_file) return + for url in self.candidate_urls: + try: + partial_file, save_file = self._fetch_from_url(url) + if save_file: + os.rename(partial_file, save_file) + break + except FetchError as e: + tty.msg(str(e)) + pass + + if not self.archive_file: + raise FailedDownloadError(self.url) + + def _fetch_from_url(self, url): save_file = None partial_file = None if self.stage.save_filename: save_file = self.stage.save_filename partial_file = self.stage.save_filename + '.part' - - tty.msg("Fetching %s" % self.url) - + tty.msg("Fetching %s" % url) if partial_file: save_args = ['-C', '-', # continue partial downloads @@ -321,7 +326,9 @@ class URLFetchStrategy(FetchStrategy): '-D', '-', # print out HTML headers '-L', # resolve 3xx redirects - self.url, + # Timeout if can't establish a connection after 10 sec. + '--connect-timeout', '10', + url, ] if not spack.config.get('config:verify_ssl'): @@ -377,12 +384,7 @@ class URLFetchStrategy(FetchStrategy): flags=re.IGNORECASE) if content_types and 'text/html' in content_types[-1]: warn_content_type_mismatch(self.archive_file or "the archive") - - if save_file: - os.rename(partial_file, save_file) - - if not self.archive_file: - raise FailedDownloadError(self.url) + return partial_file, save_file @property @_needs_stage @@ -392,7 +394,7 @@ class URLFetchStrategy(FetchStrategy): @property def cachable(self): - return self._cache_enabled and bool(self.digest) + return self.cache_enabled and bool(self.digest) @_needs_stage def expand(self): @@ -519,6 +521,7 @@ class URLFetchStrategy(FetchStrategy): return "[no url]" +@fetcher class CacheURLFetchStrategy(URLFetchStrategy): """The resource associated with a cache URL may be out of date.""" @@ -594,7 +597,7 @@ class VCSFetchStrategy(FetchStrategy): patterns = kwargs.get('exclude', None) if patterns is not None: - if isinstance(patterns, string_types): + if isinstance(patterns, six.string_types): patterns = [patterns] for p in patterns: tar.add_default_arg('--exclude=%s' % p) @@ -618,6 +621,7 @@ class VCSFetchStrategy(FetchStrategy): return "%s<%s>" % (self.__class__, self.url) +@fetcher class GoFetchStrategy(VCSFetchStrategy): """Fetch strategy that employs the `go get` infrastructure. @@ -631,7 +635,6 @@ class GoFetchStrategy(VCSFetchStrategy): The fetched source will be moved to the standard stage sourcepath directory during the expand step. """ - enabled = True url_attr = 'go' def __init__(self, **kwargs): @@ -688,6 +691,7 @@ class GoFetchStrategy(VCSFetchStrategy): return "[go] %s" % self.url +@fetcher class GitFetchStrategy(VCSFetchStrategy): """ @@ -709,9 +713,9 @@ class GitFetchStrategy(VCSFetchStrategy): Repositories are cloned into the standard stage source path directory. """ - enabled = True url_attr = 'git' - optional_attrs = ['tag', 'branch', 'commit', 'submodules', 'get_full_repo'] + optional_attrs = ['tag', 'branch', 'commit', 'submodules', + 'get_full_repo', 'submodules_delete'] def __init__(self, **kwargs): # Discards the keywords in kwargs that may conflict with the next call @@ -722,6 +726,7 @@ class GitFetchStrategy(VCSFetchStrategy): self._git = None self.submodules = kwargs.get('submodules', False) + self.submodules_delete = kwargs.get('submodules_delete', False) self.get_full_repo = kwargs.get('get_full_repo', False) @property @@ -743,7 +748,7 @@ class GitFetchStrategy(VCSFetchStrategy): @property def cachable(self): - return self._cache_enabled and bool(self.commit or self.tag) + return self.cache_enabled and bool(self.commit or self.tag) def source_id(self): return self.commit or self.tag @@ -855,6 +860,14 @@ class GitFetchStrategy(VCSFetchStrategy): git(*pull_args, ignore_errors=1) git(*co_args) + if self.submodules_delete: + with working_dir(self.stage.source_path): + for submodule_to_delete in self.submodules_delete: + args = ['rm', submodule_to_delete] + if not spack.config.get('config:debug'): + args.insert(1, '--quiet') + git(*args) + # Init submodules if the user asked for them. if self.submodules: with working_dir(self.stage.source_path): @@ -889,6 +902,7 @@ class GitFetchStrategy(VCSFetchStrategy): return '[git] {0}'.format(self._repo_info()) +@fetcher class SvnFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a subversion repository. @@ -903,7 +917,6 @@ class SvnFetchStrategy(VCSFetchStrategy): Repositories are checked out into the standard stage source path directory. """ - enabled = True url_attr = 'svn' optional_attrs = ['revision'] @@ -926,7 +939,7 @@ class SvnFetchStrategy(VCSFetchStrategy): @property def cachable(self): - return self._cache_enabled and bool(self.revision) + return self.cache_enabled and bool(self.revision) def source_id(self): return self.revision @@ -988,6 +1001,7 @@ class SvnFetchStrategy(VCSFetchStrategy): return "[svn] %s" % self.url +@fetcher class HgFetchStrategy(VCSFetchStrategy): """ @@ -1010,7 +1024,6 @@ class HgFetchStrategy(VCSFetchStrategy): Repositories are cloned into the standard stage source path directory. """ - enabled = True url_attr = 'hg' optional_attrs = ['revision'] @@ -1040,7 +1053,7 @@ class HgFetchStrategy(VCSFetchStrategy): @property def cachable(self): - return self._cache_enabled and bool(self.revision) + return self.cache_enabled and bool(self.revision) def source_id(self): return self.revision @@ -1105,9 +1118,9 @@ class HgFetchStrategy(VCSFetchStrategy): return "[hg] %s" % self.url +@fetcher class S3FetchStrategy(URLFetchStrategy): """FetchStrategy that pulls from an S3 bucket.""" - enabled = True url_attr = 's3' def __init__(self, *args, **kwargs): @@ -1139,7 +1152,7 @@ class S3FetchStrategy(URLFetchStrategy): with open(basename, 'wb') as f: shutil.copyfileobj(stream, f) - content_type = headers['Content-type'] + content_type = web_util.get_header(headers, 'Content-type') if content_type == 'text/html': warn_content_type_mismatch(self.archive_file or "the archive") @@ -1241,10 +1254,15 @@ def _from_merged_attrs(fetcher, pkg, version): """Create a fetcher from merged package and version attributes.""" if fetcher.url_attr == 'url': url = pkg.url_for_version(version) + # TODO: refactor this logic into its own method or function + # TODO: to avoid duplication + mirrors = [spack.url.substitute_version(u, version) + for u in getattr(pkg, 'urls', [])] + attrs = {fetcher.url_attr: url, 'mirrors': mirrors} else: url = getattr(pkg, fetcher.url_attr) + attrs = {fetcher.url_attr: url} - attrs = {fetcher.url_attr: url} attrs.update(pkg.versions[version]) return fetcher(**attrs) @@ -1308,7 +1326,9 @@ def from_url_scheme(url, *args, **kwargs): { 'file': 'url', 'http': 'url', - 'https': 'url' + 'https': 'url', + 'ftp': 'url', + 'ftps': 'url', }) scheme = parsed_url.scheme diff --git a/lib/spack/spack/filesystem_view.py b/lib/spack/spack/filesystem_view.py index 3d17d7e4ce..babe1b9c0e 100644 --- a/lib/spack/spack/filesystem_view.py +++ b/lib/spack/spack/filesystem_view.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index 9a1571d75c..b41c923a16 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hash_types.py b/lib/spack/spack/hash_types.py index e4a238a93b..9404076e81 100644 --- a/lib/spack/spack/hash_types.py +++ b/lib/spack/spack/hash_types.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index 860134112e..5a39f242fd 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -12,7 +12,6 @@ Currently the following hooks are supported: - * pre_run() * pre_install(spec) * post_install(spec) * pre_uninstall(spec) @@ -60,11 +59,6 @@ class HookRunner(object): hook(*args, **kwargs) -# -# Define some functions that can be called to fire off hooks. -# -pre_run = HookRunner('pre_run') - pre_install = HookRunner('pre_install') post_install = HookRunner('post_install') diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index bfda25918e..05fce302ac 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/licensing.py b/lib/spack/spack/hooks/licensing.py index 8a56dc122f..d4e5376cda 100644 --- a/lib/spack/spack/hooks/licensing.py +++ b/lib/spack/spack/hooks/licensing.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/module_file_generation.py b/lib/spack/spack/hooks/module_file_generation.py index fb631694b9..ba30561d86 100644 --- a/lib/spack/spack/hooks/module_file_generation.py +++ b/lib/spack/spack/hooks/module_file_generation.py @@ -1,21 +1,21 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import spack.config import spack.modules import spack.modules.common import llnl.util.tty as tty -try: - enabled = spack.config.get('modules:enable') -except KeyError: - tty.debug('NO MODULE WRITTEN: list of enabled module files is empty') - enabled = [] - def _for_each_enabled(spec, method_name): """Calls a method for each enabled module""" + enabled = spack.config.get('modules:enable') + if not enabled: + tty.debug('NO MODULE WRITTEN: list of enabled module files is empty') + return + for name in enabled: generator = spack.modules.module_types[name](spec) try: @@ -26,5 +26,9 @@ def _for_each_enabled(spec, method_name): tty.warn(msg.format(method_name, str(e))) -post_install = lambda spec: _for_each_enabled(spec, 'write') -post_uninstall = lambda spec: _for_each_enabled(spec, 'remove') +def post_install(spec): + _for_each_enabled(spec, 'write') + + +def post_uninstall(spec): + _for_each_enabled(spec, 'remove') diff --git a/lib/spack/spack/hooks/permissions_setters.py b/lib/spack/spack/hooks/permissions_setters.py index 10d75906e2..0a3c2acf21 100644 --- a/lib/spack/spack/hooks/permissions_setters.py +++ b/lib/spack/spack/hooks/permissions_setters.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 422f240ce8..b4ee0c1728 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/write_install_manifest.py b/lib/spack/spack/hooks/write_install_manifest.py index 8118caae24..5b678fed88 100644 --- a/lib/spack/spack/hooks/write_install_manifest.py +++ b/lib/spack/spack/hooks/write_install_manifest.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/hooks/yaml_version_check.py b/lib/spack/spack/hooks/yaml_version_check.py deleted file mode 100644 index ff8be58481..0000000000 --- a/lib/spack/spack/hooks/yaml_version_check.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""Yaml Version Check is a module for ensuring that config file -formats are compatible with the current version of Spack.""" -import os.path -import os -import llnl.util.tty as tty -import spack.util.spack_yaml as syaml -import spack.config - - -def pre_run(): - check_compiler_yaml_version() - - -def check_compiler_yaml_version(): - config = spack.config.config - - for scope in config.file_scopes: - file_name = os.path.join(scope.path, 'compilers.yaml') - data = None - if os.path.isfile(file_name): - with open(file_name) as f: - data = syaml.load_config(f) - - if data: - compilers = data.get('compilers') - if compilers and len(compilers) > 0: - if (not isinstance(compilers, list) or - 'operating_system' not in compilers[0]['compiler']): - new_file = os.path.join(scope.path, '_old_compilers.yaml') - tty.warn('%s in out of date compilers format. ' - 'Moved to %s. Spack automatically generate ' - 'a compilers config file ' - % (file_name, new_file)) - os.rename(file_name, new_file) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index ff2ff74da3..4386f50435 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -13,6 +13,7 @@ from __future__ import print_function import sys import re import os +import os.path import inspect import pstats import argparse @@ -30,12 +31,12 @@ import spack.architecture import spack.config import spack.cmd import spack.environment as ev -import spack.hooks import spack.paths import spack.repo import spack.store import spack.util.debug import spack.util.path +import spack.util.executable as exe from spack.error import SpackError @@ -108,6 +109,34 @@ def add_all_commands(parser): parser.add_command(cmd) +def get_version(): + """Get a descriptive version of this instance of Spack. + + If this is a git repository, and if it is not on a release tag, + return a string like: + + release_version-commits_since_release-commit + + If we *are* at a release tag, or if this is not a git repo, return + the real spack release number (e.g., 0.13.3). + + """ + git_path = os.path.join(spack.paths.prefix, ".git") + if os.path.exists(git_path): + git = exe.which("git") + if git: + desc = git("-C", spack.paths.prefix, "describe", "--tags", + output=str, fail_on_error=False) + + if git.returncode == 0: + match = re.match(r"v([^-]+)-([^-]+)-g([a-f\d]+)", desc) + if match: + v, n, commit = match.groups() + return "%s-%s-%s" % (v, n, commit) + + return spack.spack_version + + def index_commands(): """create an index of commands by section for this help level""" index = {} @@ -646,6 +675,17 @@ def main(argv=None): parser.add_argument('command', nargs=argparse.REMAINDER) args, unknown = parser.parse_known_args(argv) + # Recover stored LD_LIBRARY_PATH variables from spack shell function + # This is necessary because MacOS System Integrity Protection clears + # (DY?)LD_LIBRARY_PATH variables on process start. + # Spack clears these variables before building and installing packages, + # but needs to know the prior state for commands like `spack load` and + # `spack env activate that modify the user environment. + for var in ('LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH'): + stored_var_name = 'SPACK_%s' % var + if stored_var_name in os.environ: + os.environ[var] = os.environ[stored_var_name] + # activate an environment if one was specified on the command line if not args.no_env: env = ev.find_environment(args) @@ -669,7 +709,7 @@ def main(argv=None): # -h, -H, and -V are special as they do not require a command, but # all the other options do nothing without a command. if args.version: - print(spack.spack_version) + print(get_version()) return 0 elif args.help: sys.stdout.write(parser.format_help(level=args.help)) @@ -700,9 +740,6 @@ def main(argv=None): # many operations will fail without a working directory. set_working_dir() - # pre-run hooks happen after we know we have a valid working dir - spack.hooks.pre_run() - # now we can actually execute the command. if args.spack_profile or args.sorted_profile: _profile_wrapper(command, parser, args, unknown) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index da9b20472a..3f6b2b6a0e 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -21,8 +21,10 @@ import six import ruamel.yaml.error as yaml_error +from ordereddict_backport import OrderedDict + try: - from collections.abc import Mapping + from collections.abc import Mapping # novm except ImportError: from collections import Mapping @@ -166,7 +168,7 @@ class MirrorCollection(Mapping): """A mapping of mirror names to mirrors.""" def __init__(self, mirrors=None, scope=None): - self._mirrors = dict( + self._mirrors = OrderedDict( (name, Mirror.from_dict(mirror, name)) for name, mirror in ( mirrors.items() if mirrors is not None else @@ -178,6 +180,7 @@ class MirrorCollection(Mapping): def to_yaml(self, stream=None): return syaml.dump(self.to_dict(True), stream) + # TODO: this isn't called anywhere @staticmethod def from_yaml(stream, name=None): try: diff --git a/lib/spack/spack/mixins.py b/lib/spack/spack/mixins.py index 198bd03bcb..f7162a3a84 100644 --- a/lib/spack/spack/mixins.py +++ b/lib/spack/spack/mixins.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/modules/__init__.py b/lib/spack/spack/modules/__init__.py index 20864b5f27..2fb4d94b05 100644 --- a/lib/spack/spack/modules/__init__.py +++ b/lib/spack/spack/modules/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index 2e0090a449..d9407a1bf6 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -28,35 +28,30 @@ and is divided among four classes: Each of the four classes needs to be sub-classed when implementing a new module type. """ +import collections import copy import datetime import inspect import os.path import re -import collections -import six import llnl.util.filesystem import llnl.util.tty as tty - -import spack.paths import spack.build_environment as build_environment -import spack.util.environment +import spack.error +import spack.paths +import spack.schema.environment import spack.tengine as tengine -import spack.util.path import spack.util.environment -import spack.error -import spack.util.spack_yaml as syaml import spack.util.file_permissions as fp +import spack.util.path +import spack.util.spack_yaml as syaml -#: config section for this file -configuration = spack.config.get('modules') -#: Root folders where the various module files should be written -roots = spack.config.get('config:module_roots', {}) +#: config section for this file +def configuration(): + return spack.config.get('modules', {}) -#: Inspections that needs to be done on spec prefixes -prefix_inspections = spack.config.get('modules:prefix_inspections', {}) #: Valid tokens for naming scheme and env variable names _valid_tokens = ( @@ -214,11 +209,13 @@ def root_path(name): """Returns the root folder for module file installation. Args: - name: name of the module system t be used (e.g. 'tcl') + name: name of the module system to be used (e.g. 'tcl') Returns: root folder for module file installation """ + # Root folders where the various module files should be written + roots = spack.config.get('config:module_roots', {}) path = roots.get(name, os.path.join(spack.paths.share_path, name)) return spack.util.path.canonicalize_path(path) @@ -387,12 +384,12 @@ class BaseConfiguration(object): self.spec = spec # Dictionary of configuration options that should be applied # to the spec - self.conf = merge_config_rules(self.module.configuration, self.spec) + self.conf = merge_config_rules(self.module.configuration(), self.spec) @property def naming_scheme(self): """Naming scheme suitable for non-hierarchical layouts""" - scheme = self.module.configuration.get( + scheme = self.module.configuration().get( 'naming_scheme', '{name}-{version}-{compiler.name}-{compiler.version}' ) @@ -416,22 +413,7 @@ class BaseConfiguration(object): """List of environment modifications that should be done in the module. """ - env_mods = spack.util.environment.EnvironmentModifications() - actions = self.conf.get('environment', {}) - - def process_arglist(arglist): - if method == 'unset': - for x in arglist: - yield (x,) - else: - for x in six.iteritems(arglist): - yield x - - for method, arglist in actions.items(): - for args in process_arglist(arglist): - getattr(env_mods, method)(*args) - - return env_mods + return spack.schema.environment.parse(self.conf.get('environment', {})) @property def suffixes(self): @@ -461,7 +443,7 @@ class BaseConfiguration(object): """ # A few variables for convenience of writing the method spec = self.spec - conf = self.module.configuration + conf = self.module.configuration() # Compute the list of whitelist rules that match wlrules = conf.get('whitelist', []) @@ -662,7 +644,7 @@ class BaseContext(tengine.Context): # Modifications guessed inspecting the spec prefix env = spack.util.environment.inspect_path( self.spec.prefix, - prefix_inspections, + spack.config.get('modules:prefix_inspections', {}), exclude=spack.util.environment.is_system_path ) @@ -804,10 +786,11 @@ class BaseModuleFileWriter(object): # Get the template for the module template_name = self._get_template() + import jinja2 try: env = tengine.make_environment() template = env.get_template(template_name) - except tengine.TemplateNotFound: + except jinja2.TemplateNotFound: # If the template was not found raise an exception with a little # more information msg = 'template \'{0}\' was not found for \'{1}\'' diff --git a/lib/spack/spack/modules/lmod.py b/lib/spack/spack/modules/lmod.py index e45b97773b..a9ebb0bba9 100644 --- a/lib/spack/spack/modules/lmod.py +++ b/lib/spack/spack/modules/lmod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -18,8 +18,11 @@ import spack.tengine as tengine from .common import BaseConfiguration, BaseFileLayout from .common import BaseContext, BaseModuleFileWriter + #: lmod specific part of the configuration -configuration = spack.config.get('modules:lmod', {}) +def configuration(): + return spack.config.get('modules:lmod', {}) + #: Caches the configuration {spec_hash: configuration} configuration_registry = {} @@ -98,7 +101,7 @@ class LmodConfiguration(BaseConfiguration): specified in the configuration file or the sequence is empty """ - value = configuration.get( + value = configuration().get( 'core_compilers' ) or guess_core_compilers(store=True) @@ -112,7 +115,7 @@ class LmodConfiguration(BaseConfiguration): """Returns the list of tokens that are part of the modulefile hierarchy. 'compiler' is always present. """ - tokens = configuration.get('hierarchy', []) + tokens = configuration().get('hierarchy', []) # Check if all the tokens in the hierarchy are virtual specs. # If not warn the user and raise an error. diff --git a/lib/spack/spack/modules/tcl.py b/lib/spack/spack/modules/tcl.py index 1c9d741a2d..ec8032a0e1 100644 --- a/lib/spack/spack/modules/tcl.py +++ b/lib/spack/spack/modules/tcl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -16,8 +16,11 @@ import spack.tengine as tengine from .common import BaseConfiguration, BaseFileLayout from .common import BaseContext, BaseModuleFileWriter + #: TCL specific part of the configuration -configuration = spack.config.get('modules:tcl', {}) +def configuration(): + return spack.config.get('modules:tcl', {}) + #: Caches the configuration {spec_hash: configuration} configuration_registry = {} diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index e0854a01c9..73cf342b5b 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py index 94f8ac4d9e..9f87532b85 100644 --- a/lib/spack/spack/operating_systems/__init__.py +++ b/lib/spack/spack/operating_systems/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/cnk.py b/lib/spack/spack/operating_systems/cnk.py index 99ea9455fa..53a12785da 100644 --- a/lib/spack/spack/operating_systems/cnk.py +++ b/lib/spack/spack/operating_systems/cnk.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 24075de3fd..3d4036cb47 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/cray_frontend.py b/lib/spack/spack/operating_systems/cray_frontend.py index 1e8d9f4111..15f5b23baf 100644 --- a/lib/spack/spack/operating_systems/cray_frontend.py +++ b/lib/spack/spack/operating_systems/cray_frontend.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index 8a18c27b41..a6608e6d3b 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/operating_systems/mac_os.py b/lib/spack/spack/operating_systems/mac_os.py index 2f98c0cd79..fa7718f806 100644 --- a/lib/spack/spack/operating_systems/mac_os.py +++ b/lib/spack/spack/operating_systems/mac_os.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 26335ed2ff..d67c017a70 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -510,8 +510,8 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): maintainers = [] #: List of attributes to be excluded from a package's hash. - metadata_attrs = ['homepage', 'url', 'list_url', 'extendable', 'parallel', - 'make_jobs'] + metadata_attrs = ['homepage', 'url', 'urls', 'list_url', 'extendable', + 'parallel', 'make_jobs'] def __init__(self, spec): # this determines how the package should be built. @@ -524,6 +524,12 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): # a binary cache. self.installed_from_binary_cache = False + # Ensure that only one of these two attributes are present + if getattr(self, 'url', None) and getattr(self, 'urls', None): + msg = "a package can have either a 'url' or a 'urls' attribute" + msg += " [package '{0.name}' defines both]" + raise ValueError(msg.format(self)) + # Set a default list URL (place to find available versions) if not hasattr(self, 'list_url'): self.list_url = None @@ -556,16 +562,19 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): @classmethod def possible_dependencies( cls, transitive=True, expand_virtuals=True, deptype='all', - visited=None): + visited=None, missing=None): """Return dict of possible dependencies of this package. Args: - transitive (bool): return all transitive dependencies if True, - only direct dependencies if False. - expand_virtuals (bool): expand virtual dependencies into all - possible implementations. - deptype (str or tuple): dependency types to consider - visited (set): set of names of dependencies visited so far. + transitive (bool, optional): return all transitive dependencies if + True, only direct dependencies if False (default True).. + expand_virtuals (bool, optional): expand virtual dependencies into + all possible implementations (default True) + deptype (str or tuple, optional): dependency types to consider + visited (dicct, optional): dict of names of dependencies visited so + far, mapped to their immediate dependencies' names. + missing (dict, optional): dict to populate with packages and their + *missing* dependencies. Returns: (dict): dictionary mapping dependency names to *their* @@ -576,7 +585,12 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): *immediate* dependencies. If ``expand_virtuals`` is ``False``, virtual package names wil be inserted as keys mapped to empty sets of dependencies. Virtuals, if not expanded, are treated as - though they have no immediate dependencies + though they have no immediate dependencies. + + Missing dependencies by default are ignored, but if a + missing dict is provided, it will be populated with package names + mapped to any dependencies they have that are in no + repositories. This is only populated if transitive is True. Note: the returned dict *includes* the package itself. @@ -586,6 +600,9 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): if visited is None: visited = {cls.name: set()} + if missing is None: + missing = {cls.name: set()} + for name, conditions in cls.dependencies.items(): # check whether this dependency could be of the type asked for types = [dep.type for cond, dep in conditions.items()] @@ -609,12 +626,24 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): # recursively traverse dependencies for dep_name in dep_names: - if dep_name not in visited: - visited.setdefault(dep_name, set()) - if transitive: - dep_cls = spack.repo.path.get_pkg_class(dep_name) - dep_cls.possible_dependencies( - transitive, expand_virtuals, deptype, visited) + if dep_name in visited: + continue + + visited.setdefault(dep_name, set()) + + # skip the rest if not transitive + if not transitive: + continue + + try: + dep_cls = spack.repo.path.get_pkg_class(dep_name) + except spack.repo.UnknownPackageError: + # log unknown packages + missing.setdefault(cls.name, set()).add(dep_name) + continue + + dep_cls.possible_dependencies( + transitive, expand_virtuals, deptype, visited, missing) return visited @@ -727,7 +756,9 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): return version_urls[version] # If no specific URL, use the default, class-level URL - default_url = getattr(self, 'url', None) + url = getattr(self, 'url', None) + urls = getattr(self, 'urls', [None]) + default_url = url or urls.pop(0) # if no exact match AND no class-level default, use the nearest URL if not default_url: @@ -1231,7 +1262,10 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): raise spack.error.SpackError(err_msg) hash_content = list() - source_id = fs.for_package_version(self, self.version).source_id() + try: + source_id = fs.for_package_version(self, self.version).source_id() + except fs.ExtrapolationError: + source_id = None if not source_id: # TODO? in cases where a digest or source_id isn't available, # should this attempt to download the source and set one? This @@ -1474,9 +1508,10 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): message = '{s.name}@{s.version} : marking the package explicit' tty.msg(message.format(s=self)) - def try_install_from_binary_cache(self, explicit): + def try_install_from_binary_cache(self, explicit, unsigned=False): tty.msg('Searching for binary cache of %s' % self.name) - specs = binary_distribution.get_specs() + specs = binary_distribution.get_spec(spec=self.spec, + force=False) binary_spec = spack.spec.Spec.from_dict(self.spec.to_dict()) binary_spec._mark_concrete() if binary_spec not in specs: @@ -1490,7 +1525,7 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): tty.msg('Installing %s from binary cache' % self.name) binary_distribution.extract_tarball( binary_spec, tarball, allow_root=False, - unsigned=False, force=False) + unsigned=unsigned, force=False) self.installed_from_binary_cache = True spack.store.db.add( self.spec, spack.store.layout, explicit=explicit) @@ -1509,6 +1544,7 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): ) if not compilers: dep = spack.compilers.pkg_spec_for_compiler(self.spec.compiler) + dep.architecture = self.spec.architecture # concrete CompilerSpec has less info than concrete Spec # concretize as Spec to add that information dep.concretize() @@ -1630,14 +1666,16 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): tty.msg(colorize('@*{Installing} @*g{%s}' % self.name)) if kwargs.get('use_cache', True): - if self.try_install_from_binary_cache(explicit): + if self.try_install_from_binary_cache( + explicit, unsigned=kwargs.get('unsigned', False)): tty.msg('Successfully installed %s from binary cache' % self.name) print_pkg(self.prefix) spack.hooks.post_install(self.spec) return elif kwargs.get('cache_only', False): - tty.die('No binary for %s found and cache-only specified') + tty.die('No binary for %s found and cache-only specified' + % self.name) tty.msg('No binary for %s found: installing from source' % self.name) @@ -2662,6 +2700,35 @@ def dump_packages(spec, path): spack.repo.path.dump_provenance(node, dest_pkg_dir) +def possible_dependencies(*pkg_or_spec, **kwargs): + """Get the possible dependencies of a number of packages. + + See ``PackageBase.possible_dependencies`` for details. + """ + transitive = kwargs.get('transitive', True) + expand_virtuals = kwargs.get('expand_virtuals', True) + deptype = kwargs.get('deptype', 'all') + missing = kwargs.get('missing') + + packages = [] + for pos in pkg_or_spec: + if isinstance(pos, PackageMeta): + pkg = pos + elif isinstance(pos, spack.spec.Spec): + pkg = pos.package + else: + pkg = spack.spec.Spec(pos).package + + packages.append(pkg) + + visited = {} + for pkg in packages: + pkg.possible_dependencies( + transitive, expand_virtuals, deptype, visited, missing) + + return visited + + def print_pkg(message): """Outputs a message with a package icon.""" from llnl.util.tty.color import cwrite diff --git a/lib/spack/spack/package_prefs.py b/lib/spack/spack/package_prefs.py index e73ce7c947..2801a6d123 100644 --- a/lib/spack/spack/package_prefs.py +++ b/lib/spack/spack/package_prefs.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,8 +8,6 @@ import stat from six import string_types from six import iteritems -from llnl.util.lang import classproperty - import spack.repo import spack.error from spack.util.path import canonicalize_path @@ -75,14 +73,13 @@ class PackagePrefs(object): provider_spec_list.sort(key=kf) """ - _packages_config_cache = None - _spec_cache = {} - def __init__(self, pkgname, component, vpkg=None): self.pkgname = pkgname self.component = component self.vpkg = vpkg + self._spec_order = None + def __call__(self, spec): """Return a key object (an index) that can be used to sort spec. @@ -90,8 +87,10 @@ class PackagePrefs(object): this function as Python's sort functions already ensure that the key function is called at most once per sorted element. """ - spec_order = self._specs_for_pkg( - self.pkgname, self.component, self.vpkg) + if self._spec_order is None: + self._spec_order = self._specs_for_pkg( + self.pkgname, self.component, self.vpkg) + spec_order = self._spec_order # integer is the index of the first spec in order that satisfies # spec, or it's a number larger than any position in the order. @@ -107,13 +106,6 @@ class PackagePrefs(object): match_index -= 0.5 return match_index - @classproperty - @classmethod - def _packages_config(cls): - if cls._packages_config_cache is None: - cls._packages_config_cache = get_packages_config() - return cls._packages_config_cache - @classmethod def order_for_package(cls, pkgname, component, vpkg=None, all=True): """Given a package name, sort component (e.g, version, compiler, ...), @@ -124,7 +116,7 @@ class PackagePrefs(object): pkglist.append('all') for pkg in pkglist: - pkg_entry = cls._packages_config.get(pkg) + pkg_entry = get_packages_config().get(pkg) if not pkg_entry: continue @@ -150,21 +142,9 @@ class PackagePrefs(object): return a list of CompilerSpecs, VersionLists, or Specs for that sorting list. """ - key = (pkgname, component, vpkg) - - specs = cls._spec_cache.get(key) - if specs is None: - pkglist = cls.order_for_package(pkgname, component, vpkg) - spec_type = _spec_type(component) - specs = [spec_type(s) for s in pkglist] - cls._spec_cache[key] = specs - - return specs - - @classmethod - def clear_caches(cls): - cls._packages_config_cache = None - cls._spec_cache = {} + pkglist = cls.order_for_package(pkgname, component, vpkg) + spec_type = _spec_type(component) + return [spec_type(s) for s in pkglist] @classmethod def has_preferred_providers(cls, pkgname, vpkg): @@ -180,7 +160,7 @@ class PackagePrefs(object): def preferred_variants(cls, pkg_name): """Return a VariantMap of preferred variants/values for a spec.""" for pkg in (pkg_name, 'all'): - variants = cls._packages_config.get(pkg, {}).get('variants', '') + variants = get_packages_config().get(pkg, {}).get('variants', '') if variants: break diff --git a/lib/spack/spack/package_test.py b/lib/spack/spack/package_test.py index 559a8df3f5..6e9da9a985 100644 --- a/lib/spack/spack/package_test.py +++ b/lib/spack/spack/package_test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/parse.py b/lib/spack/spack/parse.py index 6bfdec7b43..12bbcee588 100644 --- a/lib/spack/spack/parse.py +++ b/lib/spack/spack/parse.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index 73fdf4df25..bcb45387a8 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d91b01d87c..e5541eff10 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/pkgkit.py b/lib/spack/spack/pkgkit.py index 7ad7279e73..a39ede3ab7 100644 --- a/lib/spack/spack/pkgkit.py +++ b/lib/spack/spack/pkgkit.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -30,6 +30,7 @@ from spack.build_systems.perl import PerlPackage from spack.build_systems.intel import IntelPackage from spack.build_systems.meson import MesonPackage from spack.build_systems.sip import SIPPackage +from spack.build_systems.gnu import GNUMirrorPackage from spack.mixins import filter_compiler_wrappers diff --git a/lib/spack/spack/platforms/__init__.py b/lib/spack/spack/platforms/__init__.py index 94f8ac4d9e..9f87532b85 100644 --- a/lib/spack/spack/platforms/__init__.py +++ b/lib/spack/spack/platforms/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index 3823864a64..64d71743c4 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index 3499f0b164..cbeba96461 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 7c36812d9d..9a3fffd881 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index d2165c7d79..4ffd7ad8e9 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py index c9bbd02dfa..9be160731e 100644 --- a/lib/spack/spack/platforms/test.py +++ b/lib/spack/spack/platforms/test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/provider_index.py b/lib/spack/spack/provider_index.py index 1d9b31add6..9bf4af8911 100644 --- a/lib/spack/spack/provider_index.py +++ b/lib/spack/spack/provider_index.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -135,7 +135,7 @@ class ProviderIndex(object): for lspec, rspec in iproduct(lmap, rmap): try: constrained = lspec.constrained(rspec) - except spack.spec.UnsatisfiableSpecError: + except spack.error.UnsatisfiableSpecError: continue # lp and rp are left and right provider specs. @@ -144,7 +144,7 @@ class ProviderIndex(object): try: const = lp_spec.constrained(rp_spec, deps=False) result.setdefault(constrained, set()).add(const) - except spack.spec.UnsatisfiableSpecError: + except spack.error.UnsatisfiableSpecError: continue return result diff --git a/lib/spack/spack/relocate.py b/lib/spack/spack/relocate.py index 434d681035..d722e587da 100644 --- a/lib/spack/spack/relocate.py +++ b/lib/spack/spack/relocate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,6 +6,7 @@ import os import re +import shutil import platform import spack.repo import spack.cmd @@ -86,7 +87,14 @@ def get_existing_elf_rpaths(path_name): Return the RPATHS returned by patchelf --print-rpath path_name as a list of strings. """ - patchelf = Executable(get_patchelf()) + + # if we're relocating patchelf itself, use it + + if path_name[-13:] == "/bin/patchelf": + patchelf = Executable(path_name) + else: + patchelf = Executable(get_patchelf()) + try: output = patchelf('--print-rpath', '%s' % path_name, output=str, error=str) @@ -326,8 +334,18 @@ def modify_elf_object(path_name, new_rpaths): """ Replace orig_rpath with new_rpath in RPATH of elf object path_name """ + new_joined = ':'.join(new_rpaths) - patchelf = Executable(get_patchelf()) + + # if we're relocating patchelf itself, use it + + if path_name[-13:] == "/bin/patchelf": + bak_path = path_name + ".bak" + shutil.copy(path_name, bak_path) + patchelf = Executable(bak_path) + else: + patchelf = Executable(get_patchelf()) + try: patchelf('--force-rpath', '--set-rpath', '%s' % new_joined, '%s' % path_name, output=str, error=str) @@ -665,7 +683,13 @@ def file_is_relocatable(file, paths_to_relocate=None): raise ValueError('{0} is not an absolute path'.format(file)) strings = Executable('strings') - patchelf = Executable(get_patchelf()) + + # if we're relocating patchelf itself, use it + + if file[-13:] == "/bin/patchelf": + patchelf = Executable(file) + else: + patchelf = Executable(get_patchelf()) # Remove the RPATHS from the strings in the executable set_of_strings = set(strings(file, output=str).split()) diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index 3ea196dd4d..8e3dae5d47 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -20,7 +20,7 @@ import traceback from six import string_types, add_metaclass try: - from collections.abc import Mapping + from collections.abc import Mapping # novm except ImportError: from collections import Mapping @@ -932,7 +932,7 @@ class Repo(object): tty.warn("Patch file did not exist: %s" % patch.path) # Install the package.py file itself. - install(self.filename_for_package_name(spec), path) + install(self.filename_for_package_name(spec.name), path) def purge(self): """Clear entire package instance cache.""" @@ -974,20 +974,12 @@ class Repo(object): def extensions_for(self, extendee_spec): return [p for p in self.all_packages() if p.extends(extendee_spec)] - def _check_namespace(self, spec): - """Check that the spec's namespace is the same as this repository's.""" - if spec.namespace and spec.namespace != self.namespace: - raise UnknownNamespaceError(spec.namespace) - - @autospec - def dirname_for_package_name(self, spec): + def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - self._check_namespace(spec) - return os.path.join(self.packages_path, spec.name) + return os.path.join(self.packages_path, pkg_name) - @autospec - def filename_for_package_name(self, spec): + def filename_for_package_name(self, pkg_name): """Get the filename for the module we should load for a particular package. Packages for a Repo live in ``$root/<package_name>/package.py`` @@ -996,8 +988,7 @@ class Repo(object): package doesn't exist yet, so callers will need to ensure the package exists before importing. """ - self._check_namespace(spec) - pkg_dir = self.dirname_for_package_name(spec.name) + pkg_dir = self.dirname_for_package_name(pkg_name) return os.path.join(pkg_dir, package_file_name) @property diff --git a/lib/spack/spack/report.py b/lib/spack/spack/report.py index 292eeffd8c..04a20f36b8 100644 --- a/lib/spack/spack/report.py +++ b/lib/spack/spack/report.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -146,8 +146,9 @@ class InfoCollector(object): # An InstallError is considered a failure (the recipe # didn't work correctly) package['result'] = 'failure' - package['stdout'] = fetch_package_log(pkg) package['message'] = e.message or 'Installation failure' + package['stdout'] = fetch_package_log(pkg) + package['stdout'] += package['message'] package['exception'] = e.traceback except (Exception, BaseException) as e: diff --git a/lib/spack/spack/reporter.py b/lib/spack/spack/reporter.py index f99380f0c0..25d4042a5e 100644 --- a/lib/spack/spack/reporter.py +++ b/lib/spack/spack/reporter.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/reporters/__init__.py b/lib/spack/spack/reporters/__init__.py index 94f8ac4d9e..9f87532b85 100644 --- a/lib/spack/spack/reporters/__init__.py +++ b/lib/spack/spack/reporters/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py index 58095b10ba..580df7866f 100644 --- a/lib/spack/spack/reporters/cdash.py +++ b/lib/spack/spack/reporters/cdash.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -17,6 +17,7 @@ from six.moves.urllib.request import build_opener, HTTPHandler, Request from six.moves.urllib.parse import urlencode from llnl.util.filesystem import working_dir +import llnl.util.tty as tty from ordereddict_backport import OrderedDict import spack.build_environment import spack.fetch_strategy @@ -58,6 +59,7 @@ class CDash(Reporter): def __init__(self, args): Reporter.__init__(self, args) + tty.set_verbose(args.verbose) self.template_dir = os.path.join('reports', 'cdash') self.cdash_upload_url = args.cdash_upload_url @@ -65,8 +67,13 @@ class CDash(Reporter): self.buildid_regexp = re.compile("<buildId>([0-9]+)</buildId>") self.phase_regexp = re.compile(r"Executing phase: '(.*)'") - if args.package: - packages = args.package + self.authtoken = None + if 'SPACK_CDASH_AUTH_TOKEN' in os.environ: + tty.verbose("Using CDash auth token from environment") + self.authtoken = os.environ.get('SPACK_CDASH_AUTH_TOKEN') + + if args.spec: + packages = args.spec else: packages = [] for file in args.specfiles: @@ -225,7 +232,8 @@ class CDash(Reporter): # from the binary cache. spec['packages'] = [ x for x in spec['packages'] - if not x['installed_from_binary_cache'] + if 'installed_from_binary_cache' not in x or + not x['installed_from_binary_cache'] ] for package in spec['packages']: if 'stdout' in package: @@ -297,12 +305,18 @@ class CDash(Reporter): request = Request(url, data=f) request.add_header('Content-Type', 'text/xml') request.add_header('Content-Length', os.path.getsize(filename)) + if self.authtoken: + request.add_header('Authorization', + 'Bearer {0}'.format(self.authtoken)) # By default, urllib2 only support GET and POST. # CDash needs expects this file to be uploaded via PUT. request.get_method = lambda: 'PUT' response = opener.open(request) if self.current_package_name not in self.buildIds: - match = self.buildid_regexp.search(response.read()) + resp_value = response.read() + if isinstance(resp_value, bytes): + resp_value = resp_value.decode('utf-8') + match = self.buildid_regexp.search(resp_value) if match: buildid = match.group(1) self.buildIds[self.current_package_name] = buildid diff --git a/lib/spack/spack/reporters/junit.py b/lib/spack/spack/reporters/junit.py index d2c11dff4b..6c54c45b42 100644 --- a/lib/spack/spack/reporters/junit.py +++ b/lib/spack/spack/reporters/junit.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index 2786bcd1be..5cec54f6de 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/s3_handler.py b/lib/spack/spack/s3_handler.py index 2a54b9ecb1..24f0a46221 100644 --- a/lib/spack/spack/s3_handler.py +++ b/lib/spack/spack/s3_handler.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,7 +11,6 @@ import six.moves.urllib.error as urllib_error import spack.util.s3 as s3_util import spack.util.url as url_util -import spack.util.web as web_util # NOTE(opadron): Workaround issue in boto where its StreamingBody @@ -54,8 +53,7 @@ def _s3_open(url): # NOTE(opadron): Apply workaround here (see above) stream = WrapStream(obj['Body']) - headers = web_util.standardize_header_names( - obj['ResponseMetadata']['HTTPHeaders']) + headers = obj['ResponseMetadata']['HTTPHeaders'] return url, headers, stream diff --git a/lib/spack/spack/schema/__init__.py b/lib/spack/spack/schema/__init__.py index 894d1cd743..755e3d9086 100644 --- a/lib/spack/spack/schema/__init__.py +++ b/lib/spack/spack/schema/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/cdash.py b/lib/spack/spack/schema/cdash.py index 14a24c5f41..2df2ea3f32 100644 --- a/lib/spack/spack/schema/cdash.py +++ b/lib/spack/spack/schema/cdash.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/compilers.py b/lib/spack/spack/schema/compilers.py index c2f72c89d1..c994bef819 100644 --- a/lib/spack/spack/schema/compilers.py +++ b/lib/spack/spack/schema/compilers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,7 +8,7 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/compilers.py :lines: 13- """ - +import spack.schema.environment #: Properties for inclusion in other schemas properties = { @@ -68,50 +68,7 @@ properties = { {'type': 'boolean'} ] }, - 'environment': { - 'type': 'object', - 'default': {}, - 'additionalProperties': False, - 'properties': { - 'set': { - 'type': 'object', - 'patternProperties': { - # Variable name - r'\w[\w-]*': { - 'anyOf': [{'type': 'string'}, - {'type': 'number'}] - } - } - }, - 'unset': { - 'type': 'object', - 'patternProperties': { - # Variable name - r'\w[\w-]*': {'type': 'null'} - } - }, - 'prepend-path': { - 'type': 'object', - 'patternProperties': { - # Variable name - r'\w[\w-]*': { - 'anyOf': [{'type': 'string'}, - {'type': 'number'}] - } - } - }, - 'append-path': { - 'type': 'object', - 'patternProperties': { - # Variable name - r'\w[\w-]*': { - 'anyOf': [{'type': 'string'}, - {'type': 'number'}] - } - } - } - } - }, + 'environment': spack.schema.environment.definition, 'extra_rpaths': { 'type': 'array', 'default': [], diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 7d170bbc91..1378698825 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py new file mode 100644 index 0000000000..cb1ed8d63a --- /dev/null +++ b/lib/spack/spack/schema/container.py @@ -0,0 +1,82 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Schema for the 'container' subsection of Spack environments.""" + +#: Schema for the container attribute included in Spack environments +container_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + # The recipe formats that are currently supported by the command + 'format': { + 'type': 'string', + 'enum': ['docker', 'singularity'] + }, + # Describes the base image to start from and the version + # of Spack to be used + 'base': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'image': { + 'type': 'string', + 'enum': ['ubuntu:18.04', + 'ubuntu:16.04', + 'centos:7', + 'centos:6'] + }, + 'spack': { + 'type': 'string', + 'enum': ['develop', '0.14', '0.14.0'] + } + }, + 'required': ['image', 'spack'] + }, + # Whether or not to strip installed binaries + 'strip': { + 'type': 'boolean', + 'default': True + }, + # Additional system packages that are needed at runtime + 'os_packages': { + 'type': 'array', + 'items': { + 'type': 'string' + } + }, + # Add labels to the image + 'labels': { + 'type': 'object', + }, + # Add a custom extra section at the bottom of a stage + 'extra_instructions': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'build': {'type': 'string'}, + 'final': {'type': 'string'} + } + }, + # Reserved for properties that are specific to each format + 'singularity': { + 'type': 'object', + 'additionalProperties': False, + 'default': {}, + 'properties': { + 'runscript': {'type': 'string'}, + 'startscript': {'type': 'string'}, + 'test': {'type': 'string'}, + 'help': {'type': 'string'} + } + }, + 'docker': { + 'type': 'object', + 'additionalProperties': False, + 'default': {}, + } + } +} + +properties = {'container': container_schema} diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index dd3093d05d..907abc5c9a 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/environment.py b/lib/spack/spack/schema/environment.py new file mode 100644 index 0000000000..d251aeba96 --- /dev/null +++ b/lib/spack/spack/schema/environment.py @@ -0,0 +1,58 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Schema for environment modifications. Meant for inclusion in other +schemas. +""" + +array_of_strings_or_num = { + 'type': 'array', 'default': [], 'items': + {'anyOf': [{'type': 'string'}, {'type': 'number'}]} +} + +dictionary_of_strings_or_num = { + 'type': 'object', 'patternProperties': + {r'\w[\w-]*': {'anyOf': [{'type': 'string'}, {'type': 'number'}]}} +} + +definition = { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'set': dictionary_of_strings_or_num, + 'unset': array_of_strings_or_num, + 'prepend_path': dictionary_of_strings_or_num, + 'append_path': dictionary_of_strings_or_num, + 'remove_path': dictionary_of_strings_or_num + } +} + + +def parse(config_obj): + """Returns an EnvironmentModifications object containing the modifications + parsed from input. + + Args: + config_obj: a configuration dictionary conforming to the + schema definition for environment modifications + """ + import spack.util.environment as ev + try: + from collections import Sequence # novm + except ImportError: + from collections.abc import Sequence # novm + + env = ev.EnvironmentModifications() + for command, variable in config_obj.items(): + # Distinguish between commands that take only a name as argument + # (e.g. unset) and commands that take a name and a value. + if isinstance(variable, Sequence): + for name in variable: + getattr(env, command)(name) + else: + for name, value in variable.items(): + getattr(env, command)(name, value) + + return env diff --git a/lib/spack/spack/schema/gitlab_ci.py b/lib/spack/spack/schema/gitlab_ci.py index dc7b16fb21..9d0ba8a77b 100644 --- a/lib/spack/spack/schema/gitlab_ci.py +++ b/lib/spack/spack/schema/gitlab_ci.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,6 +9,24 @@ :lines: 13- """ +image_schema = { + 'oneOf': [ + { + 'type': 'string' + }, { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'entrypoint': { + 'type': 'array', + 'items': { + 'type': 'string', + }, + }, + }, + }, + ], +} #: Properties for inclusion in other schemas properties = { @@ -58,24 +76,7 @@ properties = { 'additionalProperties': True, 'required': ['tags'], 'properties': { - 'image': { - 'oneOf': [ - { - 'type': 'string' - }, { - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'entrypoint': { - 'type': 'array', - 'items': { - 'type': 'string', - }, - }, - }, - }, - ], - }, + 'image': image_schema, 'tags': { 'type': 'array', 'default': [], @@ -95,6 +96,27 @@ properties = { }, }, }, + 'enable-artifacts-buildcache': { + 'type': 'boolean', + 'default': False, + }, + 'enable-debug-messages': { + 'type': 'boolean', + 'default': False, + }, + 'final-stage-rebuild-index': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['tags'], + 'properties': { + 'image': image_schema, + 'tags': { + 'type': 'array', + 'default': [], + 'items': {'type': 'string'} + }, + }, + }, }, }, } diff --git a/lib/spack/spack/schema/merged.py b/lib/spack/spack/schema/merged.py index 76f8ffa2d5..e118acf286 100644 --- a/lib/spack/spack/schema/merged.py +++ b/lib/spack/spack/schema/merged.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -13,6 +13,7 @@ from llnl.util.lang import union_dicts import spack.schema.cdash import spack.schema.compilers import spack.schema.config +import spack.schema.container import spack.schema.gitlab_ci import spack.schema.mirrors import spack.schema.modules @@ -26,6 +27,7 @@ properties = union_dicts( spack.schema.cdash.properties, spack.schema.compilers.properties, spack.schema.config.properties, + spack.schema.container.properties, spack.schema.gitlab_ci.properties, spack.schema.mirrors.properties, spack.schema.modules.properties, diff --git a/lib/spack/spack/schema/mirrors.py b/lib/spack/spack/schema/mirrors.py index 92e6c9bca1..5bb641df05 100644 --- a/lib/spack/spack/schema/mirrors.py +++ b/lib/spack/spack/schema/mirrors.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/modules.py b/lib/spack/spack/schema/modules.py index 14f185cfcc..1fbbf614c8 100644 --- a/lib/spack/spack/schema/modules.py +++ b/lib/spack/spack/schema/modules.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,6 +8,8 @@ .. literalinclude:: _spack_root/lib/spack/spack/schema/modules.py :lines: 13- """ +import spack.schema.environment + #: Matches a spec or a multi-valued variant but not another #: valid keyword. @@ -66,17 +68,7 @@ module_file_configuration = { } } }, - 'environment': { - 'type': 'object', - 'default': {}, - 'additionalProperties': False, - 'properties': { - 'set': dictionary_of_strings, - 'unset': array_of_strings, - 'prepend_path': dictionary_of_strings, - 'append_path': dictionary_of_strings - } - } + 'environment': spack.schema.environment.definition } } diff --git a/lib/spack/spack/schema/packages.py b/lib/spack/spack/schema/packages.py index f97d82d83a..4984471c73 100644 --- a/lib/spack/spack/schema/packages.py +++ b/lib/spack/spack/schema/packages.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/projections.py b/lib/spack/spack/schema/projections.py index f076ed473b..d6f506f090 100644 --- a/lib/spack/spack/schema/projections.py +++ b/lib/spack/spack/schema/projections.py @@ -1,4 +1,4 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/repos.py b/lib/spack/spack/schema/repos.py index fe74942872..6aa9bcf57e 100644 --- a/lib/spack/spack/schema/repos.py +++ b/lib/spack/spack/schema/repos.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/schema/upstreams.py b/lib/spack/spack/schema/upstreams.py index 5925782af2..f5192aa096 100644 --- a/lib/spack/spack/schema/upstreams.py +++ b/lib/spack/spack/schema/upstreams.py @@ -1,4 +1,4 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f983e5c559..4b6bd67b6d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -81,18 +81,16 @@ import sys import collections import hashlib import itertools +import operator import os import re import six - -from operator import attrgetter import ruamel.yaml as yaml -from llnl.util.filesystem import find_headers, find_libraries, is_exe -from llnl.util.lang import key_ordering, HashableMap, ObjectWrapper, dedupe -from llnl.util.lang import check_kwargs, memoized -from llnl.util.tty.color import cwrite, colorize, cescape, get_color_when +import llnl.util.filesystem as fs +import llnl.util.lang as lang +import llnl.util.tty.color as clr import llnl.util.tty as tty import spack.paths @@ -103,48 +101,34 @@ import spack.dependency as dp import spack.error import spack.hash_types as ht import spack.parse +import spack.provider_index import spack.repo import spack.store +import spack.util.crypto +import spack.util.executable +import spack.util.module_cmd as md +import spack.util.prefix import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml - -from spack.util.module_cmd import get_path_from_module, load_module -from spack.error import NoLibrariesError, NoHeadersError -from spack.error import SpecError, UnsatisfiableSpecError -from spack.provider_index import ProviderIndex -from spack.util.crypto import prefix_bits -from spack.util.executable import Executable -from spack.util.prefix import Prefix -from spack.util.spack_yaml import syaml_dict -from spack.util.string import comma_or -from spack.variant import MultiValuedVariant, AbstractVariant -from spack.variant import BoolValuedVariant, substitute_abstract_variants -from spack.variant import VariantMap, UnknownVariantError -from spack.variant import DuplicateVariantError -from spack.variant import UnsatisfiableVariantSpecError -from spack.version import VersionList, VersionRange, Version, ver -from ruamel.yaml.error import MarkedYAMLError +import spack.util.string +import spack.variant as vt +import spack.version as vn __all__ = [ 'Spec', 'parse', - 'SpecError', 'SpecParseError', 'DuplicateDependencyError', - 'DuplicateVariantError', 'DuplicateCompilerSpecError', 'UnsupportedCompilerError', - 'UnknownVariantError', 'DuplicateArchitectureError', 'InconsistentSpecError', 'InvalidDependencyError', 'NoProviderError', 'MultipleProviderError', - 'UnsatisfiableSpecError', 'UnsatisfiableSpecNameError', 'UnsatisfiableVersionSpecError', 'UnsatisfiableCompilerSpecError', - 'UnsatisfiableVariantSpecError', 'UnsatisfiableCompilerFlagSpecError', 'UnsatisfiableArchitectureSpecError', 'UnsatisfiableProviderSpecError', @@ -183,7 +167,7 @@ _separators = '[\\%s]' % '\\'.join(color_formats.keys()) #: Versionlist constant so we don't have to build a list #: every time we call str() -_any_version = VersionList([':']) +_any_version = vn.VersionList([':']) default_format = '{name}{@version}' default_format += '{%compiler.name}{@compiler.version}{compiler_flags}' @@ -202,15 +186,15 @@ def colorize_spec(spec): # ignore compiler versions (color same as compiler) sep = match.group(0) if self.last == '%' and sep == '@': - return cescape(sep) + return clr.cescape(sep) self.last = sep - return '%s%s' % (color_formats[sep], cescape(sep)) + return '%s%s' % (color_formats[sep], clr.cescape(sep)) - return colorize(re.sub(_separators, insert_color(), str(spec)) + '@.') + return clr.colorize(re.sub(_separators, insert_color(), str(spec)) + '@.') -@key_ordering +@lang.key_ordering class ArchSpec(object): def __init__(self, spec_or_platform_tuple=(None, None, None)): """ Architecture specification a package should be built with. @@ -444,11 +428,11 @@ class ArchSpec(object): return self.platform and self.os and self.target def to_dict(self): - d = syaml_dict([ + d = syaml.syaml_dict([ ('platform', self.platform), ('platform_os', self.os), ('target', self.target.to_dict_or_value())]) - return syaml_dict([('arch', d)]) + return syaml.syaml_dict([('arch', d)]) @staticmethod def from_dict(d): @@ -487,7 +471,7 @@ class ArchSpec(object): return string in str(self) or string in self.target -@key_ordering +@lang.key_ordering class CompilerSpec(object): """The CompilerSpec field represents the compiler or range of compiler versions that a package should be built with. CompilerSpecs have a @@ -516,8 +500,8 @@ class CompilerSpec(object): elif nargs == 2: name, version = args self.name = name - self.versions = VersionList() - self.versions.add(ver(version)) + self.versions = vn.VersionList() + self.versions.add(vn.ver(version)) else: raise TypeError( @@ -558,7 +542,7 @@ class CompilerSpec(object): @property def version(self): if not self.concrete: - raise SpecError("Spec is not concrete: " + str(self)) + raise spack.error.SpecError("Spec is not concrete: " + str(self)) return self.versions[0] def copy(self): @@ -571,15 +555,15 @@ class CompilerSpec(object): return (self.name, self.versions) def to_dict(self): - d = syaml_dict([('name', self.name)]) + d = syaml.syaml_dict([('name', self.name)]) d.update(self.versions.to_dict()) - return syaml_dict([('compiler', d)]) + return syaml.syaml_dict([('compiler', d)]) @staticmethod def from_dict(d): d = d['compiler'] - return CompilerSpec(d['name'], VersionList.from_dict(d)) + return CompilerSpec(d['name'], vn.VersionList.from_dict(d)) def __str__(self): out = self.name @@ -592,7 +576,7 @@ class CompilerSpec(object): return str(self) -@key_ordering +@lang.key_ordering class DependencySpec(object): """DependencySpecs connect two nodes in the DAG, and contain deptypes. @@ -640,7 +624,7 @@ _valid_compiler_flags = [ 'cflags', 'cxxflags', 'fflags', 'ldflags', 'ldlibs', 'cppflags'] -class FlagMap(HashableMap): +class FlagMap(lang.HashableMap): def __init__(self, spec): super(FlagMap, self).__init__() @@ -698,7 +682,7 @@ class FlagMap(HashableMap): for key in sorted_keys) + cond_symbol -class DependencyMap(HashableMap): +class DependencyMap(lang.HashableMap): """Each spec has a DependencyMap containing specs for its dependencies. The DependencyMap is keyed by name. """ @@ -725,8 +709,8 @@ def _command_default_handler(descriptor, spec, cls): """ path = os.path.join(spec.prefix.bin, spec.name) - if is_exe(path): - return Executable(path) + if fs.is_exe(path): + return spack.util.executable.Executable(path) else: msg = 'Unable to locate {0} command in {1}' raise RuntimeError(msg.format(spec.name, spec.prefix.bin)) @@ -750,13 +734,14 @@ def _headers_default_handler(descriptor, spec, cls): Raises: NoHeadersError: If no headers are found """ - headers = find_headers('*', root=spec.prefix.include, recursive=True) + headers = fs.find_headers('*', root=spec.prefix.include, recursive=True) if headers: return headers else: msg = 'Unable to locate {0} headers in {1}' - raise NoHeadersError(msg.format(spec.name, spec.prefix.include)) + raise spack.error.NoHeadersError( + msg.format(spec.name, spec.prefix.include)) def _libs_default_handler(descriptor, spec, cls): @@ -801,12 +786,13 @@ def _libs_default_handler(descriptor, spec, cls): ([False] if ('~shared' in spec) else [True, False]) for shared in search_shared: - libs = find_libraries(name, spec.prefix, shared=shared, recursive=True) + libs = fs.find_libraries( + name, spec.prefix, shared=shared, recursive=True) if libs: return libs msg = 'Unable to recursively locate {0} libraries in {1}' - raise NoLibrariesError(msg.format(spec.name, spec.prefix)) + raise spack.error.NoLibrariesError(msg.format(spec.name, spec.prefix)) class ForwardQueryToPackage(object): @@ -920,7 +906,7 @@ class ForwardQueryToPackage(object): raise AttributeError(msg.format(cls_name, self.attribute_name)) -class SpecBuildInterface(ObjectWrapper): +class SpecBuildInterface(lang.ObjectWrapper): command = ForwardQueryToPackage( 'command', default_handler=_command_default_handler @@ -952,7 +938,7 @@ class SpecBuildInterface(ObjectWrapper): ) -@key_ordering +@lang.key_ordering class Spec(object): #: Cache for spec's prefix, computed lazily in the corresponding property @@ -985,8 +971,8 @@ class Spec(object): # init an empty spec that matches anything. self.name = None - self.versions = VersionList(':') - self.variants = VariantMap(self) + self.versions = vn.VersionList(':') + self.variants = vt.VariantMap(self) self.architecture = None self.compiler = None self.external_path = None @@ -1088,9 +1074,9 @@ class Spec(object): # map to '+foo' and '~foo' respectively. As such they need a # BoolValuedVariant instance. if str(value).upper() == 'TRUE' or str(value).upper() == 'FALSE': - self.variants[name] = BoolValuedVariant(name, value) + self.variants[name] = vt.BoolValuedVariant(name, value) else: - self.variants[name] = AbstractVariant(name, value) + self.variants[name] = vt.AbstractVariant(name, value) def _set_architecture(self, **kwargs): """Called by the parser to set the architecture.""" @@ -1131,6 +1117,18 @@ class Spec(object): self._dependencies[spec.name] = dspec spec._dependents[self.name] = dspec + def _add_default_platform(self): + """If a spec has an os or a target and no platform, give it + the default platform. + + This is private because it is used by the parser -- it's not + expected to be used outside of ``spec.py``. + + """ + arch = self.architecture + if arch and not arch.platform and (arch.os or arch.target): + self._set_architecture(platform=spack.architecture.platform().name) + # # Public interface # @@ -1256,7 +1254,7 @@ class Spec(object): depth = kwargs.get('depth', False) key_fun = kwargs.get('key', id) if isinstance(key_fun, six.string_types): - key_fun = attrgetter(key_fun) + key_fun = operator.attrgetter(key_fun) yield_root = kwargs.get('root', True) cover = kwargs.get('cover', 'nodes') direction = kwargs.get('direction', 'children') @@ -1340,7 +1338,7 @@ class Spec(object): @property def prefix(self): if not self._concrete: - raise SpecError("Spec is not concrete: " + str(self)) + raise spack.error.SpecError("Spec is not concrete: " + str(self)) if self._prefix is None: upstream, record = spack.store.db.query_by_spec_hash( @@ -1353,7 +1351,7 @@ class Spec(object): @prefix.setter def prefix(self, value): - self._prefix = Prefix(value) + self._prefix = spack.util.prefix.Prefix(value) def _spec_hash(self, hash): """Utility method for computing different types of Spec hashes. @@ -1481,7 +1479,7 @@ class Spec(object): Arguments: hash (SpecHashDescriptor) type of hash to generate. """ - d = syaml_dict() + d = syaml.syaml_dict() if self.versions: d.update(self.versions.to_dict()) @@ -1495,7 +1493,7 @@ class Spec(object): if self.namespace: d['namespace'] = self.namespace - params = syaml_dict( + params = syaml.syaml_dict( sorted( v.yaml_entry() for _, v in self.variants.items() ) @@ -1524,15 +1522,15 @@ class Spec(object): deps = self.dependencies_dict(deptype=hash.deptype) if deps: - d['dependencies'] = syaml_dict([ + d['dependencies'] = syaml.syaml_dict([ (name, - syaml_dict([ + syaml.syaml_dict([ ('hash', dspec.spec._cached_hash(hash)), ('type', sorted(str(s) for s in dspec.deptypes))]) ) for name, dspec in sorted(deps.items()) ]) - return syaml_dict([(self.name, d)]) + return syaml.syaml_dict([(self.name, d)]) def to_dict(self, hash=ht.dag_hash): """Create a dictionary suitable for writing this spec to YAML or JSON. @@ -1607,7 +1605,7 @@ class Spec(object): node[s.name]['build_hash'] = s.build_hash() node_list.append(node) - return syaml_dict([('spec', node_list)]) + return syaml.syaml_dict([('spec', node_list)]) def to_record_dict(self): """Return a "flat" dictionary with name and hash as top-level keys. @@ -1628,7 +1626,7 @@ class Spec(object): But is otherwise the same as ``to_node_dict()``. """ - dictionary = syaml_dict() + dictionary = syaml.syaml_dict() dictionary["name"] = self.name dictionary["hash"] = self.dag_hash() dictionary.update(self.to_node_dict()[self.name]) @@ -1652,7 +1650,7 @@ class Spec(object): spec._build_hash = node.get('build_hash', None) if 'version' in node or 'versions' in node: - spec.versions = VersionList.from_dict(node) + spec.versions = vn.VersionList.from_dict(node) if 'arch' in node: spec.architecture = ArchSpec.from_dict(node) @@ -1667,11 +1665,11 @@ class Spec(object): if name in _valid_compiler_flags: spec.compiler_flags[name] = value else: - spec.variants[name] = MultiValuedVariant.from_node_dict( + spec.variants[name] = vt.MultiValuedVariant.from_node_dict( name, value) elif 'variants' in node: for name, value in node['variants'].items(): - spec.variants[name] = MultiValuedVariant.from_node_dict( + spec.variants[name] = vt.MultiValuedVariant.from_node_dict( name, value ) for name in FlagMap.valid_compiler_flags(): @@ -1700,7 +1698,7 @@ class Spec(object): patches = node['patches'] if len(patches) > 0: mvar = spec.variants.setdefault( - 'patches', MultiValuedVariant('patches', ()) + 'patches', vt.MultiValuedVariant('patches', ()) ) mvar.value = patches # FIXME: Monkey patches mvar to store patches order @@ -1738,7 +1736,8 @@ class Spec(object): # new format: elements of dependency spec are keyed. dag_hash, deptypes = elt['hash'], elt['type'] else: - raise SpecError("Couldn't parse dependency types in spec.") + raise spack.error.SpecError( + "Couldn't parse dependency types in spec.") yield dep_name, dag_hash, list(deptypes) @@ -1907,7 +1906,7 @@ class Spec(object): # dependencies are the following elements. dep_list = [Spec.from_node_dict(node) for node in nodes] if not dep_list: - raise SpecError("YAML spec contains no nodes.") + raise spack.error.SpecError("YAML spec contains no nodes.") deps = dict((spec.name, spec) for spec in dep_list) spec = dep_list[0] @@ -1936,7 +1935,7 @@ class Spec(object): try: data = yaml.load(stream) return Spec.from_dict(data) - except MarkedYAMLError as e: + except yaml.error.MarkedYAMLError as e: raise syaml.SpackYAMLError("error parsing YAML spec:", str(e)) @staticmethod @@ -2032,7 +2031,8 @@ class Spec(object): a problem. """ # Make an index of stuff this spec already provides - self_index = ProviderIndex(self.traverse(), restrict=True) + self_index = spack.provider_index.ProviderIndex( + self.traverse(), restrict=True) changed = False done = False @@ -2072,7 +2072,7 @@ class Spec(object): # constraints. copy.normalize(force=True) break - except SpecError: + except spack.error.SpecError: # On error, we'll try the next replacement. continue @@ -2133,7 +2133,8 @@ class Spec(object): normalize() for more details on this. """ if not self.name: - raise SpecError("Attempting to concretize anonymous spec") + raise spack.error.SpecError( + "Attempting to concretize anonymous spec") if self._concrete: return @@ -2182,7 +2183,7 @@ class Spec(object): # Add any patches from the package to the spec. patches = [] for cond, patch_list in s.package_class.patches.items(): - if s.satisfies(cond): + if s.satisfies(cond, strict=True): for patch in patch_list: patches.append(patch) if patches: @@ -2201,7 +2202,7 @@ class Spec(object): patches = [] for cond, dependency in pkg_deps[dspec.spec.name].items(): - if dspec.parent.satisfies(cond): + if dspec.parent.satisfies(cond, strict=True): for pcond, patch_list in dependency.patches.items(): if dspec.spec.satisfies(pcond): for patch in patch_list: @@ -2214,9 +2215,9 @@ class Spec(object): if id(spec) not in spec_to_patches: continue - patches = list(dedupe(spec_to_patches[id(spec)])) + patches = list(lang.dedupe(spec_to_patches[id(spec)])) mvar = spec.variants.setdefault( - 'patches', MultiValuedVariant('patches', ()) + 'patches', vt.MultiValuedVariant('patches', ()) ) mvar.value = tuple(p.sha256 for p in patches) # FIXME: Monkey patches mvar to store patches order @@ -2234,9 +2235,9 @@ class Spec(object): compiler = spack.compilers.compiler_for_spec( s.compiler, s.architecture) for mod in compiler.modules: - load_module(mod) + md.load_module(mod) - s.external_path = get_path_from_module(s.external_module) + s.external_path = md.get_path_from_module(s.external_module) # Mark everything in the spec as concrete, as well. self._mark_concrete() @@ -2336,7 +2337,7 @@ class Spec(object): return flat_deps - except UnsatisfiableSpecError as e: + except spack.error.UnsatisfiableSpecError as e: # Here, the DAG contains two instances of the same package # with inconsistent constraints. Users cannot produce # inconsistent specs like this on the command line: the @@ -2369,7 +2370,7 @@ class Spec(object): """ conditions = self.package_class.dependencies[name] - substitute_abstract_variants(self) + vt.substitute_abstract_variants(self) # evaluate when specs to figure out constraints on the dependency. dep = None for when_spec, dependency in conditions.items(): @@ -2378,7 +2379,7 @@ class Spec(object): dep = dp.Dependency(self.name, Spec(name), type=()) try: dep.merge(dependency) - except UnsatisfiableSpecError as e: + except spack.error.UnsatisfiableSpecError as e: e.message = ( "Conflicting conditional dependencies for spec" "\n\n\t{0}\n\n" @@ -2468,7 +2469,7 @@ class Spec(object): if provider: dep = provider else: - index = ProviderIndex([dep], restrict=True) + index = spack.provider_index.ProviderIndex([dep], restrict=True) items = list(spec_deps.items()) for name, vspec in items: if not vspec.virtual: @@ -2498,7 +2499,7 @@ class Spec(object): # merge package/vdep information into spec try: changed |= spec_deps[dep.name].constrain(dep) - except UnsatisfiableSpecError as e: + except spack.error.UnsatisfiableSpecError as e: fmt = 'An unsatisfiable {0}'.format(e.constraint_type) fmt += ' constraint has been detected for spec:' fmt += '\n\n{0}\n\n'.format(spec_deps[dep.name].tree(indent=4)) @@ -2585,7 +2586,8 @@ class Spec(object): detection, to ensure that the spec is actually a DAG. """ if not self.name: - raise SpecError("Attempting to normalize anonymous spec") + raise spack.error.SpecError( + "Attempting to normalize anonymous spec") # Set _normal and _concrete to False when forced if force: @@ -2615,7 +2617,7 @@ class Spec(object): # Initialize index of virtual dependency providers if # concretize didn't pass us one already - provider_index = ProviderIndex( + provider_index = spack.provider_index.ProviderIndex( [s for s in all_spec_deps.values()], restrict=True) # traverse the package DAG and fill out dependencies according @@ -2663,9 +2665,9 @@ class Spec(object): not_existing = set(spec.variants) - ( set(pkg_variants) | set(spack.directives.reserved_names)) if not_existing: - raise UnknownVariantError(spec.name, not_existing) + raise vt.UnknownVariantError(spec, not_existing) - substitute_abstract_variants(spec) + vt.substitute_abstract_variants(spec) def constrain(self, other, deps=True): """Merge the constraints of other with self. @@ -2679,7 +2681,7 @@ class Spec(object): if self.satisfies(other): return False else: - raise UnsatisfiableSpecError( + raise spack.error.UnsatisfiableSpecError( self, other, 'constrain a concrete spec' ) @@ -2700,7 +2702,7 @@ class Spec(object): for v in [x for x in other.variants if x in self.variants]: if not self.variants[v].compatible(other.variants[v]): - raise UnsatisfiableVariantSpecError( + raise vt.UnsatisfiableVariantSpecError( self.variants[v], other.variants[v] ) @@ -2928,8 +2930,10 @@ class Spec(object): return False # For virtual dependencies, we need to dig a little deeper. - self_index = ProviderIndex(self.traverse(), restrict=True) - other_index = ProviderIndex(other.traverse(), restrict=True) + self_index = spack.provider_index.ProviderIndex( + self.traverse(), restrict=True) + other_index = spack.provider_index.ProviderIndex( + other.traverse(), restrict=True) # This handles cases where there are already providers for both vpkgs if not self_index.satisfies(other_index): @@ -2954,7 +2958,7 @@ class Spec(object): return [spec for spec in self.traverse() if spec.virtual] @property - @memoized + @lang.memoized def patches(self): """Return patch objects for any patch sha256 sums on this Spec. @@ -2965,7 +2969,7 @@ class Spec(object): patches from install directories, but it probably should. """ if not self.concrete: - raise SpecError("Spec is not concrete: " + str(self)) + raise spack.error.SpecError("Spec is not concrete: " + str(self)) if 'patches' not in self.variants: return [] @@ -3135,7 +3139,8 @@ class Spec(object): @property def version(self): if not self.versions.concrete: - raise SpecError("Spec version is not concrete: " + str(self)) + raise spack.error.SpecError( + "Spec version is not concrete: " + str(self)) return self.versions[0] def __getitem__(self, name): @@ -3375,10 +3380,10 @@ class Spec(object): out = six.StringIO() def write(s, c=None): - f = cescape(s) + f = clr.cescape(s) if c is not None: f = color_formats[c] + f + '@.' - cwrite(f, stream=out, color=color) + clr.cwrite(f, stream=out, color=color) def write_attribute(spec, attribute, color): current = spec @@ -3445,7 +3450,7 @@ class Spec(object): 'Attempted to format private attribute' ) else: - if isinstance(current, VariantMap): + if isinstance(current, vt.VariantMap): # subscript instead of getattr for variant names current = current[part] else: @@ -3463,7 +3468,7 @@ class Spec(object): m = 'Attempted to format attribute %s.' % attribute m += 'Spec.%s has no attribute %s' % (parent, part) raise SpecFormatStringError(m) - if isinstance(current, VersionList): + if isinstance(current, vn.VersionList): if current == _any_version: # We don't print empty version lists return @@ -3609,10 +3614,10 @@ class Spec(object): named_str = fmt = '' def write(s, c=None): - f = cescape(s) + f = clr.cescape(s) if c is not None: f = color_formats[c] + f + '@.' - cwrite(f, stream=out, color=color) + clr.cwrite(f, stream=out, color=color) iterator = enumerate(format_string) for i, c in iterator: @@ -3807,7 +3812,7 @@ class Spec(object): def tree(self, **kwargs): """Prints out this spec and its dependencies, tree-formatted with indentation.""" - color = kwargs.pop('color', get_color_when()) + color = kwargs.pop('color', clr.get_color_when()) depth = kwargs.pop('depth', False) hashes = kwargs.pop('hashes', False) hlen = kwargs.pop('hashlen', None) @@ -3819,7 +3824,7 @@ class Spec(object): show_types = kwargs.pop('show_types', False) deptypes = kwargs.pop('deptypes', 'all') recurse_dependencies = kwargs.pop('recurse_dependencies', True) - check_kwargs(kwargs, self.tree) + lang.check_kwargs(kwargs, self.tree) out = "" for d, dep_spec in self.traverse_edges( @@ -3836,16 +3841,17 @@ class Spec(object): if status_fn: status = status_fn(node) if node.package.installed_upstream: - out += colorize("@g{[^]} ", color=color) + out += clr.colorize("@g{[^]} ", color=color) elif status is None: - out += colorize("@K{ - } ", color=color) # not installed + out += clr.colorize("@K{ - } ", color=color) # !installed elif status: - out += colorize("@g{[+]} ", color=color) # installed + out += clr.colorize("@g{[+]} ", color=color) # installed else: - out += colorize("@r{[-]} ", color=color) # missing + out += clr.colorize("@r{[-]} ", color=color) # missing if hashes: - out += colorize('@K{%s} ', color=color) % node.dag_hash(hlen) + out += clr.colorize( + '@K{%s} ', color=color) % node.dag_hash(hlen) if show_types: types = set() @@ -4059,14 +4065,6 @@ class SpecParser(spack.parse.Parser): except spack.parse.ParseError as e: raise SpecParseError(e) - # If the spec has an os or a target and no platform, give it - # the default platform - platform_default = spack.architecture.platform().name - for spec in specs: - for s in spec.traverse(): - if s.architecture and not s.architecture.platform and \ - (s.architecture.os or s.architecture.target): - s._set_architecture(platform=platform_default) return specs def spec_from_file(self): @@ -4157,17 +4155,17 @@ class SpecParser(spack.parse.Parser): while self.next: if self.accept(AT): vlist = self.version_list() - spec.versions = VersionList() + spec.versions = vn.VersionList() for version in vlist: spec._add_version(version) elif self.accept(ON): name = self.variant() - spec.variants[name] = BoolValuedVariant(name, True) + spec.variants[name] = vt.BoolValuedVariant(name, True) elif self.accept(OFF): name = self.variant() - spec.variants[name] = BoolValuedVariant(name, False) + spec.variants[name] = vt.BoolValuedVariant(name, False) elif self.accept(PCT): spec._set_compiler(self.compiler()) @@ -4198,6 +4196,7 @@ class SpecParser(spack.parse.Parser): else: break + spec._add_default_platform() return spec def variant(self, name=None): @@ -4223,16 +4222,16 @@ class SpecParser(spack.parse.Parser): end = self.token.value elif start: # No colon, but there was a version. - return Version(start) + return vn.Version(start) else: # No colon and no id: invalid version. self.next_token_error("Invalid version specifier") if start: - start = Version(start) + start = vn.Version(start) if end: - end = Version(end) - return VersionRange(start, end) + end = vn.Version(end) + return vn.VersionRange(start, end) def version_list(self): vlist = [] @@ -4247,13 +4246,13 @@ class SpecParser(spack.parse.Parser): compiler = CompilerSpec.__new__(CompilerSpec) compiler.name = self.token.value - compiler.versions = VersionList() + compiler.versions = vn.VersionList() if self.accept(AT): vlist = self.version_list() for version in vlist: compiler._add_version(version) else: - compiler.versions = VersionList(':') + compiler.versions = vn.VersionList(':') return compiler def check_identifier(self, id=None): @@ -4309,10 +4308,10 @@ def base32_prefix_bits(hash_string, bits): % (bits, hash_string)) hash_bytes = base64.b32decode(hash_string, casefold=True) - return prefix_bits(hash_bytes, bits) + return spack.util.crypto.prefix_bits(hash_bytes, bits) -class SpecParseError(SpecError): +class SpecParseError(spack.error.SpecError): """Wrapper for ParseError for when we're parsing specs.""" def __init__(self, parse_error): super(SpecParseError, self).__init__(parse_error.message) @@ -4320,40 +4319,41 @@ class SpecParseError(SpecError): self.pos = parse_error.pos -class DuplicateDependencyError(SpecError): +class DuplicateDependencyError(spack.error.SpecError): """Raised when the same dependency occurs in a spec twice.""" -class DuplicateCompilerSpecError(SpecError): +class DuplicateCompilerSpecError(spack.error.SpecError): """Raised when the same compiler occurs in a spec twice.""" -class UnsupportedCompilerError(SpecError): +class UnsupportedCompilerError(spack.error.SpecError): """Raised when the user asks for a compiler spack doesn't know about.""" def __init__(self, compiler_name): super(UnsupportedCompilerError, self).__init__( "The '%s' compiler is not yet supported." % compiler_name) -class DuplicateArchitectureError(SpecError): +class DuplicateArchitectureError(spack.error.SpecError): """Raised when the same architecture occurs in a spec twice.""" -class InconsistentSpecError(SpecError): +class InconsistentSpecError(spack.error.SpecError): """Raised when two nodes in the same spec DAG have inconsistent constraints.""" -class InvalidDependencyError(SpecError): +class InvalidDependencyError(spack.error.SpecError): """Raised when a dependency in a spec is not actually a dependency of the package.""" def __init__(self, pkg, deps): self.invalid_deps = deps super(InvalidDependencyError, self).__init__( - 'Package {0} does not depend on {1}'.format(pkg, comma_or(deps))) + 'Package {0} does not depend on {1}'.format( + pkg, spack.util.string.comma_or(deps))) -class NoProviderError(SpecError): +class NoProviderError(spack.error.SpecError): """Raised when there is no package that provides a particular virtual dependency. """ @@ -4363,7 +4363,7 @@ class NoProviderError(SpecError): self.vpkg = vpkg -class MultipleProviderError(SpecError): +class MultipleProviderError(spack.error.SpecError): """Raised when there is no package that provides a particular virtual dependency. """ @@ -4376,42 +4376,42 @@ class MultipleProviderError(SpecError): self.providers = providers -class UnsatisfiableSpecNameError(UnsatisfiableSpecError): +class UnsatisfiableSpecNameError(spack.error.UnsatisfiableSpecError): """Raised when two specs aren't even for the same package.""" def __init__(self, provided, required): super(UnsatisfiableSpecNameError, self).__init__( provided, required, "name") -class UnsatisfiableVersionSpecError(UnsatisfiableSpecError): +class UnsatisfiableVersionSpecError(spack.error.UnsatisfiableSpecError): """Raised when a spec version conflicts with package constraints.""" def __init__(self, provided, required): super(UnsatisfiableVersionSpecError, self).__init__( provided, required, "version") -class UnsatisfiableCompilerSpecError(UnsatisfiableSpecError): +class UnsatisfiableCompilerSpecError(spack.error.UnsatisfiableSpecError): """Raised when a spec comiler conflicts with package constraints.""" def __init__(self, provided, required): super(UnsatisfiableCompilerSpecError, self).__init__( provided, required, "compiler") -class UnsatisfiableCompilerFlagSpecError(UnsatisfiableSpecError): +class UnsatisfiableCompilerFlagSpecError(spack.error.UnsatisfiableSpecError): """Raised when a spec variant conflicts with package constraints.""" def __init__(self, provided, required): super(UnsatisfiableCompilerFlagSpecError, self).__init__( provided, required, "compiler_flags") -class UnsatisfiableArchitectureSpecError(UnsatisfiableSpecError): +class UnsatisfiableArchitectureSpecError(spack.error.UnsatisfiableSpecError): """Raised when a spec architecture conflicts with package constraints.""" def __init__(self, provided, required): super(UnsatisfiableArchitectureSpecError, self).__init__( provided, required, "architecture") -class UnsatisfiableProviderSpecError(UnsatisfiableSpecError): +class UnsatisfiableProviderSpecError(spack.error.UnsatisfiableSpecError): """Raised when a provider is supplied but constraints don't match a vpkg requirement""" def __init__(self, provided, required): @@ -4421,14 +4421,14 @@ class UnsatisfiableProviderSpecError(UnsatisfiableSpecError): # TODO: get rid of this and be more specific about particular incompatible # dep constraints -class UnsatisfiableDependencySpecError(UnsatisfiableSpecError): +class UnsatisfiableDependencySpecError(spack.error.UnsatisfiableSpecError): """Raised when some dependency of constrained specs are incompatible""" def __init__(self, provided, required): super(UnsatisfiableDependencySpecError, self).__init__( provided, required, "dependency") -class AmbiguousHashError(SpecError): +class AmbiguousHashError(spack.error.SpecError): def __init__(self, msg, *specs): spec_fmt = '{namespace}.{name}{@version}{%compiler}{compiler_flags}' spec_fmt += '{variants}{arch=architecture}{/hash:7}' @@ -4437,21 +4437,21 @@ class AmbiguousHashError(SpecError): super(AmbiguousHashError, self).__init__(msg + specs_str) -class InvalidHashError(SpecError): +class InvalidHashError(spack.error.SpecError): def __init__(self, spec, hash): super(InvalidHashError, self).__init__( "The spec specified by %s does not match provided spec %s" % (hash, spec)) -class NoSuchHashError(SpecError): +class NoSuchHashError(spack.error.SpecError): def __init__(self, hash): super(NoSuchHashError, self).__init__( "No installed spec matches the hash: '%s'" % hash) -class SpecFilenameError(SpecError): +class SpecFilenameError(spack.error.SpecError): """Raised when a spec file name is invalid.""" @@ -4459,7 +4459,7 @@ class NoSuchSpecFileError(SpecFilenameError): """Raised when a spec file doesn't exist.""" -class RedundantSpecError(SpecError): +class RedundantSpecError(spack.error.SpecError): def __init__(self, spec, addition): super(RedundantSpecError, self).__init__( "Attempting to add %s to spec %s which is already concrete." @@ -4467,7 +4467,7 @@ class RedundantSpecError(SpecError): % (addition, spec)) -class SpecFormatStringError(SpecError): +class SpecFormatStringError(spack.error.SpecError): """Called for errors in Spec format strings.""" @@ -4479,7 +4479,7 @@ class SpecFormatSigilError(SpecFormatStringError): super(SpecFormatSigilError, self).__init__(msg) -class ConflictsInSpecError(SpecError, RuntimeError): +class ConflictsInSpecError(spack.error.SpecError, RuntimeError): def __init__(self, spec, matches): message = 'Conflicts in concretized spec "{0}"\n'.format( spec.short_spec @@ -4507,10 +4507,10 @@ class ConflictsInSpecError(SpecError, RuntimeError): super(ConflictsInSpecError, self).__init__(message, long_message) -class SpecDependencyNotFoundError(SpecError): +class SpecDependencyNotFoundError(spack.error.SpecError): """Raised when a failure is encountered writing the dependencies of a spec.""" -class SpecDeprecatedError(SpecError): +class SpecDeprecatedError(spack.error.SpecError): """Raised when a spec concretizes to a deprecated spec or dependency.""" diff --git a/lib/spack/spack/spec_list.py b/lib/spack/spack/spec_list.py index d9c1be2652..63bd29a7a9 100644 --- a/lib/spack/spack/spec_list.py +++ b/lib/spack/spack/spec_list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index bf65ee0b01..917d20aa04 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -1,8 +1,10 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from __future__ import print_function + import os import stat import sys @@ -269,7 +271,7 @@ class Stage(object): else: raise ValueError( "Can't construct Stage without url or fetch strategy") - self.fetcher.set_stage(self) + self.fetcher.stage = self # self.fetcher can change with mirrors. self.default_fetcher = self.fetcher self.search_fn = search_fn @@ -456,7 +458,7 @@ class Stage(object): for fetcher in generate_fetchers(): try: - fetcher.set_stage(self) + fetcher.stage = self self.fetcher = fetcher self.fetcher.fetch() break @@ -771,7 +773,7 @@ def get_checksums_for_versions( *spack.cmd.elide_list( ["{0:{1}} {2}".format(str(v), max_len, url_dict[v]) for v in sorted_versions])) - tty.msg('') + print() archives_to_fetch = tty.get_number( "How many would you like to checksum?", default=1, abort='q') @@ -818,7 +820,7 @@ def get_checksums_for_versions( ]) num_hash = len(version_hashes) - tty.msg("Checksummed {0} version{1} of {2}".format( + tty.msg("Checksummed {0} version{1} of {2}:".format( num_hash, '' if num_hash == 1 else 's', name)) return version_lines diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index a4f7596fc2..2b80e994c7 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/tengine.py b/lib/spack/spack/tengine.py index 6aceb391cd..e1e059c6a6 100644 --- a/lib/spack/spack/tengine.py +++ b/lib/spack/spack/tengine.py @@ -1,11 +1,10 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import itertools import textwrap -import jinja2 import llnl.util.lang import six @@ -13,9 +12,6 @@ import spack.config from spack.util.path import canonicalize_path -TemplateNotFound = jinja2.TemplateNotFound - - class ContextMeta(type): """Meta class for Context. It helps reducing the boilerplate in client code. @@ -77,6 +73,10 @@ def make_environment(dirs=None): dirs = [canonicalize_path(d) for d in itertools.chain(builtins, extensions)] + # avoid importing this at the top level as it's used infrequently and + # slows down startup a bit. + import jinja2 + # Loader for the templates loader = jinja2.FileSystemLoader(dirs) # Environment of the template engine diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 94f8ac4d9e..9f87532b85 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 0a37ef9558..1f5ff28900 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -1,5 +1,5 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -176,7 +176,7 @@ def test_arch_spec_container_semantic(item, architecture_str): ('gcc@4.7.2', 'ivybridge', '-march=core-avx-i -mtune=core-avx-i'), # Check mixed toolchains ('clang@8.0.0', 'broadwell', ''), - ('clang@3.5', 'x86_64', '-march=x86-64 -mcpu=generic'), + ('clang@3.5', 'x86_64', '-march=x86-64 -mtune=generic'), # Check clang compilers with 'apple' suffix ('clang@9.1.0-apple', 'x86_64', '-march=x86-64') ]) diff --git a/lib/spack/spack/test/build_distribution.py b/lib/spack/spack/test/build_distribution.py new file mode 100644 index 0000000000..2d567aee14 --- /dev/null +++ b/lib/spack/spack/test/build_distribution.py @@ -0,0 +1,41 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import os +import os.path + +import spack.spec +import spack.binary_distribution + +install = spack.main.SpackCommand('install') + + +def test_build_tarball_overwrite( + install_mockery, mock_fetch, monkeypatch, tmpdir): + + with tmpdir.as_cwd(): + spec = spack.spec.Spec('trivial-install-test-package').concretized() + install(str(spec)) + + # Runs fine the first time, throws the second time + spack.binary_distribution.build_tarball(spec, '.', unsigned=True) + with pytest.raises(spack.binary_distribution.NoOverwriteException): + spack.binary_distribution.build_tarball(spec, '.', unsigned=True) + + # Should work fine with force=True + spack.binary_distribution.build_tarball( + spec, '.', force=True, unsigned=True) + + # Remove the tarball and try again. + # This must *also* throw, because of the existing .spec.yaml file + os.remove(os.path.join( + spack.binary_distribution.build_cache_prefix('.'), + spack.binary_distribution.tarball_directory_name(spec), + spack.binary_distribution.tarball_name(spec, '.spack'))) + + with pytest.raises(spack.binary_distribution.NoOverwriteException): + spack.binary_distribution.build_tarball(spec, '.', unsigned=True) diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py index b6a0f4b441..a7a55af0ca 100644 --- a/lib/spack/spack/test/build_environment.py +++ b/lib/spack/spack/test/build_environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -14,10 +14,9 @@ import spack.spec from spack.paths import build_env_path from spack.build_environment import dso_suffix, _static_to_shared_library from spack.util.executable import Executable -from spack.util.spack_yaml import syaml_dict, syaml_str from spack.util.environment import EnvironmentModifications -from llnl.util.filesystem import LibraryList +from llnl.util.filesystem import LibraryList, HeaderList @pytest.fixture @@ -65,6 +64,18 @@ def build_environment(working_env): del os.environ[name] +@pytest.fixture +def ensure_env_variables(config, mock_packages, monkeypatch, working_env): + """Returns a function that takes a dictionary and updates os.environ + for the test lifetime accordingly. Plugs-in mock config and repo. + """ + def _ensure(env_mods): + for name, value in env_mods.items(): + monkeypatch.setenv(name, value) + + return _ensure + + def test_static_to_shared_library(build_environment): os.environ['SPACK_TEST_COMMAND'] = 'dump-args' @@ -119,79 +130,58 @@ def test_cc_not_changed_by_modules(monkeypatch, working_env): assert os.environ['ANOTHER_VAR'] == 'THIS_IS_SET' -@pytest.mark.usefixtures('config', 'mock_packages') -def test_compiler_config_modifications(monkeypatch, working_env): - s = spack.spec.Spec('cmake') - s.concretize() - pkg = s.package - - os.environ['SOME_VAR_STR'] = '' - os.environ['SOME_VAR_NUM'] = '0' - os.environ['PATH_LIST'] = '/path/third:/path/forth' - os.environ['EMPTY_PATH_LIST'] = '' - os.environ.pop('NEW_PATH_LIST', None) - - env_mod = syaml_dict() - set_cmd = syaml_dict() - env_mod[syaml_str('set')] = set_cmd - - set_cmd[syaml_str('SOME_VAR_STR')] = syaml_str('SOME_STR') - set_cmd[syaml_str('SOME_VAR_NUM')] = 1 - - monkeypatch.setattr(pkg.compiler, 'environment', env_mod) - spack.build_environment.setup_package(pkg, False) - assert os.environ['SOME_VAR_STR'] == 'SOME_STR' - assert os.environ['SOME_VAR_NUM'] == str(1) - - env_mod = syaml_dict() - unset_cmd = syaml_dict() - env_mod[syaml_str('unset')] = unset_cmd - - unset_cmd[syaml_str('SOME_VAR_STR')] = None - - monkeypatch.setattr(pkg.compiler, 'environment', env_mod) - assert 'SOME_VAR_STR' in os.environ - spack.build_environment.setup_package(pkg, False) - assert 'SOME_VAR_STR' not in os.environ - - env_mod = syaml_dict() - set_cmd = syaml_dict() - env_mod[syaml_str('set')] = set_cmd - append_cmd = syaml_dict() - env_mod[syaml_str('append-path')] = append_cmd - unset_cmd = syaml_dict() - env_mod[syaml_str('unset')] = unset_cmd - prepend_cmd = syaml_dict() - env_mod[syaml_str('prepend-path')] = prepend_cmd - - set_cmd[syaml_str('EMPTY_PATH_LIST')] = syaml_str('/path/middle') - - append_cmd[syaml_str('PATH_LIST')] = syaml_str('/path/last') - append_cmd[syaml_str('EMPTY_PATH_LIST')] = syaml_str('/path/last') - append_cmd[syaml_str('NEW_PATH_LIST')] = syaml_str('/path/last') - - unset_cmd[syaml_str('SOME_VAR_NUM')] = None +@pytest.mark.parametrize('initial,modifications,expected', [ + # Set and unset variables + ({'SOME_VAR_STR': '', 'SOME_VAR_NUM': '0'}, + {'set': {'SOME_VAR_STR': 'SOME_STR', 'SOME_VAR_NUM': 1}}, + {'SOME_VAR_STR': 'SOME_STR', 'SOME_VAR_NUM': '1'}), + ({'SOME_VAR_STR': ''}, + {'unset': ['SOME_VAR_STR']}, + {'SOME_VAR_STR': None}), + ({}, # Set a variable that was not defined already + {'set': {'SOME_VAR_STR': 'SOME_STR'}}, + {'SOME_VAR_STR': 'SOME_STR'}), + # Append and prepend to the same variable + ({'EMPTY_PATH_LIST': '/path/middle'}, + {'prepend_path': {'EMPTY_PATH_LIST': '/path/first'}, + 'append_path': {'EMPTY_PATH_LIST': '/path/last'}}, + {'EMPTY_PATH_LIST': '/path/first:/path/middle:/path/last'}), + # Append and prepend from empty variables + ({'EMPTY_PATH_LIST': '', 'SOME_VAR_STR': ''}, + {'prepend_path': {'EMPTY_PATH_LIST': '/path/first'}, + 'append_path': {'SOME_VAR_STR': '/path/last'}}, + {'EMPTY_PATH_LIST': '/path/first', 'SOME_VAR_STR': '/path/last'}), + ({}, # Same as before but on variables that were not defined + {'prepend_path': {'EMPTY_PATH_LIST': '/path/first'}, + 'append_path': {'SOME_VAR_STR': '/path/last'}}, + {'EMPTY_PATH_LIST': '/path/first', 'SOME_VAR_STR': '/path/last'}), + # Remove a path from a list + ({'EMPTY_PATH_LIST': '/path/first:/path/middle:/path/last'}, + {'remove_path': {'EMPTY_PATH_LIST': '/path/middle'}}, + {'EMPTY_PATH_LIST': '/path/first:/path/last'}), + ({'EMPTY_PATH_LIST': '/only/path'}, + {'remove_path': {'EMPTY_PATH_LIST': '/only/path'}}, + {'EMPTY_PATH_LIST': ''}), +]) +def test_compiler_config_modifications( + initial, modifications, expected, ensure_env_variables, monkeypatch +): + # Set the environment as per prerequisites + ensure_env_variables(initial) - prepend_cmd[syaml_str('PATH_LIST')] = syaml_str('/path/first:/path/second') - prepend_cmd[syaml_str('EMPTY_PATH_LIST')] = syaml_str('/path/first') - prepend_cmd[syaml_str('NEW_PATH_LIST')] = syaml_str('/path/first') - prepend_cmd[syaml_str('SOME_VAR_NUM')] = syaml_str('/8') + # Monkeypatch a pkg.compiler.environment with the required modifications + pkg = spack.spec.Spec('cmake').concretized().package + monkeypatch.setattr(pkg.compiler, 'environment', modifications) - assert 'SOME_VAR_NUM' in os.environ - monkeypatch.setattr(pkg.compiler, 'environment', env_mod) + # Trigger the modifications spack.build_environment.setup_package(pkg, False) - # Check that the order of modifications is respected and the - # variable was unset before it was prepended. - assert os.environ['SOME_VAR_NUM'] == '/8' - expected = '/path/first:/path/second:/path/third:/path/forth:/path/last' - assert os.environ['PATH_LIST'] == expected - - expected = '/path/first:/path/middle:/path/last' - assert os.environ['EMPTY_PATH_LIST'] == expected - - expected = '/path/first:/path/last' - assert os.environ['NEW_PATH_LIST'] == expected + # Check they were applied + for name, value in expected.items(): + if value is not None: + assert os.environ[name] == value + continue + assert name not in os.environ @pytest.mark.regression('9107') @@ -243,6 +233,18 @@ def test_set_build_environment_variables( variables. """ + # https://github.com/spack/spack/issues/13969 + cuda_headers = HeaderList([ + 'prefix/include/cuda_runtime.h', + 'prefix/include/cuda/atomic', + 'prefix/include/cuda/std/detail/libcxx/include/ctype.h']) + cuda_include_dirs = cuda_headers.directories + assert(os.path.join('prefix', 'include') + in cuda_include_dirs) + assert(os.path.join('prefix', 'include', 'cuda', 'std', 'detail', + 'libcxx', 'include') + not in cuda_include_dirs) + root = spack.spec.Spec('dt-diamond') root.concretize() diff --git a/lib/spack/spack/test/build_system_guess.py b/lib/spack/spack/test/build_system_guess.py index cd6216a907..76197e9e15 100644 --- a/lib/spack/spack/test/build_system_guess.py +++ b/lib/spack/spack/test/build_system_guess.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/build_systems.py b/lib/spack/spack/test/build_systems.py index 44a20ca56a..744821a04e 100644 --- a/lib/spack/spack/test/build_systems.py +++ b/lib/spack/spack/test/build_systems.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index e3c962b9d7..c1e1db914f 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py new file mode 100644 index 0000000000..3143a938c2 --- /dev/null +++ b/lib/spack/spack/test/ci.py @@ -0,0 +1,158 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import pytest +from six.moves.urllib.error import URLError + +import spack.ci as ci +import spack.main as spack_main +import spack.config as cfg +import spack.paths as spack_paths +import spack.spec as spec +import spack.util.web as web_util + + +@pytest.fixture +def tmp_scope(): + """Creates a temporary configuration scope""" + base_name = 'internal-testing-scope' + current_overrides = set( + x.name for x in + cfg.config.matching_scopes(r'^{0}'.format(base_name))) + + num_overrides = 0 + scope_name = base_name + while scope_name in current_overrides: + scope_name = '{0}{1}'.format(base_name, num_overrides) + num_overrides += 1 + + with cfg.override(cfg.InternalConfigScope(scope_name)): + yield scope_name + + +def test_urlencode_string(): + s = 'Spack Test Project' + + s_enc = ci.url_encode_string(s) + + assert(s_enc == 'Spack+Test+Project') + + +def test_import_signing_key(mock_gnupghome): + signing_key_dir = spack_paths.mock_gpg_keys_path + signing_key_path = os.path.join(signing_key_dir, 'package-signing-key') + with open(signing_key_path) as fd: + signing_key = fd.read() + + # Just make sure this does not raise any exceptions + ci.import_signing_key(signing_key) + + +def test_configure_compilers(mutable_config): + + def assert_missing(config): + assert('install_missing_compilers' not in config or + config['install_missing_compilers'] is False) + + def assert_present(config): + assert('install_missing_compilers' in config and + config['install_missing_compilers'] is True) + + original_config = cfg.get('config') + assert_missing(original_config) + + ci.configure_compilers('FIND_ANY', scope='site') + + second_config = cfg.get('config') + assert_missing(second_config) + + ci.configure_compilers('INSTALL_MISSING') + last_config = cfg.get('config') + assert_present(last_config) + + +def test_get_concrete_specs(config, mock_packages): + root_spec = ( + 'eJztkk1uwyAQhfc5BbuuYjWObSKuUlURYP5aDBjjBPv0RU7iRI6qpKuqUtnxzZvRwHud' + 'YxSt1oCMyuVoBdI5MN8paxDYZK/ZbkLYU3kqAuA0Dtz6BgGtTB8XdG87BCgzwXbwXArY' + 'CxYQiLtqXxUTpLZxSjN/mWlwwxAQlJ7v8wpFtsvK1UXSOUyTjvRKB2Um7LBPhZD0l1md' + 'xJ7VCATfszOiXGOR9np7vwDn7lCMS8SXQNf3RCtyBTVzzNTMUMXmfWrFeR+UngEAEncS' + 'ASjKwZcid7ERNldthBxjX46mMD2PsJnlYXDs2rye3l+vroOkJJ54SXgZPklLRQmx61sm' + 'cgKNVFRO0qlpf2pojq1Ro7OG56MY+Bgc1PkIo/WkaT8OVcrDYuvZkJdtBl/+XCZ+NQBJ' + 'oKg1h6X/VdXRoyE2OWeH6lCXZdHGrauUZAWFw/YJ/0/39OefN3F4Kle3cXjYsF684ZqG' + 'Tbap/uPwbRx+YPStIQ8bvgA7G6YE' + ) + + dep_builds = 'diffutils;libiconv' + spec_map = ci.get_concrete_specs(root_spec, 'bzip2', dep_builds, 'NONE') + + assert('root' in spec_map and 'deps' in spec_map) + + nonconc_root_spec = 'archive-files' + dep_builds = '' + spec_map = ci.get_concrete_specs( + nonconc_root_spec, 'archive-files', dep_builds, 'FIND_ANY') + + assert('root' in spec_map and 'deps' in spec_map) + assert('archive-files' in spec_map) + + +def test_register_cdash_build(): + build_name = 'Some pkg' + base_url = 'http://cdash.fake.org' + project = 'spack' + site = 'spacktests' + track = 'Experimental' + + with pytest.raises(URLError): + ci.register_cdash_build(build_name, base_url, project, site, track) + + +def test_relate_cdash_builds(config, mock_packages): + root_spec = ( + 'eJztkk1uwyAQhfc5BbuuYjWObSKuUlURYP5aDBjjBPv0RU7iRI6qpKuqUtnxzZvRwHud' + 'YxSt1oCMyuVoBdI5MN8paxDYZK/ZbkLYU3kqAuA0Dtz6BgGtTB8XdG87BCgzwXbwXArY' + 'CxYQiLtqXxUTpLZxSjN/mWlwwxAQlJ7v8wpFtsvK1UXSOUyTjvRKB2Um7LBPhZD0l1md' + 'xJ7VCATfszOiXGOR9np7vwDn7lCMS8SXQNf3RCtyBTVzzNTMUMXmfWrFeR+UngEAEncS' + 'ASjKwZcid7ERNldthBxjX46mMD2PsJnlYXDs2rye3l+vroOkJJ54SXgZPklLRQmx61sm' + 'cgKNVFRO0qlpf2pojq1Ro7OG56MY+Bgc1PkIo/WkaT8OVcrDYuvZkJdtBl/+XCZ+NQBJ' + 'oKg1h6X/VdXRoyE2OWeH6lCXZdHGrauUZAWFw/YJ/0/39OefN3F4Kle3cXjYsF684ZqG' + 'Tbap/uPwbRx+YPStIQ8bvgA7G6YE' + ) + + dep_builds = 'diffutils;libiconv' + spec_map = ci.get_concrete_specs(root_spec, 'bzip2', dep_builds, 'NONE') + cdash_api_url = 'http://cdash.fake.org' + job_build_id = '42' + cdash_project = 'spack' + cdashids_mirror_url = 'https://my.fake.mirror' + + with pytest.raises(web_util.SpackWebError): + ci.relate_cdash_builds(spec_map, cdash_api_url, job_build_id, + cdash_project, cdashids_mirror_url) + + # Just make sure passing None for build id doesn't throw exceptions + ci.relate_cdash_builds(spec_map, cdash_api_url, None, cdash_project, + cdashids_mirror_url) + + +def test_read_write_cdash_ids(config, tmp_scope, tmpdir, mock_packages): + working_dir = tmpdir.join('working_dir') + mirror_dir = working_dir.join('mirror') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + mirror_cmd = spack_main.SpackCommand('mirror') + mirror_cmd('add', '--scope', tmp_scope, 'test_mirror', mirror_url) + + mock_spec = spec.Spec('archive-files').concretized() + orig_cdashid = '42' + + ci.write_cdashid_to_mirror(orig_cdashid, mock_spec, mirror_url) + + # Now read it back + read_cdashid = ci.read_cdashid_from_mirror(mock_spec, mirror_url) + + assert(str(read_cdashid) == orig_cdashid) diff --git a/lib/spack/spack/test/cmd/__init__.py b/lib/spack/spack/test/cmd/__init__.py new file mode 100644 index 0000000000..9f87532b85 --- /dev/null +++ b/lib/spack/spack/test/cmd/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/activate.py b/lib/spack/spack/test/cmd/activate.py index efe07255fe..a3546230f1 100644 --- a/lib/spack/spack/test/cmd/activate.py +++ b/lib/spack/spack/test/cmd/activate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/arch.py b/lib/spack/spack/test/cmd/arch.py index 66c8f8e5dc..ec3c92a3d0 100644 --- a/lib/spack/spack/test/cmd/arch.py +++ b/lib/spack/spack/test/cmd/arch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/blame.py b/lib/spack/spack/test/cmd/blame.py index f1ac2b0b3a..61bdf41084 100644 --- a/lib/spack/spack/test/cmd/blame.py +++ b/lib/spack/spack/test/cmd/blame.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/build_env.py b/lib/spack/spack/test/cmd/build_env.py index 379de521b2..b7de759a2e 100644 --- a/lib/spack/spack/test/cmd/build_env.py +++ b/lib/spack/spack/test/cmd/build_env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index 2c4f351d86..064daeb063 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -1,4 +1,4 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,11 +8,19 @@ import platform import pytest import spack.main - +import spack.binary_distribution buildcache = spack.main.SpackCommand('buildcache') +@pytest.fixture() +def mock_get_specs(database, monkeypatch): + specs = database.query_local() + monkeypatch.setattr( + spack.binary_distribution, 'get_specs', lambda x, y: specs + ) + + @pytest.mark.skipif( platform.system().lower() != 'linux', reason='implementation for MacOS still missing' @@ -20,3 +28,16 @@ buildcache = spack.main.SpackCommand('buildcache') @pytest.mark.db def test_buildcache_preview_just_runs(database): buildcache('preview', 'mpileaks') + + +@pytest.mark.skipif( + platform.system().lower() != 'linux', + reason='implementation for MacOS still missing' +) +@pytest.mark.db +@pytest.mark.regression('13757') +def test_buildcache_list_duplicates(mock_get_specs, capsys): + with capsys.disabled(): + output = buildcache('list', 'mpileaks', '@2.3') + + assert output.count('mpileaks') == 3 diff --git a/lib/spack/spack/test/cmd/cd.py b/lib/spack/spack/test/cmd/cd.py index 356587b955..e3900c0d8f 100644 --- a/lib/spack/spack/test/cmd/cd.py +++ b/lib/spack/spack/test/cmd/cd.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py new file mode 100644 index 0000000000..49663b3409 --- /dev/null +++ b/lib/spack/spack/test/cmd/ci.py @@ -0,0 +1,573 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import filecmp +import os +import pytest + +import llnl.util.filesystem as fs + +import spack +import spack.ci as ci +import spack.config +import spack.environment as ev +import spack.hash_types as ht +from spack.main import SpackCommand +import spack.paths as spack_paths +import spack.repo as repo +from spack.spec import Spec +from spack.test.conftest import MockPackage, MockPackageMultiRepo +import spack.util.executable as exe +import spack.util.spack_yaml as syaml + + +ci_cmd = SpackCommand('ci') +env_cmd = SpackCommand('env') +mirror_cmd = SpackCommand('mirror') +gpg_cmd = SpackCommand('gpg') +install_cmd = SpackCommand('install') +buildcache_cmd = SpackCommand('buildcache') +git = exe.which('git', required=True) + + +@pytest.fixture() +def env_deactivate(): + yield + spack.environment._active_environment = None + os.environ.pop('SPACK_ENV', None) + + +def initialize_new_repo(repo_path, initial_commit=False): + if not os.path.exists(repo_path): + os.makedirs(repo_path) + + with fs.working_dir(repo_path): + init_args = ['init', '.'] + # if not initial_commit: + # init_args.append('--bare') + + git(*init_args) + + if initial_commit: + readme_contents = "This is the project README\n" + readme_path = os.path.join(repo_path, 'README.md') + with open(readme_path, 'w') as fd: + fd.write(readme_contents) + git('add', '.') + git('commit', '-m', 'Project initial commit') + + +def get_repo_status(repo_path): + with fs.working_dir(repo_path): + output = git('rev-parse', '--abbrev-ref', 'HEAD', output=str) + current_branch = output.split()[0] + + output = git('rev-parse', 'HEAD', output=str) + current_sha = output.split()[0] + + return current_branch, current_sha + + +def set_env_var(key, val): + os.environ[key] = val + + +def test_specs_staging(config): + """Make sure we achieve the best possible staging for the following +spec DAG:: + + a + /| + c b + |\ + e d + |\ + f g + +In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage, +and then 'd', 'b', and 'a' to be put in the next three stages, respectively. + +""" + default = ('build', 'link') + + g = MockPackage('g', [], []) + f = MockPackage('f', [], []) + e = MockPackage('e', [], []) + d = MockPackage('d', [f, g], [default, default]) + c = MockPackage('c', [], []) + b = MockPackage('b', [d, e], [default, default]) + a = MockPackage('a', [b, c], [default, default]) + + mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g]) + + with repo.swap(mock_repo): + spec_a = Spec('a') + spec_a.concretize() + + spec_a_label = ci.spec_deps_key_label(spec_a)[1] + spec_b_label = ci.spec_deps_key_label(spec_a['b'])[1] + spec_c_label = ci.spec_deps_key_label(spec_a['c'])[1] + spec_d_label = ci.spec_deps_key_label(spec_a['d'])[1] + spec_e_label = ci.spec_deps_key_label(spec_a['e'])[1] + spec_f_label = ci.spec_deps_key_label(spec_a['f'])[1] + spec_g_label = ci.spec_deps_key_label(spec_a['g'])[1] + + spec_labels, dependencies, stages = ci.stage_spec_jobs([spec_a]) + + assert (len(stages) == 4) + + assert (len(stages[0]) == 4) + assert (spec_c_label in stages[0]) + assert (spec_e_label in stages[0]) + assert (spec_f_label in stages[0]) + assert (spec_g_label in stages[0]) + + assert (len(stages[1]) == 1) + assert (spec_d_label in stages[1]) + + assert (len(stages[2]) == 1) + assert (spec_b_label in stages[2]) + + assert (len(stages[3]) == 1) + assert (spec_a_label in stages[3]) + + +def test_ci_generate_with_env(tmpdir, mutable_mock_env_path, env_deactivate, + install_mockery, mock_packages): + """Make sure we can get a .gitlab-ci.yml from an environment file + which has the gitlab-ci, cdash, and mirrors sections.""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + definitions: + - bootstrap: + - cmake@3.4.3 + - old-gcc-pkgs: + - archive-files + - callpath + - hypre@0.2.15 + specs: + - matrix: + - [$old-gcc-pkgs] + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + bootstrap: + - name: bootstrap + compiler-agnostic: true + mappings: + - match: + - arch=test-debian6-x86_64 + runner-attributes: + tags: + - donotcare + image: donotcare + final-stage-rebuild-index: + image: donotcare + tags: [donotcare] + cdash: + build-group: Not important + url: https://my.fake.cdash + project: Not used + site: Nothing +""") + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outputfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + ci_cmd('generate', '--output-file', outputfile) + + with open(outputfile) as f: + contents = f.read() + yaml_contents = syaml.load(contents) + found_spec = False + for ci_key in yaml_contents.keys(): + if '(bootstrap)' in ci_key: + found_spec = True + assert('cmake' in ci_key) + assert(found_spec) + assert('stages' in yaml_contents) + assert(len(yaml_contents['stages']) == 6) + assert(yaml_contents['stages'][0] == 'stage-0') + assert(yaml_contents['stages'][5] == 'stage-rebuild-index') + + +def test_ci_generate_with_env_missing_section(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages): + """Make sure we get a reasonable message if we omit gitlab-ci section""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + mirrors: + some-mirror: https://my.fake.mirror +""") + + expect_out = 'Error: Environment yaml does not have "gitlab-ci" section' + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + + with ev.read('test'): + output = ci_cmd('generate', fail_on_error=False, output=str) + assert(expect_out in output) + + +def test_ci_generate_with_cdash_token(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages): + """Make sure we it doesn't break if we configure cdash""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + enable-artifacts-buildcache: True + enable-debug-messages: True + mappings: + - match: + - archive-files + runner-attributes: + tags: + - donotcare + image: donotcare + cdash: + build-group: Not important + url: https://my.fake.cdash + project: Not used + site: Nothing +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + + with ev.read('test'): + fake_token = 'notreallyatokenbutshouldnotmatter' + os.environ['SPACK_CDASH_AUTH_TOKEN'] = fake_token + copy_to_file = str(tmpdir.join('backup-ci.yml')) + output = ci_cmd('generate', '--copy-to', copy_to_file, output=str) + # That fake token should still have resulted in being unable to + # register build group with cdash, but the workload should + # still have been generated. + expect = 'Problem populating buildgroup' + assert(expect in output) + + dir_contents = os.listdir(tmpdir.strpath) + + print(dir_contents) + + assert('backup-ci.yml' in dir_contents) + + orig_file = str(tmpdir.join('.gitlab-ci.yml')) + + assert(filecmp.cmp(orig_file, copy_to_file) is True) + + +def test_ci_generate_with_external_pkg(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages): + """Make sure we do not generate jobs for external pkgs""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + - externaltest + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + mappings: + - match: + - archive-files + - externaltest + runner-attributes: + tags: + - donotcare + image: donotcare +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outputfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + ci_cmd('generate', '--output-file', outputfile) + + with open(outputfile) as f: + contents = f.read() + print('generated contents: ') + print(contents) + yaml_contents = syaml.load(contents) + for ci_key in yaml_contents.keys(): + if 'externaltool' in ci_key: + print('Erroneously staged "externaltool" pkg') + assert(False) + + +def test_ci_generate_debug_with_custom_spack(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages): + """Make sure we generate cloning of spack in job script if needed""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + enable-artifacts-buildcache: True + enable-debug-messages: True + mappings: + - match: + - archive-files + runner-attributes: + tags: + - donotcare + image: donotcare +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + spack_repo = 'https://github.com/usera/spack.git' + spack_ref = 'custom-branch' + expected_clone_str = 'git clone "{0}"'.format(spack_repo) + + ci_cmd('generate', '--output-file', outfile, '--spack-repo', + spack_repo, '--spack-ref', spack_ref) + + with open(outfile) as f: + contents = f.read() + yaml_contents = syaml.load(contents) + for ci_key in yaml_contents.keys(): + if '(specs)' in ci_key: + next_job = yaml_contents[ci_key] + print(next_job) + assert('before_script' in next_job) + before_script = next_job['before_script'] + for step in before_script: + if expected_clone_str in step: + break + else: + msg = 'job "{0}" did not clone spack repo'.format( + ci_key) + print(msg) + assert(False) + + assert('script' in next_job) + script = next_job['script'] + for step in script: + if 'spack -d ci rebuild' in step: + break + else: + msg = 'job {0} missing rebuild command'.format( + ci_key) + print(msg) + assert(False) + + +def test_ci_rebuild_basic(tmpdir, mutable_mock_env_path, env_deactivate, + install_mockery, mock_packages, + mock_gnupghome): + working_dir = tmpdir.join('working_dir') + + mirror_dir = working_dir.join('mirror') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + signing_key_dir = spack_paths.mock_gpg_keys_path + signing_key_path = os.path.join(signing_key_dir, 'package-signing-key') + with open(signing_key_path) as fd: + signing_key = fd.read() + + spack_yaml_contents = """ +spack: + definitions: + - packages: [archive-files] + specs: + - $packages + mirrors: + test-mirror: {0} + gitlab-ci: + enable-artifacts-buildcache: True + mappings: + - match: + - archive-files + runner-attributes: + tags: + - donotcare + image: donotcare + cdash: + build-group: Not important + url: https://my.fake.cdash + project: Not used + site: Nothing +""".format(mirror_url) + + print('spack.yaml:\n{0}\n'.format(spack_yaml_contents)) + + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write(spack_yaml_contents) + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + with ev.read('test'): + root_spec = ('eJyNjsGOwyAMRO/5Ct96alRFFK34ldUqcohJ6BJAQFHUry9Nk66' + 'UXNY3v5mxJ3qSojoDBjnqTGelDUVRQZlMIWpnBZya+nJa0Mv1Fg' + 'G8waRcmAQkimkHWxcF9NRptHyVEoaBkoD5i7ecLVC6yZd/YTtpc' + 'SIBg5Tr/mnA6mt9qTZL9CiLr7trk7StJyd/F81jKGoqoe2gVAaH' + '0uT7ZwPeH9A875HaA9MfidHdHxgxjgJuTGVtIrvfHGtynjkGyzi' + 'xRrkHy94t1lftvv1n4AkVK3kQ') + + # Create environment variables as gitlab would do it + set_env_var('CI_PROJECT_DIR', working_dir.strpath) + set_env_var('SPACK_SIGNING_KEY', signing_key) + set_env_var('SPACK_ROOT_SPEC', root_spec) + set_env_var('SPACK_JOB_SPEC_PKG_NAME', 'archive-files') + set_env_var('SPACK_COMPILER_ACTION', 'NONE') + set_env_var('SPACK_CDASH_BUILD_NAME', '(specs) archive-files') + set_env_var('SPACK_RELATED_BUILDS_CDASH', '') + + rebuild_output = ci_cmd( + 'rebuild', fail_on_error=False, output=str) + + print(rebuild_output) + + +def test_ci_pushyaml(tmpdir): + fake_yaml_contents = """generate ci jobs: + script: + - "./share/spack/qa/gitlab/generate-gitlab-ci-yml.sh" + tags: + - "spack-pre-ci" + artifacts: + paths: + - ci-generation + when: always + """ + local_repo_path = tmpdir.join('local_repo') + initialize_new_repo(local_repo_path.strpath, True) + + remote_repo_path = tmpdir.join('remote_repo') + initialize_new_repo(remote_repo_path.strpath) + + current_branch, current_sha = get_repo_status(local_repo_path.strpath) + + print('local repo info: {0}, {1}'.format(current_branch, current_sha)) + + local_jobs_yaml = local_repo_path.join('.gitlab-ci.yml') + with local_jobs_yaml.open('w') as f: + f.write(fake_yaml_contents) + + pushyaml_args = [ + 'pushyaml', + '--downstream-repo', remote_repo_path.strpath, + '--branch-name', current_branch, + '--commit-sha', current_sha, + ] + + with fs.working_dir(local_repo_path.strpath): + ci_cmd(*pushyaml_args) + + with fs.working_dir(remote_repo_path.strpath): + branch_to_checkout = 'multi-ci-{0}'.format(current_branch) + git('checkout', branch_to_checkout) + with open('.gitlab-ci.yml') as fd: + pushed_contents = fd.read() + assert pushed_contents == fake_yaml_contents + + +@pytest.mark.disable_clean_stage_check +def test_push_mirror_contents(tmpdir, mutable_mock_env_path, env_deactivate, + install_mockery, mock_packages, mock_fetch, + mock_stage, mock_gnupghome): + working_dir = tmpdir.join('working_dir') + + mirror_dir = working_dir.join('mirror') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + signing_key_dir = spack_paths.mock_gpg_keys_path + signing_key_path = os.path.join(signing_key_dir, 'package-signing-key') + with open(signing_key_path) as fd: + signing_key = fd.read() + + ci.import_signing_key(signing_key) + + spack_yaml_contents = """ +spack: + definitions: + - packages: [patchelf] + specs: + - $packages + mirrors: + test-mirror: {0} +""".format(mirror_url) + + print('spack.yaml:\n{0}\n'.format(spack_yaml_contents)) + + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write(spack_yaml_contents) + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + with ev.read('test') as env: + spec_map = ci.get_concrete_specs( + 'patchelf', 'patchelf', '', 'FIND_ANY') + concrete_spec = spec_map['patchelf'] + spec_yaml = concrete_spec.to_yaml(hash=ht.build_hash) + yaml_path = str(tmpdir.join('spec.yaml')) + with open(yaml_path, 'w') as ypfd: + ypfd.write(spec_yaml) + + install_cmd('--keep-stage', yaml_path) + + # env, spec, yaml_path, mirror_url, build_id + ci.push_mirror_contents( + env, concrete_spec, yaml_path, mirror_url, '42') + + buildcache_list_output = buildcache_cmd('list', output=str) + + assert('patchelf' in buildcache_list_output) + + logs_dir = working_dir.join('logs_dir') + if not os.path.exists(logs_dir.strpath): + os.makedirs(logs_dir.strpath) + + ci.copy_stage_logs_to_artifacts(concrete_spec, logs_dir.strpath) + + logs_dir_list = os.listdir(logs_dir.strpath) + + assert('spack-build-env.txt' in logs_dir_list) + assert('spack-build-out.txt' in logs_dir_list) + + # Also just make sure that if something goes wrong with the + # stage logs copy, no exception is thrown + ci.copy_stage_logs_to_artifacts(None, logs_dir.strpath) + + dl_dir = working_dir.join('download_dir') + if not os.path.exists(dl_dir.strpath): + os.makedirs(dl_dir.strpath) + + buildcache_cmd('download', '--spec-yaml', yaml_path, '--path', + dl_dir.strpath, '--require-cdashid') + + dl_dir_list = os.listdir(dl_dir.strpath) + + assert(len(dl_dir_list) == 3) diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index a0e8d89aac..c3e9d79f23 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index f0b80ea031..409dea7c51 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -1,15 +1,20 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import re -import pytest +import filecmp +import os +import shutil +import subprocess -from llnl.util.argparsewriter import ArgparseWriter +import pytest import spack.cmd +from spack.cmd.commands import _positional_to_subroutine import spack.main +import spack.paths + commands = spack.main.SpackCommand('commands') @@ -17,38 +22,64 @@ parser = spack.main.make_argument_parser() spack.main.add_all_commands(parser) -def test_commands_by_name(): +def test_names(): """Test default output of spack commands.""" - out = commands() - assert out.strip().split('\n') == sorted(spack.cmd.all_commands()) + out1 = commands().strip().split('\n') + assert out1 == spack.cmd.all_commands() + assert 'rm' not in out1 + + out2 = commands('--aliases').strip().split('\n') + assert out1 != out2 + assert 'rm' in out2 + + out3 = commands('--format=names').strip().split('\n') + assert out1 == out3 def test_subcommands(): """Test subcommand traversal.""" - out = commands('--format=subcommands') - assert 'spack mirror create' in out - assert 'spack buildcache list' in out - assert 'spack repo add' in out - assert 'spack pkg diff' in out - assert 'spack url parse' in out - assert 'spack view symlink' in out - - class Subcommands(ArgparseWriter): - def begin_command(self, prog): - assert prog in out + out1 = commands('--format=subcommands') + assert 'spack mirror create' in out1 + assert 'spack buildcache list' in out1 + assert 'spack repo add' in out1 + assert 'spack pkg diff' in out1 + assert 'spack url parse' in out1 + assert 'spack view symlink' in out1 + assert 'spack rm' not in out1 + assert 'spack compiler add' not in out1 - Subcommands().write(parser) + out2 = commands('--aliases', '--format=subcommands') + assert 'spack mirror create' in out2 + assert 'spack buildcache list' in out2 + assert 'spack repo add' in out2 + assert 'spack pkg diff' in out2 + assert 'spack url parse' in out2 + assert 'spack view symlink' in out2 + assert 'spack rm' in out2 + assert 'spack compiler add' in out2 def test_rst(): """Do some simple sanity checks of the rst writer.""" - out = commands('--format=rst') + out1 = commands('--format=rst') + assert 'spack mirror create' in out1 + assert 'spack buildcache list' in out1 + assert 'spack repo add' in out1 + assert 'spack pkg diff' in out1 + assert 'spack url parse' in out1 + assert 'spack view symlink' in out1 + assert 'spack rm' not in out1 + assert 'spack compiler add' not in out1 - class Subcommands(ArgparseWriter): - def begin_command(self, prog): - assert prog in out - assert re.sub(r' ', '-', prog) in out - Subcommands().write(parser) + out2 = commands('--aliases', '--format=rst') + assert 'spack mirror create' in out2 + assert 'spack buildcache list' in out2 + assert 'spack repo add' in out2 + assert 'spack pkg diff' in out2 + assert 'spack url parse' in out2 + assert 'spack view symlink' in out2 + assert 'spack rm' in out2 + assert 'spack compiler add' in out2 def test_rst_with_input_files(tmpdir): @@ -109,3 +140,126 @@ def test_rst_update(tmpdir): assert update_file.exists() with update_file.open() as f: assert f.read() == 'empty\n' + + +def test_update_with_header(tmpdir): + update_file = tmpdir.join('output') + + # not yet created when commands is run + commands('--update', str(update_file)) + assert update_file.exists() + with update_file.open() as f: + assert f.read() + fake_header = 'this is a header!\n\n' + + filename = tmpdir.join('header.txt') + with filename.open('w') as f: + f.write(fake_header) + + # created, newer than commands, but older than header + commands('--update', str(update_file), '--header', str(filename)) + + # newer than commands and header + commands('--update', str(update_file), '--header', str(filename)) + + +@pytest.mark.xfail +def test_no_pipe_error(): + """Make sure we don't see any pipe errors when piping output.""" + + proc = subprocess.Popen( + ['spack', 'commands', '--format=rst'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # Call close() on stdout to cause a broken pipe + proc.stdout.close() + proc.wait() + stderr = proc.stderr.read().decode('utf-8') + + assert 'Broken pipe' not in stderr + + +def test_bash_completion(): + """Test the bash completion writer.""" + out1 = commands('--format=bash') + + # Make sure header not included + assert '_bash_completion_spack() {' not in out1 + assert '_all_packages() {' not in out1 + + # Make sure subcommands appear + assert '_spack_remove() {' in out1 + assert '_spack_compiler_find() {' in out1 + + # Make sure aliases don't appear + assert '_spack_rm() {' not in out1 + assert '_spack_compiler_add() {' not in out1 + + # Make sure options appear + assert '-h --help' in out1 + + # Make sure subcommands are called + for function in _positional_to_subroutine.values(): + assert function in out1 + + out2 = commands('--aliases', '--format=bash') + + # Make sure aliases appear + assert '_spack_rm() {' in out2 + assert '_spack_compiler_add() {' in out2 + + +def test_update_completion_arg(tmpdir, monkeypatch): + mock_infile = tmpdir.join("spack-completion.in") + mock_bashfile = tmpdir.join("spack-completion.bash") + + mock_args = { + "bash": { + "aliases": True, + "format": "bash", + "header": str(mock_infile), + "update": str(mock_bashfile), + }, + } + + # make a mock completion file missing the --update-completion argument + real_args = spack.cmd.commands.update_completion_args + shutil.copy(real_args['bash']['header'], mock_args['bash']['header']) + with open(real_args['bash']['update']) as old: + old_file = old.read() + with open(mock_args['bash']['update'], 'w') as mock: + mock.write(old_file.replace("--update-completion", "")) + mock_bashfile.setmtime(0) # ensure mtime triggers update + + monkeypatch.setattr( + spack.cmd.commands, 'update_completion_args', mock_args) + + # ensure things fail if --update-completion isn't specified alone + with pytest.raises(spack.main.SpackCommandError): + commands("--update-completion", "-a") + + # ensure arg is restored + assert "--update-completion" not in mock_bashfile.read() + commands("--update-completion") + assert "--update-completion" in mock_bashfile.read() + + +def test_updated_completion_scripts(tmpdir): + """Make sure our shell tab completion scripts remain up-to-date.""" + + msg = ("It looks like Spack's command-line interface has been modified. " + "Please update Spack's shell tab completion scripts by running:\n\n" + " spack commands --update-completion\n\n" + "and adding the changed files to your pull request.") + + for shell in ['bash']: # 'zsh', 'fish']: + header = os.path.join( + spack.paths.share_path, shell, 'spack-completion.in') + script = 'spack-completion.{0}'.format(shell) + old_script = os.path.join(spack.paths.share_path, script) + new_script = str(tmpdir.join(script)) + + commands('--aliases', '--format', shell, + '--header', header, '--update', new_script) + + assert filecmp.cmp(old_script, new_script), msg diff --git a/lib/spack/spack/test/cmd/common/__init__.py b/lib/spack/spack/test/cmd/common/__init__.py new file mode 100644 index 0000000000..9f87532b85 --- /dev/null +++ b/lib/spack/spack/test/cmd/common/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/common/arguments.py b/lib/spack/spack/test/cmd/common/arguments.py index 82ae495611..5117607a2e 100644 --- a/lib/spack/spack/test/cmd/common/arguments.py +++ b/lib/spack/spack/test/cmd/common/arguments.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/compiler_command.py b/lib/spack/spack/test/cmd/compiler_command.py index a4ceb63822..15949d08d4 100644 --- a/lib/spack/spack/test/cmd/compiler_command.py +++ b/lib/spack/spack/test/cmd/compiler_command.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py index 0d6cc7ec40..82a9d814ea 100644 --- a/lib/spack/spack/test/cmd/config.py +++ b/lib/spack/spack/test/cmd/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -14,13 +14,13 @@ from spack.main import SpackCommand config = SpackCommand('config') -def test_get_config_scope(mock_config): +def test_get_config_scope(mock_low_high_config): assert config('get', 'compilers').strip() == 'compilers: {}' -def test_get_config_scope_merged(mock_config): - low_path = mock_config.scopes['low'].path - high_path = mock_config.scopes['high'].path +def test_get_config_scope_merged(mock_low_high_config): + low_path = mock_low_high_config.scopes['low'].path + high_path = mock_low_high_config.scopes['high'].path mkdirp(low_path) mkdirp(high_path) @@ -91,3 +91,9 @@ def test_config_edit_fails_correctly_with_no_env(mutable_mock_env_path): def test_config_get_fails_correctly_with_no_env(mutable_mock_env_path): output = config('get', fail_on_error=False) assert "requires a section argument or an active environment" in output + + +def test_config_list(): + output = config('list') + assert 'compilers' in output + assert 'packages' in output diff --git a/lib/spack/spack/test/cmd/create.py b/lib/spack/spack/test/cmd/create.py index 27d2b252bb..4262744317 100644 --- a/lib/spack/spack/test/cmd/create.py +++ b/lib/spack/spack/test/cmd/create.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -95,12 +95,16 @@ def test_create_template(parser, mock_test_repo, args, name, expected): (' ', 'name must be provided'), ('bad#name', 'name can only contain'), ]) -def test_create_template_bad_name(parser, mock_test_repo, name, expected): +def test_create_template_bad_name( + parser, mock_test_repo, name, expected, capsys): """Test template creation with bad name options.""" constr_args = parser.parse_args(['--skip-editor', '-n', name]) - with pytest.raises(SystemExit, matches=expected): + with pytest.raises(SystemExit): spack.cmd.create.create(parser, constr_args) + captured = capsys.readouterr() + assert expected in str(captured) + def test_build_system_guesser_no_stage(parser): """Test build system guesser when stage not provided.""" @@ -108,7 +112,7 @@ def test_build_system_guesser_no_stage(parser): # Ensure get the expected build system with pytest.raises(AttributeError, - matches="'NoneType' object has no attribute"): + match="'NoneType' object has no attribute"): guesser(None, '/the/url/does/not/matter') @@ -142,7 +146,7 @@ def test_get_name_urls(parser, url, expected): assert name == expected -def test_get_name_error(parser, monkeypatch): +def test_get_name_error(parser, monkeypatch, capsys): """Test get_name UndetectableNameError exception path.""" def _parse_name_offset(path, v): raise UndetectableNameError(path) @@ -152,5 +156,7 @@ def test_get_name_error(parser, monkeypatch): url = 'downloads.sourceforge.net/noapp/' args = parser.parse_args([url]) - with pytest.raises(SystemExit, matches="Couldn't guess a name"): + with pytest.raises(SystemExit): spack.cmd.create.get_name(args) + captured = capsys.readouterr() + assert "Couldn't guess a name" in str(captured) diff --git a/lib/spack/spack/test/cmd/debug.py b/lib/spack/spack/test/cmd/debug.py index aba46c2398..4a06276abf 100644 --- a/lib/spack/spack/test/cmd/debug.py +++ b/lib/spack/spack/test/cmd/debug.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/dependencies.py b/lib/spack/spack/test/cmd/dependencies.py index d38591e907..fc47069181 100644 --- a/lib/spack/spack/test/cmd/dependencies.py +++ b/lib/spack/spack/test/cmd/dependencies.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/dependents.py b/lib/spack/spack/test/cmd/dependents.py index 69c680a3c8..1001e8764f 100644 --- a/lib/spack/spack/test/cmd/dependents.py +++ b/lib/spack/spack/test/cmd/dependents.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/deprecate.py b/lib/spack/spack/test/cmd/deprecate.py index 6bc87fa5e3..7b38c177f5 100644 --- a/lib/spack/spack/test/cmd/deprecate.py +++ b/lib/spack/spack/test/cmd/deprecate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/dev_build.py b/lib/spack/spack/test/cmd/dev_build.py index 835edac138..a007513067 100644 --- a/lib/spack/spack/test/cmd/dev_build.py +++ b/lib/spack/spack/test/cmd/dev_build.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 9b3ca6e66e..9e8c424ce0 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -26,7 +26,7 @@ import spack.util.spack_json as sjson # everything here uses the mock_env_path pytestmark = pytest.mark.usefixtures( - 'mutable_mock_env_path', 'config', 'mutable_mock_packages') + 'mutable_mock_env_path', 'config', 'mutable_mock_repo') env = SpackCommand('env') install = SpackCommand('install') @@ -63,6 +63,27 @@ def test_add(): assert Spec('mpileaks') in e.user_specs +def test_env_add_virtual(): + env('create', 'test') + + e = ev.read('test') + e.add('mpi') + e.concretize() + + hashes = e.concretized_order + assert len(hashes) == 1 + spec = e.specs_by_hash[hashes[0]] + assert spec.satisfies('mpi') + + +def test_env_add_nonexistant_fails(): + env('create', 'test') + + e = ev.read('test') + with pytest.raises(ev.SpackEnvironmentError, match=r'no such package'): + e.add('thispackagedoesnotexist') + + def test_env_list(mutable_mock_env_path): env('create', 'foo') env('create', 'bar') @@ -382,13 +403,11 @@ env: mpileaks: version: [2.2] """ - spack.package_prefs.PackagePrefs.clear_caches() - _env_create('test', StringIO(test_config)) e = ev.read('test') - ev.prepare_config_scope(e) - e.concretize() + with e: + e.concretize() assert any(x.satisfies('mpileaks@2.2') for x in e._get_environment_specs()) @@ -402,8 +421,6 @@ env: specs: - mpileaks """ - spack.package_prefs.PackagePrefs.clear_caches() - _env_create('test', StringIO(test_config)) e = ev.read('test') @@ -414,8 +431,8 @@ packages: version: [2.2] """) - ev.prepare_config_scope(e) - e.concretize() + with e: + e.concretize() assert any(x.satisfies('mpileaks@2.2') for x in e._get_environment_specs()) @@ -431,7 +448,6 @@ env: - mpileaks """ % config_scope_path - spack.package_prefs.PackagePrefs.clear_caches() _env_create('test', StringIO(test_config)) e = ev.read('test') @@ -444,8 +460,8 @@ packages: version: [2.2] """) - ev.prepare_config_scope(e) - e.concretize() + with e: + e.concretize() assert any(x.satisfies('mpileaks@2.2') for x in e._get_environment_specs()) @@ -462,9 +478,6 @@ env: specs: - mpileaks """ - - spack.package_prefs.PackagePrefs.clear_caches() - _env_create('test', StringIO(test_config)) e = ev.read('test') @@ -477,8 +490,8 @@ packages: version: [0.8.11] """) - ev.prepare_config_scope(e) - e.concretize() + with e: + e.concretize() # ensure included scope took effect assert any( @@ -498,8 +511,6 @@ env: specs: - mpileaks """ - spack.package_prefs.PackagePrefs.clear_caches() - _env_create('test', StringIO(test_config)) e = ev.read('test') @@ -519,8 +530,8 @@ packages: version: [0.8.12] """) - ev.prepare_config_scope(e) - e.concretize() + with e: + e.concretize() assert any( x.satisfies('mpileaks@2.2') for x in e._get_environment_specs()) @@ -765,13 +776,13 @@ def test_indirect_build_dep(): @pytest.mark.usefixtures('config') def test_store_different_build_deps(): r"""Ensure that an environment can store two instances of a build-only -Dependency: + dependency:: - x y - /| (l) | (b) - (b) | y z2 - \| (b) # noqa: W605 - z1 + x y + /| (l) | (b) + (b) | y z2 + \| (b) + z1 """ default = ('build', 'link') @@ -1777,7 +1788,7 @@ def test_duplicate_packages_raise_when_concretizing_together(): def test_env_write_only_non_default(): - print(env('create', 'test')) + env('create', 'test') e = ev.read('test') with open(e.manifest_path, 'r') as f: diff --git a/lib/spack/spack/test/cmd/extensions.py b/lib/spack/spack/test/cmd/extensions.py index 544133698f..505573a7bc 100644 --- a/lib/spack/spack/test/cmd/extensions.py +++ b/lib/spack/spack/test/cmd/extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index 45b065fce9..8516569592 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -1,20 +1,25 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import json +import os import pytest import spack.cmd as cmd import spack.cmd.find +import spack.user_environment as uenv from spack.main import SpackCommand from spack.spec import Spec from spack.util.pattern import Bunch +import spack.environment as ev find = SpackCommand('find') +env = SpackCommand('env') +install = SpackCommand('install') base32_alphabet = 'abcdefghijklmnopqrstuvwxyz234567' @@ -302,3 +307,27 @@ def test_find_no_sections(database, config): def test_find_command_basic_usage(database): output = find() assert 'mpileaks' in output + + +@pytest.mark.regression('9875') +def test_find_prefix_in_env(mutable_mock_env_path, install_mockery, mock_fetch, + mock_packages, mock_archive, config): + """Test `find` formats requiring concrete specs work in environments.""" + env('create', 'test') + with ev.read('test'): + install('mpileaks') + find('-p') + find('-l') + find('-L') + # Would throw error on regression + + +def test_find_loaded(database, working_env): + output = find('--loaded', '--group') + assert output == '' # 0 packages installed printed separately + + os.environ[uenv.spack_loaded_hashes_var] = ':'.join( + [x.dag_hash() for x in spack.store.db.query()]) + output = find('--loaded') + expected = find() + assert output == expected diff --git a/lib/spack/spack/test/cmd/flake8.py b/lib/spack/spack/test/cmd/flake8.py index b95db93364..d4bfc5618d 100644 --- a/lib/spack/spack/test/cmd/flake8.py +++ b/lib/spack/spack/test/cmd/flake8.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -45,7 +45,7 @@ def flake8_package(): def test_changed_files(parser, flake8_package): - args = parser.parse_args() + args = parser.parse_args([]) # changed_files returns file paths relative to the root # directory of Spack. Convert to absolute file paths. diff --git a/lib/spack/spack/test/cmd/gc.py b/lib/spack/spack/test/cmd/gc.py new file mode 100644 index 0000000000..22c85a1d78 --- /dev/null +++ b/lib/spack/spack/test/cmd/gc.py @@ -0,0 +1,46 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import spack.environment as ev +import spack.spec +import spack.main + +gc = spack.main.SpackCommand('gc') + + +@pytest.mark.db +def test_no_packages_to_remove(config, mutable_database, capsys): + with capsys.disabled(): + output = gc('-y') + assert 'There are no unused specs.' in output + + +@pytest.mark.db +def test_packages_are_removed(config, mutable_database, capsys): + s = spack.spec.Spec('simple-inheritance') + s.concretize() + s.package.do_install(fake=True, explicit=True) + with capsys.disabled(): + output = gc('-y') + assert 'Successfully uninstalled cmake' in output + + +@pytest.mark.db +def test_gc_with_environment( + config, mutable_database, mutable_mock_env_path, capsys +): + s = spack.spec.Spec('simple-inheritance') + s.concretize() + s.package.do_install(fake=True, explicit=True) + + e = ev.create('test_gc') + e.add('cmake') + with e: + with capsys.disabled(): + output = gc('-y') + assert 'Restricting the garbage collection' in output + assert 'There are no unused specs' in output diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py index f42282d72d..4333a38fe2 100644 --- a/lib/spack/spack/test/cmd/gpg.py +++ b/lib/spack/spack/test/cmd/gpg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -7,37 +7,55 @@ import os import pytest +import llnl.util.filesystem as fs + +import spack.util.executable +import spack.util.gpg + from spack.paths import mock_gpg_data_path, mock_gpg_keys_path -import spack.util.gpg as gpg_util from spack.main import SpackCommand from spack.util.executable import ProcessError -@pytest.fixture(scope='function') -def testing_gpg_directory(tmpdir): - old_gpg_path = gpg_util.GNUPGHOME - gpg_util.GNUPGHOME = str(tmpdir.join('gpg')) - yield - gpg_util.GNUPGHOME = old_gpg_path +#: spack command used by tests below +gpg = SpackCommand('gpg') + +# test gpg command detection +@pytest.mark.parametrize('cmd_name,version', [ + ('gpg', 'undetectable'), # undetectable version + ('gpg', 'gpg (GnuPG) 1.3.4'), # insufficient version + ('gpg', 'gpg (GnuPG) 2.2.19'), # sufficient version + ('gpg2', 'gpg (GnuPG) 2.2.19'), # gpg2 command +]) +def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch): + with tmpdir.as_cwd(): + with open(cmd_name, 'w') as f: + f.write("""\ +#!/bin/sh +echo "{version}" +""".format(version=version)) + fs.set_executable(cmd_name) -@pytest.fixture(scope='function') -def gpg(): - return SpackCommand('gpg') + monkeypatch.setitem(os.environ, "PATH", str(tmpdir)) + if version == 'undetectable' or version.endswith('1.3.4'): + with pytest.raises(spack.util.gpg.SpackGPGError): + exe = spack.util.gpg.Gpg.gpg() + else: + exe = spack.util.gpg.Gpg.gpg() + assert isinstance(exe, spack.util.executable.Executable) -def has_gnupg2(): - try: - gpg_util.Gpg.gpg()('--version', output=os.devnull) - return True - except Exception: - return False +def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch): + monkeypatch.setitem(os.environ, "PATH", str(tmpdir)) + with pytest.raises(spack.util.gpg.SpackGPGError): + spack.util.gpg.Gpg.gpg() @pytest.mark.maybeslow -@pytest.mark.skipif(not has_gnupg2(), +@pytest.mark.skipif(not spack.util.gpg.Gpg.gpg(), reason='These tests require gnupg2') -def test_gpg(gpg, tmpdir, testing_gpg_directory): +def test_gpg(tmpdir, mock_gnupghome): # Verify a file with an empty keyring. with pytest.raises(ProcessError): gpg('verify', os.path.join(mock_gpg_data_path, 'content.txt')) @@ -77,7 +95,7 @@ def test_gpg(gpg, tmpdir, testing_gpg_directory): '--export', str(keypath), 'Spack testing 1', 'spack@googlegroups.com') - keyfp = gpg_util.Gpg.signing_keys()[0] + keyfp = spack.util.gpg.Gpg.signing_keys()[0] # List the keys. # TODO: Test the output here. diff --git a/lib/spack/spack/test/cmd/graph.py b/lib/spack/spack/test/cmd/graph.py index 0ddbc4f149..c402f35a9f 100644 --- a/lib/spack/spack/test/cmd/graph.py +++ b/lib/spack/spack/test/cmd/graph.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/help.py b/lib/spack/spack/test/cmd/help.py index c733474fda..00edfa187a 100644 --- a/lib/spack/spack/test/cmd/help.py +++ b/lib/spack/spack/test/cmd/help.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/info.py b/lib/spack/spack/test/cmd/info.py index 1928794560..b57ef3257b 100644 --- a/lib/spack/spack/test/cmd/info.py +++ b/lib/spack/spack/test/cmd/info.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index db8cf01f48..4c6c0860aa 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -30,14 +30,6 @@ env = SpackCommand('env') add = SpackCommand('add') -@pytest.fixture(scope='module') -def parser(): - """Returns the parser for the module command""" - parser = argparse.ArgumentParser() - spack.cmd.install.setup_parser(parser) - return parser - - @pytest.fixture() def noop_install(monkeypatch): def noop(*args, **kwargs): @@ -115,7 +107,9 @@ def test_install_package_already_installed( (['--clean'], False), (['--dirty'], True), ]) -def test_install_dirty_flag(parser, arguments, expected): +def test_install_dirty_flag(arguments, expected): + parser = argparse.ArgumentParser() + spack.cmd.install.setup_parser(parser) args = parser.parse_args(arguments) assert args.dirty == expected @@ -664,3 +658,32 @@ def test_install_only_dependencies_of_all_in_env( assert not os.path.exists(root.prefix) for dep in root.traverse(root=False): assert os.path.exists(dep.prefix) + + +def test_install_help_does_not_show_cdash_options(capsys): + """Make sure `spack install --help` does not describe CDash arguments""" + with pytest.raises(SystemExit): + install('--help') + captured = capsys.readouterr() + assert 'CDash URL' not in captured.out + + +def test_install_help_cdash(capsys): + """Make sure `spack install --help-cdash` describes CDash arguments""" + install_cmd = SpackCommand('install') + out = install_cmd('--help-cdash') + assert 'CDash URL' in out + + +@pytest.mark.disable_clean_stage_check +def test_cdash_auth_token(tmpdir, install_mockery, capfd): + # capfd interferes with Spack's capturing + with tmpdir.as_cwd(): + with capfd.disabled(): + os.environ['SPACK_CDASH_AUTH_TOKEN'] = 'asdf' + out = install( + '-v', + '--log-file=cdash_reports', + '--log-format=cdash', + 'a') + assert 'Using CDash auth token from environment' in out diff --git a/lib/spack/spack/test/cmd/license.py b/lib/spack/spack/test/cmd/license.py index 630b93f68b..614d093c5f 100644 --- a/lib/spack/spack/test/cmd/license.py +++ b/lib/spack/spack/test/cmd/license.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -31,7 +31,7 @@ def test_verify(tmpdir): lgpl_header = source_dir.join('lgpl_header.py') with lgpl_header.open('w') as f: f.write("""\ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: LGPL-2.1-only @@ -48,7 +48,7 @@ def test_verify(tmpdir): correct_header = source_dir.join('correct_header.py') with correct_header.open('w') as f: f.write("""\ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/list.py b/lib/spack/spack/test/cmd/list.py index 6590b9ab2a..5d18787bc7 100644 --- a/lib/spack/spack/test/cmd/list.py +++ b/lib/spack/spack/test/cmd/list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/load.py b/lib/spack/spack/test/cmd/load.py new file mode 100644 index 0000000000..a10b99d45b --- /dev/null +++ b/lib/spack/spack/test/cmd/load.py @@ -0,0 +1,125 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os +from spack.main import SpackCommand +import spack.spec +import spack.user_environment as uenv + +load = SpackCommand('load') +unload = SpackCommand('unload') +install = SpackCommand('install') +location = SpackCommand('location') + + +def test_load(install_mockery, mock_fetch, mock_archive, mock_packages): + """Test that the commands generated by load add the specified prefix + inspections. Also test that Spack records loaded specs by hash in the + user environment. + + CMAKE_PREFIX_PATH is the only prefix inspection guaranteed for fake + packages, since it keys on the prefix instead of a subdir.""" + install('mpileaks') + mpileaks_spec = spack.spec.Spec('mpileaks').concretized() + + sh_out = load('--sh', '--only', 'package', 'mpileaks') + csh_out = load('--csh', '--only', 'package', 'mpileaks') + + # Test prefix inspections + sh_out_test = 'export CMAKE_PREFIX_PATH=%s' % mpileaks_spec.prefix + csh_out_test = 'setenv CMAKE_PREFIX_PATH %s' % mpileaks_spec.prefix + assert sh_out_test in sh_out + assert csh_out_test in csh_out + + # Test hashes recorded properly + hash_test_replacements = (uenv.spack_loaded_hashes_var, + mpileaks_spec.dag_hash()) + sh_hash_test = 'export %s=%s' % hash_test_replacements + csh_hash_test = 'setenv %s %s' % hash_test_replacements + assert sh_hash_test in sh_out + assert csh_hash_test in csh_out + + +def test_load_recursive(install_mockery, mock_fetch, mock_archive, + mock_packages): + """Test that the '-r' option to the load command prepends dependency prefix + inspections in post-order""" + install('mpileaks') + mpileaks_spec = spack.spec.Spec('mpileaks').concretized() + + sh_out = load('--sh', 'mpileaks') + csh_out = load('--csh', 'mpileaks') + + # Test prefix inspections + prefix_test_replacement = ':'.join(reversed( + [s.prefix for s in mpileaks_spec.traverse(order='post')])) + + sh_prefix_test = 'export CMAKE_PREFIX_PATH=%s' % prefix_test_replacement + csh_prefix_test = 'setenv CMAKE_PREFIX_PATH %s' % prefix_test_replacement + assert sh_prefix_test in sh_out + assert csh_prefix_test in csh_out + + # Test spack records loaded hashes properly + hash_test_replacement = (uenv.spack_loaded_hashes_var, ':'.join(reversed( + [s.dag_hash() for s in mpileaks_spec.traverse(order='post')]))) + sh_hash_test = 'export %s=%s' % hash_test_replacement + csh_hash_test = 'setenv %s %s' % hash_test_replacement + assert sh_hash_test in sh_out + assert csh_hash_test in csh_out + + +def test_load_includes_run_env(install_mockery, mock_fetch, mock_archive, + mock_packages): + """Tests that environment changes from the package's + `setup_run_environment` method are added to the user environment in + addition to the prefix inspections""" + install('mpileaks') + + sh_out = load('--sh', 'mpileaks') + csh_out = load('--csh', 'mpileaks') + + assert 'export FOOBAR=mpileaks' in sh_out + assert 'setenv FOOBAR mpileaks' in csh_out + + +def test_load_fails_no_shell(install_mockery, mock_fetch, mock_archive, + mock_packages): + """Test that spack load prints an error message without a shell.""" + install('mpileaks') + + out = load('mpileaks', fail_on_error=False) + assert "To initialize spack's shell commands" in out + + +def test_unload(install_mockery, mock_fetch, mock_archive, mock_packages, + working_env): + """Tests that any variables set in the user environment are undone by the + unload command""" + install('mpileaks') + mpileaks_spec = spack.spec.Spec('mpileaks').concretized() + + # Set so unload has something to do + os.environ['FOOBAR'] = 'mpileaks' + os.environ[uenv.spack_loaded_hashes_var] = '%s:%s' % ( + mpileaks_spec.dag_hash(), 'garbage') + + sh_out = unload('--sh', 'mpileaks') + csh_out = unload('--csh', 'mpileaks') + + assert 'unset FOOBAR' in sh_out + assert 'unsetenv FOOBAR' in csh_out + + assert 'export %s=garbage' % uenv.spack_loaded_hashes_var in sh_out + assert 'setenv %s garbage' % uenv.spack_loaded_hashes_var in csh_out + + +def test_unload_fails_no_shell(install_mockery, mock_fetch, mock_archive, + mock_packages, working_env): + """Test that spack unload prints an error message without a shell.""" + install('mpileaks') + mpileaks_spec = spack.spec.Spec('mpileaks').concretized() + os.environ[uenv.spack_loaded_hashes_var] = mpileaks_spec.dag_hash() + + out = unload('mpileaks', fail_on_error=False) + assert "To initialize spack's shell commands" in out diff --git a/lib/spack/spack/test/cmd/location.py b/lib/spack/spack/test/cmd/location.py index f48cdb75e7..3977cba3e5 100644 --- a/lib/spack/spack/test/cmd/location.py +++ b/lib/spack/spack/test/cmd/location.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/maintainers.py b/lib/spack/spack/test/cmd/maintainers.py index 5ddf176c39..4c566c7513 100644 --- a/lib/spack/spack/test/cmd/maintainers.py +++ b/lib/spack/spack/test/cmd/maintainers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/mirror.py b/lib/spack/spack/test/cmd/mirror.py index 889d81f98b..d62d7df432 100644 --- a/lib/spack/spack/test/cmd/mirror.py +++ b/lib/spack/spack/test/cmd/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,7 +6,7 @@ import pytest import os -from spack.main import SpackCommand +from spack.main import SpackCommand, SpackCommandError import spack.environment as ev import spack.config @@ -16,6 +16,25 @@ add = SpackCommand('add') concretize = SpackCommand('concretize') +@pytest.fixture +def tmp_scope(): + """Creates a temporary configuration scope""" + + base_name = 'internal-testing-scope' + current_overrides = set( + x.name for x in + spack.config.config.matching_scopes(r'^{0}'.format(base_name))) + + num_overrides = 0 + scope_name = base_name + while scope_name in current_overrides: + scope_name = '{0}{1}'.format(base_name, num_overrides) + num_overrides += 1 + + with spack.config.override(spack.config.InternalConfigScope(scope_name)): + yield scope_name + + @pytest.mark.disable_clean_stage_check @pytest.mark.regression('8083') def test_regression_8083(tmpdir, capfd, mock_packages, mock_fetch, config): @@ -45,3 +64,49 @@ def test_mirror_from_env(tmpdir, mock_packages, mock_fetch, config, mirror_res = os.listdir(os.path.join(mirror_dir, spec.name)) expected = ['%s.tar.gz' % spec.format('{name}-{version}')] assert mirror_res == expected + + +def test_mirror_crud(tmp_scope, capsys): + with capsys.disabled(): + mirror('add', '--scope', tmp_scope, 'mirror', 'http://spack.io') + + output = mirror('remove', '--scope', tmp_scope, 'mirror') + assert 'Removed mirror' in output + + mirror('add', '--scope', tmp_scope, 'mirror', 'http://spack.io') + + # no-op + output = mirror('set-url', '--scope', tmp_scope, + 'mirror', 'http://spack.io') + assert 'Url already set' in output + + output = mirror('set-url', '--scope', tmp_scope, + '--push', 'mirror', 's3://spack-public') + assert 'Changed (push) url' in output + + # no-op + output = mirror('set-url', '--scope', tmp_scope, + '--push', 'mirror', 's3://spack-public') + assert 'Url already set' in output + + output = mirror('remove', '--scope', tmp_scope, 'mirror') + assert 'Removed mirror' in output + + output = mirror('list', '--scope', tmp_scope) + assert 'No mirrors configured' in output + + +def test_mirror_nonexisting(tmp_scope): + with pytest.raises(SpackCommandError): + mirror('remove', '--scope', tmp_scope, 'not-a-mirror') + + with pytest.raises(SpackCommandError): + mirror('set-url', '--scope', tmp_scope, + 'not-a-mirror', 'http://spack.io') + + +def test_mirror_name_collision(tmp_scope): + mirror('add', '--scope', tmp_scope, 'first', '1') + + with pytest.raises(SpackCommandError): + mirror('add', '--scope', tmp_scope, 'first', '1') diff --git a/lib/spack/spack/test/cmd/module.py b/lib/spack/spack/test/cmd/module.py index 6f2ffcbf85..f9aeb4af66 100644 --- a/lib/spack/spack/test/cmd/module.py +++ b/lib/spack/spack/test/cmd/module.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -10,9 +10,21 @@ import pytest import spack.main import spack.modules +from spack.test.conftest import use_store, use_configuration, use_repo module = spack.main.SpackCommand('module') +#: make sure module files are generated for all the tests here +@pytest.fixture(scope='module', autouse=True) +def ensure_module_files_are_there( + mock_repo_path, mock_store, mock_configuration): + """Generate module files for module tests.""" + module = spack.main.SpackCommand('module') + with use_store(mock_store): + with use_configuration(mock_configuration): + with use_repo(mock_repo_path): + module('tcl', 'refresh', '-y') + def _module_files(module_type, *specs): specs = [spack.spec.Spec(x).concretized() for x in specs] @@ -20,11 +32,6 @@ def _module_files(module_type, *specs): return [writer_cls(spec).layout.filename for spec in specs] -@pytest.fixture(scope='module', autouse=True) -def ensure_module_files_are_there(database): - module('tcl', 'refresh', '-y') - - @pytest.fixture( params=[ ['rm', 'doesnotexist'], # Try to remove a non existing module diff --git a/lib/spack/spack/test/cmd/pkg.py b/lib/spack/spack/test/cmd/pkg.py index e7d1006cc1..fa324272bf 100644 --- a/lib/spack/spack/test/cmd/pkg.py +++ b/lib/spack/spack/test/cmd/pkg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/print_shell_vars.py b/lib/spack/spack/test/cmd/print_shell_vars.py index e5522ea016..5f4f8164eb 100644 --- a/lib/spack/spack/test/cmd/print_shell_vars.py +++ b/lib/spack/spack/test/cmd/print_shell_vars.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/providers.py b/lib/spack/spack/test/cmd/providers.py index f10faeeb90..3d537c553f 100644 --- a/lib/spack/spack/test/cmd/providers.py +++ b/lib/spack/spack/test/cmd/providers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/python.py b/lib/spack/spack/test/cmd/python.py index c4988f4269..074c295622 100644 --- a/lib/spack/spack/test/cmd/python.py +++ b/lib/spack/spack/test/cmd/python.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/reindex.py b/lib/spack/spack/test/cmd/reindex.py index 0ee36b3001..feb5545ae6 100644 --- a/lib/spack/spack/test/cmd/reindex.py +++ b/lib/spack/spack/test/cmd/reindex.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/release_jobs.py b/lib/spack/spack/test/cmd/release_jobs.py deleted file mode 100644 index 0dbe8c5d5c..0000000000 --- a/lib/spack/spack/test/cmd/release_jobs.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os -import pytest -import re - -import spack -import spack.environment as ev -from spack import repo -from spack.cmd.release_jobs import stage_spec_jobs, spec_deps_key_label -from spack.main import SpackCommand -from spack.spec import Spec -from spack.test.conftest import MockPackage, MockPackageMultiRepo - - -env = SpackCommand('env') -release_jobs = SpackCommand('release-jobs') - - -@pytest.fixture() -def env_deactivate(): - yield - spack.environment._active_environment = None - os.environ.pop('SPACK_ENV', None) - - -def test_specs_staging(config): - """Make sure we achieve the best possible staging for the following -spec DAG:: - - a - /| - c b - |\ - e d - |\ - f g - -In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage, -and then 'd', 'b', and 'a' to be put in the next three stages, respectively. - -""" - default = ('build', 'link') - - g = MockPackage('g', [], []) - f = MockPackage('f', [], []) - e = MockPackage('e', [], []) - d = MockPackage('d', [f, g], [default, default]) - c = MockPackage('c', [], []) - b = MockPackage('b', [d, e], [default, default]) - a = MockPackage('a', [b, c], [default, default]) - - mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g]) - - with repo.swap(mock_repo): - spec_a = Spec('a') - spec_a.concretize() - - spec_a_label = spec_deps_key_label(spec_a)[1] - spec_b_label = spec_deps_key_label(spec_a['b'])[1] - spec_c_label = spec_deps_key_label(spec_a['c'])[1] - spec_d_label = spec_deps_key_label(spec_a['d'])[1] - spec_e_label = spec_deps_key_label(spec_a['e'])[1] - spec_f_label = spec_deps_key_label(spec_a['f'])[1] - spec_g_label = spec_deps_key_label(spec_a['g'])[1] - - spec_labels, dependencies, stages = stage_spec_jobs([spec_a]) - - assert (len(stages) == 4) - - assert (len(stages[0]) == 4) - assert (spec_c_label in stages[0]) - assert (spec_e_label in stages[0]) - assert (spec_f_label in stages[0]) - assert (spec_g_label in stages[0]) - - assert (len(stages[1]) == 1) - assert (spec_d_label in stages[1]) - - assert (len(stages[2]) == 1) - assert (spec_b_label in stages[2]) - - assert (len(stages[3]) == 1) - assert (spec_a_label in stages[3]) - - -def test_release_jobs_with_env(tmpdir, mutable_mock_env_path, env_deactivate, - install_mockery, mock_packages): - """Make sure we can get a .gitlab-ci.yml from an environment file - which has the gitlab-ci, cdash, and mirrors sections.""" - filename = str(tmpdir.join('spack.yaml')) - with open(filename, 'w') as f: - f.write("""\ -spack: - definitions: - - packages: [archive-files] - specs: - - $packages - mirrors: - some-mirror: https://my.fake.mirror - gitlab-ci: - mappings: - - match: - - archive-files - runner-attributes: - tags: - - donotcare - image: donotcare - cdash: - build-group: Not important - url: https://my.fake.cdash - project: Not used - site: Nothing -""") - with tmpdir.as_cwd(): - env('create', 'test', './spack.yaml') - outputfile = str(tmpdir.join('.gitlab-ci.yml')) - - with ev.read('test'): - release_jobs('--output-file', outputfile) - - with open(outputfile) as f: - contents = f.read().replace(os.linesep, '') - assert('archive-files' in contents) - assert(re.search(r'stages:\s*\[\s*stage-0', contents)) diff --git a/lib/spack/spack/test/cmd/resource.py b/lib/spack/spack/test/cmd/resource.py index 62fa1c2d11..4fc5fd5c50 100644 --- a/lib/spack/spack/test/cmd/resource.py +++ b/lib/spack/spack/test/cmd/resource.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/spec.py b/lib/spack/spack/test/cmd/spec.py index 4637e14a1f..3c50a66ce7 100644 --- a/lib/spack/spack/test/cmd/spec.py +++ b/lib/spack/spack/test/cmd/spec.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,7 +9,7 @@ import pytest import spack.spec from spack.main import SpackCommand -pytestmark = pytest.mark.usefixtures('config', 'mutable_mock_packages') +pytestmark = pytest.mark.usefixtures('config', 'mutable_mock_repo') spec = SpackCommand('spec') diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py new file mode 100644 index 0000000000..9a64209cfa --- /dev/null +++ b/lib/spack/spack/test/cmd/test.py @@ -0,0 +1,94 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.main import SpackCommand + +spack_test = SpackCommand('test') + + +def test_list(): + output = spack_test('--list') + assert "test.py" in output + assert "spec_semantics.py" in output + assert "test_list" not in output + + +def test_list_with_pytest_arg(): + output = spack_test('--list', 'cmd/test.py') + assert output.strip() == "cmd/test.py" + + +def test_list_with_keywords(): + output = spack_test('--list', '-k', 'cmd/test.py') + assert output.strip() == "cmd/test.py" + + +def test_list_long(capsys): + with capsys.disabled(): + output = spack_test('--list-long') + assert "test.py::\n" in output + assert "test_list" in output + assert "test_list_with_pytest_arg" in output + assert "test_list_with_keywords" in output + assert "test_list_long" in output + assert "test_list_long_with_pytest_arg" in output + assert "test_list_names" in output + assert "test_list_names_with_pytest_arg" in output + + assert "spec_dag.py::\n" in output + assert 'test_installed_deps' in output + assert 'test_test_deptype' in output + + +def test_list_long_with_pytest_arg(capsys): + with capsys.disabled(): + output = spack_test('--list-long', 'cmd/test.py') + assert "test.py::\n" in output + assert "test_list" in output + assert "test_list_with_pytest_arg" in output + assert "test_list_with_keywords" in output + assert "test_list_long" in output + assert "test_list_long_with_pytest_arg" in output + assert "test_list_names" in output + assert "test_list_names_with_pytest_arg" in output + + assert "spec_dag.py::\n" not in output + assert 'test_installed_deps' not in output + assert 'test_test_deptype' not in output + + +def test_list_names(): + output = spack_test('--list-names') + assert "test.py::test_list\n" in output + assert "test.py::test_list_with_pytest_arg\n" in output + assert "test.py::test_list_with_keywords\n" in output + assert "test.py::test_list_long\n" in output + assert "test.py::test_list_long_with_pytest_arg\n" in output + assert "test.py::test_list_names\n" in output + assert "test.py::test_list_names_with_pytest_arg\n" in output + + assert "spec_dag.py::test_installed_deps\n" in output + assert 'spec_dag.py::test_test_deptype\n' in output + + +def test_list_names_with_pytest_arg(): + output = spack_test('--list-names', 'cmd/test.py') + assert "test.py::test_list\n" in output + assert "test.py::test_list_with_pytest_arg\n" in output + assert "test.py::test_list_with_keywords\n" in output + assert "test.py::test_list_long\n" in output + assert "test.py::test_list_long_with_pytest_arg\n" in output + assert "test.py::test_list_names\n" in output + assert "test.py::test_list_names_with_pytest_arg\n" in output + + assert "spec_dag.py::test_installed_deps\n" not in output + assert 'spec_dag.py::test_test_deptype\n' not in output + + +def test_pytest_help(): + output = spack_test('--pytest-help') + assert "-k EXPRESSION" in output + assert "pytest-warnings:" in output + assert "--collect-only" in output diff --git a/lib/spack/spack/test/cmd/test_compiler_cmd.py b/lib/spack/spack/test/cmd/test_compiler_cmd.py index 381ff12ae9..3c34f720ec 100644 --- a/lib/spack/spack/test/cmd/test_compiler_cmd.py +++ b/lib/spack/spack/test/cmd/test_compiler_cmd.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py index c9a23c48cf..882f075ec7 100644 --- a/lib/spack/spack/test/cmd/uninstall.py +++ b/lib/spack/spack/test/cmd/uninstall.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/url.py b/lib/spack/spack/test/cmd/url.py index 6593c94d98..54ef3d55cf 100644 --- a/lib/spack/spack/test/cmd/url.py +++ b/lib/spack/spack/test/cmd/url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/verify.py b/lib/spack/spack/test/cmd/verify.py index 4896cb05c8..dda992fe49 100644 --- a/lib/spack/spack/test/cmd/verify.py +++ b/lib/spack/spack/test/cmd/verify.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/versions.py b/lib/spack/spack/test/cmd/versions.py index 07761be1d4..1e2bcbedb3 100644 --- a/lib/spack/spack/test/cmd/versions.py +++ b/lib/spack/spack/test/cmd/versions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd/view.py b/lib/spack/spack/test/cmd/view.py index f7375d8903..c52cd12325 100644 --- a/lib/spack/spack/test/cmd/view.py +++ b/lib/spack/spack/test/cmd/view.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/cmd_extensions.py b/lib/spack/spack/test/cmd_extensions.py index bb018eedb5..486ac925cf 100644 --- a/lib/spack/spack/test/cmd_extensions.py +++ b/lib/spack/spack/test/cmd_extensions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/compilers.py b/lib/spack/spack/test/compilers.py index ac107170b0..9395ddba80 100644 --- a/lib/spack/spack/test/compilers.py +++ b/lib/spack/spack/test/compilers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -348,6 +348,14 @@ def test_fj_flags(): ('clang version 8.0.0-3~ubuntu18.04.1 (tags/RELEASE_800/final)\n' 'Target: x86_64-pc-linux-gnu\n' 'Thread model: posix\n' + 'InstalledDir: /usr/bin\n', '8.0.0'), + ('clang version 9.0.1-+201911131414230800840845a1eea-1~exp1~20191113231141.78\n' # noqa + 'Target: x86_64-pc-linux-gnu\n' + 'Thread model: posix\n' + 'InstalledDir: /usr/bin\n', '9.0.1'), + ('clang version 8.0.0-3 (tags/RELEASE_800/final)\n' + 'Target: aarch64-unknown-linux-gnu\n' + 'Thread model: posix\n' 'InstalledDir: /usr/bin\n', '8.0.0') ]) def test_clang_version_detection(version_str, expected_version): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 4b17e755ed..72092a956f 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,9 +11,9 @@ import spack.concretize import spack.repo from spack.concretize import find_spec, NoValidVersionError +from spack.error import SpecError from spack.package_prefs import PackagePrefs -from spack.spec import Spec, CompilerSpec -from spack.spec import ConflictsInSpecError, SpecError +from spack.spec import Spec, CompilerSpec, ConflictsInSpecError from spack.version import ver from spack.test.conftest import MockPackage, MockPackageMultiRepo import spack.compilers @@ -276,7 +276,7 @@ class TestConcretize(object): Spec('hypre').concretize() def test_concretize_two_virtuals_with_one_bound( - self, mutable_mock_packages + self, mutable_mock_repo ): """Test a package with multiple virtual dependencies and one preset.""" Spec('hypre ^openblas').concretize() @@ -302,7 +302,7 @@ class TestConcretize(object): with pytest.raises(spack.spec.MultipleProviderError): s.concretize() - def test_no_matching_compiler_specs(self, mock_config): + def test_no_matching_compiler_specs(self, mock_low_high_config): # only relevant when not building compilers as needed with spack.concretize.enable_compiler_existence_check(): s = Spec('a %gcc@0.0.0') diff --git a/lib/spack/spack/test/concretize_preferences.py b/lib/spack/spack/test/concretize_preferences.py index e5fa4a851d..0d0d870506 100644 --- a/lib/spack/spack/test/concretize_preferences.py +++ b/lib/spack/spack/test/concretize_preferences.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -11,6 +11,7 @@ import spack.repo import spack.util.spack_yaml as syaml from spack.config import ConfigScope, ConfigError from spack.spec import Spec +from spack.version import Version @pytest.fixture() @@ -23,7 +24,6 @@ def concretize_scope(config, tmpdir): yield config.pop_scope() - spack.package_prefs.PackagePrefs.clear_caches() spack.repo.path._provider_index = None @@ -60,7 +60,6 @@ def update_packages(pkgname, section, value): """Update config and reread package list""" conf = {pkgname: {section: value}} spack.config.set('packages', conf, scope='concretize') - spack.package_prefs.PackagePrefs.clear_caches() def assert_variant_values(spec, **variants): @@ -85,7 +84,7 @@ class TestConcretizePreferences(object): 'mpileaks', debug=True, opt=True, shared=False, static=False ) - def test_preferred_compilers(self, mutable_mock_packages): + def test_preferred_compilers(self, mutable_mock_repo): """Test preferred compilers are applied correctly """ update_packages('mpileaks', 'compiler', ['clang@3.3']) @@ -96,7 +95,7 @@ class TestConcretizePreferences(object): spec = concretize('mpileaks') assert spec.compiler == spack.spec.CompilerSpec('gcc@4.5.0') - def test_preferred_target(self, mutable_mock_packages): + def test_preferred_target(self, mutable_mock_repo): """Test preferred compilers are applied correctly """ spec = concretize('mpich') @@ -121,16 +120,16 @@ class TestConcretizePreferences(object): """ update_packages('mpileaks', 'version', ['2.3']) spec = concretize('mpileaks') - assert spec.version == spack.spec.Version('2.3') + assert spec.version == Version('2.3') update_packages('mpileaks', 'version', ['2.2']) spec = concretize('mpileaks') - assert spec.version == spack.spec.Version('2.2') + assert spec.version == Version('2.2') def test_preferred_versions_mixed_version_types(self): update_packages('mixedversions', 'version', ['2.0']) spec = concretize('mixedversions') - assert spec.version == spack.spec.Version('2.0') + assert spec.version == Version('2.0') def test_preferred_providers(self): """Test preferred providers of virtual packages are @@ -148,35 +147,35 @@ class TestConcretizePreferences(object): """"Test packages with some version marked as preferred=True""" spec = Spec('preferred-test') spec.concretize() - assert spec.version == spack.spec.Version('0.2.15') + assert spec.version == Version('0.2.15') # now add packages.yaml with versions other than preferred # ensure that once config is in place, non-preferred version is used update_packages('preferred-test', 'version', ['0.2.16']) spec = Spec('preferred-test') spec.concretize() - assert spec.version == spack.spec.Version('0.2.16') + assert spec.version == Version('0.2.16') def test_develop(self): """Test concretization with develop-like versions""" spec = Spec('develop-test') spec.concretize() - assert spec.version == spack.spec.Version('0.2.15') + assert spec.version == Version('0.2.15') spec = Spec('develop-test2') spec.concretize() - assert spec.version == spack.spec.Version('0.2.15') + assert spec.version == Version('0.2.15') # now add packages.yaml with develop-like versions # ensure that once config is in place, develop-like version is used update_packages('develop-test', 'version', ['develop']) spec = Spec('develop-test') spec.concretize() - assert spec.version == spack.spec.Version('develop') + assert spec.version == Version('develop') update_packages('develop-test2', 'version', ['0.2.15.develop']) spec = Spec('develop-test2') spec.concretize() - assert spec.version == spack.spec.Version('0.2.15.develop') + assert spec.version == Version('0.2.15.develop') def test_no_virtuals_in_packages_yaml(self): """Verify that virtuals are not allowed in packages.yaml.""" @@ -204,7 +203,6 @@ all: spack.config.set('packages', conf, scope='concretize') # should be no error for 'all': - spack.package_prefs.PackagePrefs.clear_caches() spack.package_prefs.get_packages_config() def test_external_mpi(self): diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 605c37d468..feb2b9cae4 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -203,7 +203,7 @@ def compiler_specs(): return CompilerSpecs(a=a, b=b) -def test_write_key_in_memory(mock_config, compiler_specs): +def test_write_key_in_memory(mock_low_high_config, compiler_specs): # Write b_comps "on top of" a_comps. spack.config.set('compilers', a_comps['compilers'], scope='low') spack.config.set('compilers', b_comps['compilers'], scope='high') @@ -213,7 +213,7 @@ def test_write_key_in_memory(mock_config, compiler_specs): check_compiler_config(b_comps['compilers'], *compiler_specs.b) -def test_write_key_to_disk(mock_config, compiler_specs): +def test_write_key_to_disk(mock_low_high_config, compiler_specs): # Write b_comps "on top of" a_comps. spack.config.set('compilers', a_comps['compilers'], scope='low') spack.config.set('compilers', b_comps['compilers'], scope='high') @@ -226,7 +226,7 @@ def test_write_key_to_disk(mock_config, compiler_specs): check_compiler_config(b_comps['compilers'], *compiler_specs.b) -def test_write_to_same_priority_file(mock_config, compiler_specs): +def test_write_to_same_priority_file(mock_low_high_config, compiler_specs): # Write b_comps in the same file as a_comps. spack.config.set('compilers', a_comps['compilers'], scope='low') spack.config.set('compilers', b_comps['compilers'], scope='low') @@ -247,7 +247,7 @@ repos_high = {'repos': ["/some/other/path"]} # repos -def test_write_list_in_memory(mock_config): +def test_write_list_in_memory(mock_low_high_config): spack.config.set('repos', repos_low['repos'], scope='low') spack.config.set('repos', repos_high['repos'], scope='high') @@ -255,7 +255,7 @@ def test_write_list_in_memory(mock_config): assert config == repos_high['repos'] + repos_low['repos'] -def test_substitute_config_variables(mock_config): +def test_substitute_config_variables(mock_low_high_config): prefix = spack.paths.prefix.lstrip('/') assert os.path.join( @@ -315,7 +315,7 @@ packages_merge_high = { @pytest.mark.regression('7924') -def test_merge_with_defaults(mock_config, write_config_file): +def test_merge_with_defaults(mock_low_high_config, write_config_file): """This ensures that specified preferences merge with defaults as expected. Originally all defaults were initialized with the exact same object, which led to aliasing problems. Therefore @@ -331,14 +331,14 @@ def test_merge_with_defaults(mock_config, write_config_file): assert cfg['baz']['version'] == ['c'] -def test_substitute_user(mock_config): +def test_substitute_user(mock_low_high_config): user = getpass.getuser() assert '/foo/bar/' + user + '/baz' == canonicalize_path( '/foo/bar/$user/baz' ) -def test_substitute_tempdir(mock_config): +def test_substitute_tempdir(mock_low_high_config): tempdir = tempfile.gettempdir() assert tempdir == canonicalize_path('$tempdir') assert tempdir + '/foo/bar/baz' == canonicalize_path( @@ -346,12 +346,12 @@ def test_substitute_tempdir(mock_config): ) -def test_read_config(mock_config, write_config_file): +def test_read_config(mock_low_high_config, write_config_file): write_config_file('config', config_low, 'low') assert spack.config.get('config') == config_low['config'] -def test_read_config_override_all(mock_config, write_config_file): +def test_read_config_override_all(mock_low_high_config, write_config_file): write_config_file('config', config_low, 'low') write_config_file('config', config_override_all, 'high') assert spack.config.get('config') == { @@ -359,7 +359,7 @@ def test_read_config_override_all(mock_config, write_config_file): } -def test_read_config_override_key(mock_config, write_config_file): +def test_read_config_override_key(mock_low_high_config, write_config_file): write_config_file('config', config_low, 'low') write_config_file('config', config_override_key, 'high') assert spack.config.get('config') == { @@ -368,7 +368,7 @@ def test_read_config_override_key(mock_config, write_config_file): } -def test_read_config_merge_list(mock_config, write_config_file): +def test_read_config_merge_list(mock_low_high_config, write_config_file): write_config_file('config', config_low, 'low') write_config_file('config', config_merge_list, 'high') assert spack.config.get('config') == { @@ -377,7 +377,7 @@ def test_read_config_merge_list(mock_config, write_config_file): } -def test_read_config_override_list(mock_config, write_config_file): +def test_read_config_override_list(mock_low_high_config, write_config_file): write_config_file('config', config_low, 'low') write_config_file('config', config_override_list, 'high') assert spack.config.get('config') == { @@ -386,33 +386,34 @@ def test_read_config_override_list(mock_config, write_config_file): } -def test_internal_config_update(mock_config, write_config_file): +def test_internal_config_update(mock_low_high_config, write_config_file): write_config_file('config', config_low, 'low') - before = mock_config.get('config') + before = mock_low_high_config.get('config') assert before['install_tree'] == 'install_tree_path' # add an internal configuration scope scope = spack.config.InternalConfigScope('command_line') assert 'InternalConfigScope' in repr(scope) - mock_config.push_scope(scope) + mock_low_high_config.push_scope(scope) - command_config = mock_config.get('config', scope='command_line') + command_config = mock_low_high_config.get('config', scope='command_line') command_config['install_tree'] = 'foo/bar' - mock_config.set('config', command_config, scope='command_line') + mock_low_high_config.set('config', command_config, scope='command_line') - after = mock_config.get('config') + after = mock_low_high_config.get('config') assert after['install_tree'] == 'foo/bar' -def test_internal_config_filename(mock_config, write_config_file): +def test_internal_config_filename(mock_low_high_config, write_config_file): write_config_file('config', config_low, 'low') - mock_config.push_scope(spack.config.InternalConfigScope('command_line')) + mock_low_high_config.push_scope( + spack.config.InternalConfigScope('command_line')) with pytest.raises(NotImplementedError): - mock_config.get_config_filename('command_line', 'config') + mock_low_high_config.get_config_filename('command_line', 'config') def test_mark_internal(): @@ -585,7 +586,7 @@ mirrors: assert "mirrors.yaml:5" in str(e) -def test_bad_config_section(mock_config): +def test_bad_config_section(mock_low_high_config): """Test that getting or setting a bad section gives an error.""" with pytest.raises(spack.config.ConfigSectionError): spack.config.set('foobar', 'foobar') @@ -595,7 +596,7 @@ def test_bad_config_section(mock_config): @pytest.mark.skipif(os.getuid() == 0, reason='user is root') -def test_bad_command_line_scopes(tmpdir, mock_config): +def test_bad_command_line_scopes(tmpdir, mock_low_high_config): cfg = spack.config.Configuration() with tmpdir.as_cwd(): @@ -844,7 +845,7 @@ compilers: @pytest.mark.regression('13045') def test_dotkit_in_config_does_not_raise( - mock_config, write_config_file, capsys + mock_low_high_config, write_config_file, capsys ): write_config_file('config', {'config': {'module_roots': {'dotkit': '/some/path'}}}, diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 969e2471e4..97bbb69b52 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1,16 +1,17 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections -import copy +import contextlib import errno import inspect import itertools import os import os.path import shutil +import tempfile import xml.etree.ElementTree import ordereddict_backport @@ -33,6 +34,8 @@ import spack.platforms.test import spack.repo import spack.stage import spack.util.executable +import spack.util.gpg + from spack.util.pattern import Bunch from spack.dependency import Dependency from spack.package import PackageBase @@ -167,7 +170,7 @@ def ignore_stage_files(): Used to track which leftover files in the stage have been seen. """ # to start with, ignore the .lock file at the stage root. - return set(['.lock', spack.stage._source_path_subdir]) + return set(['.lock', spack.stage._source_path_subdir, 'build_cache']) def remove_whatever_it_is(path): @@ -240,9 +243,6 @@ def mock_fetch_cache(monkeypatch): return MockCacheFetcher() class MockCacheFetcher(object): - def set_stage(self, stage): - pass - def fetch(self): raise FetchError('Mock cache always fails for tests') @@ -273,31 +273,75 @@ def _skip_if_missing_executables(request): spack.architecture.real_platform = spack.architecture.platform spack.architecture.platform = lambda: spack.platforms.test.Test() -########## -# Test-specific fixtures -########## + +# +# Context managers used by fixtures +# +# Because these context managers modify global state, they should really +# ONLY be used persistently (i.e., around yield statements) in +# function-scoped fixtures, OR in autouse session- or module-scoped +# fixtures. +# +# If they're used in regular tests or in module-scoped fixtures that are +# then injected as function arguments, weird things can happen, because +# the original state won't be restored until *after* the fixture is +# destroyed. This makes sense for an autouse fixture, where you know +# everything in the module/session is going to need the modified +# behavior, but modifying global state for one function in a way that +# won't be restored until after the module or session is done essentially +# leaves garbage behind for other tests. +# +# In general, we should module- or session-scope the *STATE* required for +# these global objects, but we shouldn't module- or session-scope their +# *USE*, or things can get really confusing. +# + +@contextlib.contextmanager +def use_configuration(config): + """Context manager to swap out the global Spack configuration.""" + saved = spack.config.config + spack.config.config = config + yield + spack.config.config = saved +@contextlib.contextmanager +def use_store(store): + """Context manager to swap out the global Spack store.""" + saved = spack.store.store + spack.store.store = store + yield + spack.store.store = saved + + +@contextlib.contextmanager +def use_repo(repo): + """Context manager to swap out the global Spack repo path.""" + with spack.repo.swap(repo): + yield + + +# +# Test-specific fixtures +# @pytest.fixture(scope='session') -def repo_path(): - """Session scoped RepoPath object pointing to the mock repository""" - return spack.repo.RepoPath(spack.paths.mock_packages_path) +def mock_repo_path(): + yield spack.repo.RepoPath(spack.paths.mock_packages_path) -@pytest.fixture(scope='module') -def mock_packages(repo_path): +@pytest.fixture(scope='function') +def mock_packages(mock_repo_path): """Use the 'builtin.mock' repository instead of 'builtin'""" - mock_repo = copy.deepcopy(repo_path) - with spack.repo.swap(mock_repo): - yield + with use_repo(mock_repo_path): + yield mock_repo_path @pytest.fixture(scope='function') -def mutable_mock_packages(mock_packages, repo_path): +def mutable_mock_repo(mock_repo_path): """Function-scoped mock packages, for tests that need to modify them.""" - mock_repo = copy.deepcopy(repo_path) - with spack.repo.swap(mock_repo): - yield + mock_repo_path = spack.repo.RepoPath(spack.paths.mock_packages_path) + with use_repo(mock_repo_path): + yield mock_repo_path @pytest.fixture(scope='session') @@ -322,23 +366,16 @@ def configuration_dir(tmpdir_factory, linux_os): """ tmpdir = tmpdir_factory.mktemp('configurations') - # Name of the yaml files in the test/data folder - test_path = py.path.local(spack.paths.test_path) - compilers_yaml = test_path.join('data', 'compilers.yaml') - packages_yaml = test_path.join('data', 'packages.yaml') - config_yaml = test_path.join('data', 'config.yaml') - repos_yaml = test_path.join('data', 'repos.yaml') + # <test_path>/data/config has mock config yaml files in it + # copy these to the site config. + test_config = py.path.local(spack.paths.test_path).join('data', 'config') + test_config.copy(tmpdir.join('site')) - # Create temporary 'site' and 'user' folders - tmpdir.ensure('site', dir=True) + # Create temporary 'defaults', 'site' and 'user' folders tmpdir.ensure('user', dir=True) - # Copy the configurations that don't need further work - packages_yaml.copy(tmpdir.join('site', 'packages.yaml')) - config_yaml.copy(tmpdir.join('site', 'config.yaml')) - repos_yaml.copy(tmpdir.join('site', 'repos.yaml')) - - # Write the one that needs modifications + # Slightly modify compilers.yaml to look like Linux + compilers_yaml = test_config.join('compilers.yaml') content = ''.join(compilers_yaml.read()).format(linux_os) t = tmpdir.join('site', 'compilers.yaml') t.write(content) @@ -348,14 +385,9 @@ def configuration_dir(tmpdir_factory, linux_os): shutil.rmtree(str(tmpdir)) -@pytest.fixture(scope='module') -def config(configuration_dir): - """Hooks the mock configuration files into spack.config""" - # Set up a mock config scope - spack.package_prefs.PackagePrefs.clear_caches() - - real_configuration = spack.config.config - +@pytest.fixture(scope='session') +def mock_configuration(configuration_dir): + """Create a persistent Configuration object from the configuration_dir.""" defaults = spack.config.InternalConfigScope( '_builtin', spack.config.config_defaults ) @@ -365,48 +397,43 @@ def config(configuration_dir): for name in ['site', 'system', 'user']] test_scopes.append(spack.config.InternalConfigScope('command_line')) - spack.config.config = spack.config.Configuration(*test_scopes) + yield spack.config.Configuration(*test_scopes) - yield spack.config.config - spack.config.config = real_configuration - spack.package_prefs.PackagePrefs.clear_caches() +@pytest.fixture(scope='function') +def config(mock_configuration): + """This fixture activates/deactivates the mock configuration.""" + with use_configuration(mock_configuration): + yield mock_configuration @pytest.fixture(scope='function') def mutable_config(tmpdir_factory, configuration_dir, monkeypatch): """Like config, but tests can modify the configuration.""" - spack.package_prefs.PackagePrefs.clear_caches() - mutable_dir = tmpdir_factory.mktemp('mutable_config').join('tmp') configuration_dir.copy(mutable_dir) cfg = spack.config.Configuration( *[spack.config.ConfigScope(name, str(mutable_dir)) for name in ['site', 'system', 'user']]) - monkeypatch.setattr(spack.config, 'config', cfg) # This is essential, otherwise the cache will create weird side effects # that will compromise subsequent tests if compilers.yaml is modified monkeypatch.setattr(spack.compilers, '_cache_config_file', []) - yield spack.config.config - - spack.package_prefs.PackagePrefs.clear_caches() + with use_configuration(cfg): + yield cfg @pytest.fixture() -def mock_config(tmpdir): +def mock_low_high_config(tmpdir): """Mocks two configuration scopes: 'low' and 'high'.""" - real_configuration = spack.config.config - - spack.config.config = spack.config.Configuration( + config = spack.config.Configuration( *[spack.config.ConfigScope(name, str(tmpdir.join(name))) for name in ['low', 'high']]) - yield spack.config.config - - spack.config.config = real_configuration + with use_configuration(config): + yield config def _populate(mock_db): @@ -453,30 +480,41 @@ def _store_dir_and_cache(tmpdir_factory): return store, cache -@pytest.fixture(scope='module') -def database(tmpdir_factory, mock_packages, config, _store_dir_and_cache): +@pytest.fixture(scope='session') +def mock_store(tmpdir_factory, mock_repo_path, mock_configuration, + _store_dir_and_cache): """Creates a read-only mock database with some packages installed note that the ref count for dyninst here will be 3, as it's recycled across each install. + + This does not actually activate the store for use by Spack -- see the + ``database`` fixture for that. + """ - real_store = spack.store.store store_path, store_cache = _store_dir_and_cache - - mock_store = spack.store.Store(str(store_path)) - spack.store.store = mock_store + store = spack.store.Store(str(store_path)) # If the cache does not exist populate the store and create it if not os.path.exists(str(store_cache.join('.spack-db'))): - _populate(mock_store.db) + with use_configuration(mock_configuration): + with use_store(store): + with use_repo(mock_repo_path): + _populate(store.db) store_path.copy(store_cache, mode=True, stat=True) - # Make the database read-only to ensure we can't modify entries + # Make the DB filesystem read-only to ensure we can't modify entries store_path.join('.spack-db').chmod(mode=0o555, rec=1) - yield mock_store.db + yield store store_path.join('.spack-db').chmod(mode=0o755, rec=1) - spack.store.store = real_store + + +@pytest.fixture(scope='function') +def database(mock_store, mock_packages, config): + """This activates the mock store, packages, AND config.""" + with use_store(mock_store): + yield mock_store.db @pytest.fixture(scope='function') @@ -612,15 +650,21 @@ def module_configuration(monkeypatch, request): with open(file) as f: configuration = yaml.load(f) + def mock_config_function(): + return configuration + + def writer_key_function(): + return mock_config_function()[writer_key] + monkeypatch.setattr( spack.modules.common, 'configuration', - configuration + mock_config_function ) monkeypatch.setattr( writer_mod, 'configuration', - configuration[writer_key] + writer_key_function ) monkeypatch.setattr( writer_mod, @@ -629,6 +673,22 @@ def module_configuration(monkeypatch, request): ) return _impl + +@pytest.fixture() +def mock_gnupghome(monkeypatch): + # GNU PGP can't handle paths longer than 108 characters (wtf!@#$) so we + # have to make our own tmpdir with a shorter name than pytest's. + # This comes up because tmp paths on macOS are already long-ish, and + # pytest makes them longer. + short_name_tmpdir = tempfile.mkdtemp() + monkeypatch.setattr(spack.util.gpg, 'GNUPGHOME', short_name_tmpdir) + monkeypatch.setattr(spack.util.gpg.Gpg, '_gpg', None) + + yield + + # clean up, since we are doing this manually + shutil.rmtree(short_name_tmpdir) + ########## # Fake archives and repositories ########## @@ -684,11 +744,31 @@ def mock_archive(request, tmpdir_factory): @pytest.fixture(scope='session') def mock_git_repository(tmpdir_factory): - """Creates a very simple git repository with two branches and - two commits. + """Creates a simple git repository with two branches, + two commits and two submodules. Each submodule has one commit. """ git = spack.util.executable.which('git', required=True) + suburls = [] + for submodule_count in range(2): + tmpdir = tmpdir_factory.mktemp('mock-git-repo-submodule-dir-{0}' + .format(submodule_count)) + tmpdir.ensure(spack.stage._source_path_subdir, dir=True) + repodir = tmpdir.join(spack.stage._source_path_subdir) + suburls.append((submodule_count, 'file://' + str(repodir))) + + # Initialize the repository + with repodir.as_cwd(): + git('init') + git('config', 'user.name', 'Spack') + git('config', 'user.email', 'spack@spack.io') + + # r0 is just the first commit + submodule_file = 'r0_file_{0}'.format(submodule_count) + repodir.ensure(submodule_file) + git('add', submodule_file) + git('commit', '-m', 'mock-git-repo r0 {0}'.format(submodule_count)) + tmpdir = tmpdir_factory.mktemp('mock-git-repo-dir') tmpdir.ensure(spack.stage._source_path_subdir, dir=True) repodir = tmpdir.join(spack.stage._source_path_subdir) @@ -699,6 +779,9 @@ def mock_git_repository(tmpdir_factory): git('config', 'user.name', 'Spack') git('config', 'user.email', 'spack@spack.io') url = 'file://' + str(repodir) + for number, suburl in suburls: + git('submodule', 'add', suburl, + 'third_party/submodule{0}'.format(number)) # r0 is just the first commit r0_file = 'r0_file' diff --git a/lib/spack/spack/test/container/cli.py b/lib/spack/spack/test/container/cli.py new file mode 100644 index 0000000000..8e5403f072 --- /dev/null +++ b/lib/spack/spack/test/container/cli.py @@ -0,0 +1,16 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import llnl.util.filesystem as fs +import spack.main + + +containerize = spack.main.SpackCommand('containerize') + + +def test_command(configuration_dir, capsys): + with capsys.disabled(): + with fs.working_dir(configuration_dir): + output = containerize() + assert 'FROM spack/ubuntu-bionic' in output diff --git a/lib/spack/spack/test/container/conftest.py b/lib/spack/spack/test/container/conftest.py new file mode 100644 index 0000000000..802b34c5f8 --- /dev/null +++ b/lib/spack/spack/test/container/conftest.py @@ -0,0 +1,43 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + +import spack.util.spack_yaml as syaml + + +@pytest.fixture() +def minimal_configuration(): + return { + 'spack': { + 'specs': [ + 'gromacs', + 'mpich', + 'fftw precision=float' + ], + 'container': { + 'format': 'docker', + 'base': { + 'image': 'ubuntu:18.04', + 'spack': 'develop' + } + } + } + } + + +@pytest.fixture() +def config_dumper(tmpdir): + """Function that dumps an environment config in a temporary folder.""" + def dumper(configuration): + content = syaml.dump(configuration, default_flow_style=False) + config_file = tmpdir / 'spack.yaml' + config_file.write(content) + return str(tmpdir) + return dumper + + +@pytest.fixture() +def configuration_dir(minimal_configuration, config_dumper): + return config_dumper(minimal_configuration) diff --git a/lib/spack/spack/test/container/docker.py b/lib/spack/spack/test/container/docker.py new file mode 100644 index 0000000000..fbdc085828 --- /dev/null +++ b/lib/spack/spack/test/container/docker.py @@ -0,0 +1,74 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.container.writers as writers + + +def test_manifest(minimal_configuration): + writer = writers.create(minimal_configuration) + manifest_str = writer.manifest + for line in manifest_str.split('\n'): + assert 'echo' in line + + +def test_build_and_run_images(minimal_configuration): + writer = writers.create(minimal_configuration) + + # Test the output of run property + run = writer.run + assert run.image == 'ubuntu:18.04' + + # Test the output of the build property + build = writer.build + assert build.image == 'spack/ubuntu-bionic' + assert build.tag == 'latest' + + +def test_packages(minimal_configuration): + # In this minimal configuration we don't have packages + writer = writers.create(minimal_configuration) + assert writer.os_packages is None + + # If we add them a list should be returned + pkgs = ['libgomp1'] + minimal_configuration['spack']['container']['os_packages'] = pkgs + writer = writers.create(minimal_configuration) + p = writer.os_packages + assert p.update + assert p.install + assert p.clean + assert p.list == pkgs + + +def test_ensure_render_works(minimal_configuration): + # Here we just want to ensure that nothing is raised + writer = writers.create(minimal_configuration) + writer() + + +def test_strip_is_set_from_config(minimal_configuration): + writer = writers.create(minimal_configuration) + assert writer.strip is True + + minimal_configuration['spack']['container']['strip'] = False + writer = writers.create(minimal_configuration) + assert writer.strip is False + + +def test_extra_instructions_is_set_from_config(minimal_configuration): + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (None, None) + + test_line = 'RUN echo Hello world!' + e = minimal_configuration['spack']['container'] + e['extra_instructions'] = {} + e['extra_instructions']['build'] = test_line + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (test_line, None) + + e['extra_instructions']['final'] = test_line + del e['extra_instructions']['build'] + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (None, test_line) diff --git a/lib/spack/spack/test/container/images.py b/lib/spack/spack/test/container/images.py new file mode 100644 index 0000000000..808676c39a --- /dev/null +++ b/lib/spack/spack/test/container/images.py @@ -0,0 +1,58 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os.path + +import pytest + +import spack.container + + +@pytest.mark.parametrize('image,spack_version,expected', [ + ('ubuntu:18.04', 'develop', ('spack/ubuntu-bionic', 'latest')), + ('ubuntu:18.04', '0.14.0', ('spack/ubuntu-bionic', '0.14.0')), +]) +def test_build_info(image, spack_version, expected): + output = spack.container.images.build_info(image, spack_version) + assert output == expected + + +@pytest.mark.parametrize('image,spack_version', [ + ('ubuntu:18.04', 'doesnotexist') +]) +def test_build_info_error(image, spack_version): + with pytest.raises(ValueError, match=r"has no tag for"): + spack.container.images.build_info(image, spack_version) + + +@pytest.mark.parametrize('image', [ + 'ubuntu:18.04' +]) +def test_package_info(image): + update, install, clean = spack.container.images.package_info(image) + assert update + assert install + assert clean + + +@pytest.mark.parametrize('extra_config,expected_msg', [ + ({'modules': {'enable': ['tcl']}}, 'the subsection "modules" in'), + ({'concretization': 'separately'}, 'the "concretization" attribute'), + ({'config': {'install_tree': '/some/dir'}}, + 'the "config:install_tree" attribute has been set'), + ({'view': '/some/dir'}, 'the "view" attribute has been set') +]) +def test_validate( + extra_config, expected_msg, minimal_configuration, config_dumper +): + minimal_configuration['spack'].update(extra_config) + spack_yaml_dir = config_dumper(minimal_configuration) + spack_yaml = os.path.join(spack_yaml_dir, 'spack.yaml') + + with pytest.warns(UserWarning) as w: + spack.container.validate(spack_yaml) + + # Tests are designed to raise only one warning + assert len(w) == 1 + assert expected_msg in str(w.pop().message) diff --git a/lib/spack/spack/test/container/schema.py b/lib/spack/spack/test/container/schema.py new file mode 100644 index 0000000000..3f33a3f9f7 --- /dev/null +++ b/lib/spack/spack/test/container/schema.py @@ -0,0 +1,16 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.container +import spack.schema.container + + +def test_images_in_schema(): + properties = spack.schema.container.container_schema['properties'] + allowed_images = set( + properties['base']['properties']['image']['enum'] + ) + images_in_json = set(x for x in spack.container.images.data()) + assert images_in_json == allowed_images diff --git a/lib/spack/spack/test/container/singularity.py b/lib/spack/spack/test/container/singularity.py new file mode 100644 index 0000000000..445a119f6c --- /dev/null +++ b/lib/spack/spack/test/container/singularity.py @@ -0,0 +1,42 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + +import spack.container.writers as writers + + +@pytest.fixture +def singularity_configuration(minimal_configuration): + minimal_configuration['spack']['container']['format'] = 'singularity' + return minimal_configuration + + +def test_ensure_render_works(singularity_configuration): + container_config = singularity_configuration['spack']['container'] + assert container_config['format'] == 'singularity' + # Here we just want to ensure that nothing is raised + writer = writers.create(singularity_configuration) + writer() + + +@pytest.mark.parametrize('properties,expected', [ + ({'runscript': '/opt/view/bin/h5ls'}, + {'runscript': '/opt/view/bin/h5ls', + 'startscript': '', + 'test': '', + 'help': ''}) +]) +def test_singularity_specific_properties( + properties, expected, singularity_configuration +): + # Set the property in the configuration + container_config = singularity_configuration['spack']['container'] + for name, value in properties.items(): + container_config.setdefault('singularity', {})[name] = value + + # Assert the properties return the expected values + writer = writers.create(singularity_configuration) + for name, value in expected.items(): + assert getattr(writer, name) == value diff --git a/lib/spack/spack/test/data/compilers.yaml b/lib/spack/spack/test/data/config/compilers.yaml index 7aec138473..7aec138473 100644 --- a/lib/spack/spack/test/data/compilers.yaml +++ b/lib/spack/spack/test/data/config/compilers.yaml diff --git a/lib/spack/spack/test/data/config.yaml b/lib/spack/spack/test/data/config/config.yaml index 2d2c2e356a..0cf0d32cb0 100644 --- a/lib/spack/spack/test/data/config.yaml +++ b/lib/spack/spack/test/data/config/config.yaml @@ -11,4 +11,6 @@ config: misc_cache: ~/.spack/cache verify_ssl: true checksum: true - dirty: True + dirty: false + module_roots: + tcl: $spack/share/spack/modules diff --git a/lib/spack/spack/test/data/packages.yaml b/lib/spack/spack/test/data/config/packages.yaml index c7256ddb33..c7256ddb33 100644 --- a/lib/spack/spack/test/data/packages.yaml +++ b/lib/spack/spack/test/data/config/packages.yaml diff --git a/lib/spack/spack/test/data/repos.yaml b/lib/spack/spack/test/data/config/repos.yaml index 4fbbfe9d62..4fbbfe9d62 100644 --- a/lib/spack/spack/test/data/repos.yaml +++ b/lib/spack/spack/test/data/config/repos.yaml diff --git a/lib/spack/spack/test/data/sourceme_first.sh b/lib/spack/spack/test/data/sourceme_first.sh index 5d6b705e22..8fa1774b17 100644 --- a/lib/spack/spack/test/data/sourceme_first.sh +++ b/lib/spack/spack/test/data/sourceme_first.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_parameters.sh b/lib/spack/spack/test/data/sourceme_parameters.sh index bfe2c0c480..d11590d224 100644 --- a/lib/spack/spack/test/data/sourceme_parameters.sh +++ b/lib/spack/spack/test/data/sourceme_parameters.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_second.sh b/lib/spack/spack/test/data/sourceme_second.sh index 0cb0f45eb5..3f16153627 100644 --- a/lib/spack/spack/test/data/sourceme_second.sh +++ b/lib/spack/spack/test/data/sourceme_second.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_unicode.sh b/lib/spack/spack/test/data/sourceme_unicode.sh index cc3f59c4e3..d819126bb6 100644 --- a/lib/spack/spack/test/data/sourceme_unicode.sh +++ b/lib/spack/spack/test/data/sourceme_unicode.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/sourceme_unset.sh b/lib/spack/spack/test/data/sourceme_unset.sh index e14aeb3f89..0366833838 100644 --- a/lib/spack/spack/test/data/sourceme_unset.sh +++ b/lib/spack/spack/test/data/sourceme_unset.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/data/targets/linux-centos7-cascadelake b/lib/spack/spack/test/data/targets/linux-centos7-cascadelake new file mode 100644 index 0000000000..e409c3d07a --- /dev/null +++ b/lib/spack/spack/test/data/targets/linux-centos7-cascadelake @@ -0,0 +1,20 @@ +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 85 +model name : Intel(R) Xeon(R) Platinum 8260M CPU @ 2.40GHz +stepping : 7 +microcode : 0x5000024 +cpu MHz : 2400.000 +cache size : 36608 KB +physical id : 0 +siblings : 48 +core id : 0 +cpu cores : 24 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 22 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf eagerfpu pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch epb cat_l3 cdp_l3 intel_ppin intel_pt ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req pku ospke avx512_vnni md_clear spec_ctrl intel_stibp flush_l1d arch_capabilities
\ No newline at end of file diff --git a/lib/spack/spack/test/data/targets/linux-centos7-thunderx2 b/lib/spack/spack/test/data/targets/linux-centos7-thunderx2 new file mode 100644 index 0000000000..2447306bac --- /dev/null +++ b/lib/spack/spack/test/data/targets/linux-centos7-thunderx2 @@ -0,0 +1,8 @@ +processor : 0 +BogoMIPS : 400.00 +Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics cpuid asimdrdm +CPU implementer : 0x43 +CPU architecture: 8 +CPU variant : 0x1 +CPU part : 0x0af +CPU revision : 1 diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 06ddc83a47..a2b9677ec6 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -41,10 +41,11 @@ def test_store(tmpdir): @pytest.fixture() def upstream_and_downstream_db(tmpdir_factory, gen_mock_layout): mock_db_root = str(tmpdir_factory.mktemp('mock_db_root')) - upstream_db = spack.database.Database(mock_db_root) + upstream_write_db = spack.database.Database(mock_db_root) + upstream_db = spack.database.Database(mock_db_root, is_upstream=True) # Generate initial DB file to avoid reindex - with open(upstream_db._index_path, 'w') as db_file: - upstream_db._write_to_file(db_file) + with open(upstream_write_db._index_path, 'w') as db_file: + upstream_write_db._write_to_file(db_file) upstream_layout = gen_mock_layout('/a/') downstream_db_root = str( @@ -55,13 +56,14 @@ def upstream_and_downstream_db(tmpdir_factory, gen_mock_layout): downstream_db._write_to_file(db_file) downstream_layout = gen_mock_layout('/b/') - yield upstream_db, upstream_layout, downstream_db, downstream_layout + yield upstream_write_db, upstream_db, upstream_layout,\ + downstream_db, downstream_layout @pytest.mark.usefixtures('config') def test_installed_upstream(upstream_and_downstream_db): - upstream_db, upstream_layout, downstream_db, downstream_layout = ( - upstream_and_downstream_db) + upstream_write_db, upstream_db, upstream_layout,\ + downstream_db, downstream_layout = (upstream_and_downstream_db) default = ('build', 'link') x = MockPackage('x', [], []) @@ -75,7 +77,14 @@ def test_installed_upstream(upstream_and_downstream_db): spec.concretize() for dep in spec.traverse(root=False): - upstream_db.add(dep, upstream_layout) + upstream_write_db.add(dep, upstream_layout) + upstream_db._read() + + for dep in spec.traverse(root=False): + record = downstream_db.get_by_hash(dep.dag_hash()) + assert record is not None + with pytest.raises(spack.database.ForbiddenLockError): + record = upstream_db.get_by_hash(dep.dag_hash()) new_spec = spack.spec.Spec('w') new_spec.concretize() @@ -96,8 +105,8 @@ def test_installed_upstream(upstream_and_downstream_db): @pytest.mark.usefixtures('config') def test_removed_upstream_dep(upstream_and_downstream_db): - upstream_db, upstream_layout, downstream_db, downstream_layout = ( - upstream_and_downstream_db) + upstream_write_db, upstream_db, upstream_layout,\ + downstream_db, downstream_layout = (upstream_and_downstream_db) default = ('build', 'link') z = MockPackage('z', [], []) @@ -108,13 +117,15 @@ def test_removed_upstream_dep(upstream_and_downstream_db): spec = spack.spec.Spec('y') spec.concretize() - upstream_db.add(spec['z'], upstream_layout) + upstream_write_db.add(spec['z'], upstream_layout) + upstream_db._read() new_spec = spack.spec.Spec('y') new_spec.concretize() downstream_db.add(new_spec, downstream_layout) - upstream_db.remove(new_spec['z']) + upstream_write_db.remove(new_spec['z']) + upstream_db._read() new_downstream = spack.database.Database( downstream_db.root, upstream_dbs=[upstream_db]) @@ -129,8 +140,8 @@ def test_add_to_upstream_after_downstream(upstream_and_downstream_db): DB. When a package is recorded as installed in both, the results should refer to the downstream DB. """ - upstream_db, upstream_layout, downstream_db, downstream_layout = ( - upstream_and_downstream_db) + upstream_write_db, upstream_db, upstream_layout,\ + downstream_db, downstream_layout = (upstream_and_downstream_db) x = MockPackage('x', [], []) mock_repo = MockPackageMultiRepo([x]) @@ -141,7 +152,8 @@ def test_add_to_upstream_after_downstream(upstream_and_downstream_db): downstream_db.add(spec, downstream_layout) - upstream_db.add(spec, upstream_layout) + upstream_write_db.add(spec, upstream_layout) + upstream_db._read() upstream, record = downstream_db.query_by_spec_hash(spec.dag_hash()) # Even though the package is recorded as installed in the upstream DB, @@ -706,3 +718,14 @@ def test_uninstall_by_spec(mutable_database): else: mutable_database.remove(spec) assert len(mutable_database.query()) == 0 + + +def test_query_unused_specs(mutable_database): + # This spec installs a fake cmake as a build only dependency + s = spack.spec.Spec('simple-inheritance') + s.concretize() + s.package.do_install(fake=True, explicit=True) + + unused = spack.store.db.unused_specs + assert len(unused) == 1 + assert unused[0].name == 'cmake' diff --git a/lib/spack/spack/test/directives.py b/lib/spack/spack/test/directives.py index 2f2e4d55a8..ae1b6b7e2a 100644 --- a/lib/spack/spack/test/directives.py +++ b/lib/spack/spack/test/directives.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 619b617bab..c9ef68b3cf 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/environment_modifications.py b/lib/spack/spack/test/environment_modifications.py index 1e1759ddbc..9983594a84 100644 --- a/lib/spack/spack/test/environment_modifications.py +++ b/lib/spack/spack/test/environment_modifications.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/fetch_strategy.py b/lib/spack/spack/test/fetch_strategy.py new file mode 100644 index 0000000000..5470d88c13 --- /dev/null +++ b/lib/spack/spack/test/fetch_strategy.py @@ -0,0 +1,17 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +from spack.fetch_strategy import from_url_scheme + + +def test_fetchstrategy_bad_url_scheme(): + """Ensure that trying to make a fetch strategy from a URL with an + unsupported scheme fails as expected.""" + + with pytest.raises(ValueError): + fetcher = from_url_scheme( # noqa: F841 + 'bogus-scheme://example.com/a/b/c') diff --git a/lib/spack/spack/test/flag_handlers.py b/lib/spack/spack/test/flag_handlers.py index 094afe595a..6c7651ad19 100644 --- a/lib/spack/spack/test/flag_handlers.py +++ b/lib/spack/spack/test/flag_handlers.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index ae1e027752..8dc57da793 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -19,7 +19,6 @@ from spack.version import ver from spack.fetch_strategy import GitFetchStrategy from spack.util.executable import which - pytestmark = pytest.mark.skipif( not which('git'), reason='requires git to be installed') @@ -88,7 +87,7 @@ def test_fetch(type_of_test, secure, mock_git_repository, config, - mutable_mock_packages, + mutable_mock_repo, git_version): """Tries to: @@ -137,7 +136,7 @@ def test_fetch(type_of_test, @pytest.mark.parametrize("type_of_test", ['branch', 'commit']) -def test_debug_fetch(type_of_test, mock_git_repository, config): +def test_debug_fetch(mock_packages, type_of_test, mock_git_repository, config): """Fetch the repo with debug enabled.""" # Retrieve the right test parameters t = mock_git_repository.checks[type_of_test] @@ -169,14 +168,14 @@ def test_git_extra_fetch(tmpdir): def test_needs_stage(): """Trigger a NoStageError when attempt a fetch without a stage.""" with pytest.raises(spack.fetch_strategy.NoStageError, - matches=_mock_transport_error): + match=r"set_stage.*before calling fetch"): fetcher = GitFetchStrategy(git='file:///not-a-real-git-repo') fetcher.fetch() @pytest.mark.parametrize("get_full_repo", [True, False]) def test_get_full_repo(get_full_repo, git_version, mock_git_repository, - config, mutable_mock_packages): + config, mutable_mock_repo): """Ensure that we can clone a full repository.""" if git_version < ver('1.7.1'): @@ -217,3 +216,59 @@ def test_get_full_repo(get_full_repo, git_version, mock_git_repository, else: assert(nbranches == 2) assert(ncommits == 1) + + +@pytest.mark.disable_clean_stage_check +@pytest.mark.parametrize("submodules", [True, False]) +def test_gitsubmodule(submodules, mock_git_repository, config, + mutable_mock_repo): + """ + Test GitFetchStrategy behavior with submodules + """ + type_of_test = 'tag-branch' + t = mock_git_repository.checks[type_of_test] + + # Construct the package under test + spec = Spec('git-test') + spec.concretize() + pkg = spack.repo.get(spec) + args = copy.copy(t.args) + args['submodules'] = submodules + pkg.versions[ver('git')] = args + pkg.do_stage() + with working_dir(pkg.stage.source_path): + for submodule_count in range(2): + file_path = os.path.join(pkg.stage.source_path, + 'third_party/submodule{0}/r0_file_{0}' + .format(submodule_count)) + if submodules: + assert os.path.isfile(file_path) + else: + assert not os.path.isfile(file_path) + + +@pytest.mark.disable_clean_stage_check +def test_gitsubmodules_delete(mock_git_repository, config, mutable_mock_repo): + """ + Test GitFetchStrategy behavior with submodules_delete + """ + type_of_test = 'tag-branch' + t = mock_git_repository.checks[type_of_test] + + # Construct the package under test + spec = Spec('git-test') + spec.concretize() + pkg = spack.repo.get(spec) + args = copy.copy(t.args) + args['submodules'] = True + args['submodules_delete'] = ['third_party/submodule0', + 'third_party/submodule1'] + pkg.versions[ver('git')] = args + pkg.do_stage() + with working_dir(pkg.stage.source_path): + file_path = os.path.join(pkg.stage.source_path, + 'third_party/submodule0') + assert not os.path.isdir(file_path) + file_path = os.path.join(pkg.stage.source_path, + 'third_party/submodule1') + assert not os.path.isdir(file_path) diff --git a/lib/spack/spack/test/graph.py b/lib/spack/spack/test/graph.py index 57e7d54933..7ad1a75a90 100644 --- a/lib/spack/spack/test/graph.py +++ b/lib/spack/spack/test/graph.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 109e3d9d59..2cdeace3e4 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -29,7 +29,7 @@ def test_fetch( secure, mock_hg_repository, config, - mutable_mock_packages + mutable_mock_repo ): """Tries to: diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 4bed12456a..914ae527a0 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -10,6 +10,7 @@ import shutil from llnl.util.filesystem import mkdirp, touch, working_dir from spack.package import InstallError, PackageBase, PackageStillNeededError +import spack.error import spack.patch import spack.repo import spack.store @@ -128,7 +129,7 @@ def test_dont_add_patches_to_installed_package(install_mockery, mock_fetch): def test_installed_dependency_request_conflicts( - install_mockery, mock_fetch, mutable_mock_packages): + install_mockery, mock_fetch, mutable_mock_repo): dependency = Spec('dependency-install') dependency.concretize() dependency.package.do_install() @@ -136,12 +137,12 @@ def test_installed_dependency_request_conflicts( dependency_hash = dependency.dag_hash() dependent = Spec( 'conflicting-dependent ^/' + dependency_hash) - with pytest.raises(spack.spec.UnsatisfiableSpecError): + with pytest.raises(spack.error.UnsatisfiableSpecError): dependent.concretize() def test_install_dependency_symlinks_pkg( - install_mockery, mock_fetch, mutable_mock_packages): + install_mockery, mock_fetch, mutable_mock_repo): """Test dependency flattening/symlinks mock package.""" spec = Spec('flatten-deps') spec.concretize() @@ -154,7 +155,7 @@ def test_install_dependency_symlinks_pkg( def test_flatten_deps( - install_mockery, mock_fetch, mutable_mock_packages): + install_mockery, mock_fetch, mutable_mock_repo): """Explicitly test the flattening code for coverage purposes.""" # Unfortunately, executing the 'flatten-deps' spec's installation does # not affect code coverage results, so be explicit here. @@ -315,14 +316,14 @@ def test_uninstall_by_spec_errors(mutable_database): """Test exceptional cases with the uninstall command.""" # Try to uninstall a spec that has not been installed - rec = mutable_database.get_record('zmpi') - with pytest.raises(InstallError, matches="not installed"): - PackageBase.uninstall_by_spec(rec.spec) + spec = Spec('dependent-install') + spec.concretize() + with pytest.raises(InstallError, match="is not installed"): + PackageBase.uninstall_by_spec(spec) # Try an unforced uninstall of a spec with dependencies rec = mutable_database.get_record('mpich') - - with pytest.raises(PackageStillNeededError, matches="cannot uninstall"): + with pytest.raises(PackageStillNeededError, match="Cannot uninstall"): PackageBase.uninstall_by_spec(rec.spec) diff --git a/lib/spack/spack/test/link_paths.py b/lib/spack/spack/test/link_paths.py index 5f26a272c6..27e42d2194 100644 --- a/lib/spack/spack/test/link_paths.py +++ b/lib/spack/spack/test/link_paths.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/__init__.py b/lib/spack/spack/test/llnl/util/__init__.py new file mode 100644 index 0000000000..9f87532b85 --- /dev/null +++ b/lib/spack/spack/test/llnl/util/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/argparsewriter.py b/lib/spack/spack/test/llnl/util/argparsewriter.py new file mode 100644 index 0000000000..127149bbaa --- /dev/null +++ b/lib/spack/spack/test/llnl/util/argparsewriter.py @@ -0,0 +1,37 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Tests for ``llnl/util/argparsewriter.py`` + +These tests are fairly minimal, and ArgparseWriter is more extensively +tested in ``cmd/commands.py``. +""" + +import pytest + +import llnl.util.argparsewriter as aw + +import spack.main + + +parser = spack.main.make_argument_parser() +spack.main.add_all_commands(parser) + + +def test_format_not_overridden(): + writer = aw.ArgparseWriter('spack') + + with pytest.raises(NotImplementedError): + writer.write(parser) + + +def test_completion_format_not_overridden(): + writer = aw.ArgparseCompletionWriter('spack') + + assert writer.positionals([]) == '' + assert writer.optionals([]) == '' + assert writer.subcommands([]) == '' + + writer.write(parser) diff --git a/lib/spack/spack/test/llnl/util/cpu.py b/lib/spack/spack/test/llnl/util/cpu.py index f75c02f4b0..319d9e684e 100644 --- a/lib/spack/spack/test/llnl/util/cpu.py +++ b/lib/spack/spack/test/llnl/util/cpu.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -32,6 +32,8 @@ from llnl.util.cpu import Microarchitecture # noqa 'linux-scientific7-piledriver', 'linux-rhel6-piledriver', 'linux-centos7-power8le', + 'linux-centos7-thunderx2', + 'linux-centos7-cascadelake', 'darwin-mojave-ivybridge', 'darwin-mojave-haswell', 'darwin-mojave-skylake', @@ -86,6 +88,7 @@ def supported_target(request): return request.param +@pytest.mark.regression('13803') def test_target_detection(expected_target): detected_target = llnl.util.cpu.host() assert detected_target == expected_target @@ -121,6 +124,8 @@ def test_equality(supported_target): ('piledriver <= steamroller', True), ('zen2 >= zen', True), ('zen >= zen', True), + ('aarch64 <= thunderx2', True), + ('aarch64 <= a64fx', True), # Test unrelated microarchitectures ('power8 < skylake', False), ('power8 <= skylake', False), @@ -205,12 +210,16 @@ def test_target_json_schema(): ('nehalem', 'gcc', '4.9.3', '-march=nehalem -mtune=nehalem'), ('nehalem', 'gcc', '4.8.5', '-march=corei7 -mtune=corei7'), ('sandybridge', 'gcc', '4.8.5', '-march=corei7-avx -mtune=corei7-avx'), + ('thunderx2', 'gcc', '4.8.5', '-march=armv8-a'), + ('thunderx2', 'gcc', '4.9.3', '-march=armv8-a+crc+crypto'), # Test Clang / LLVM - ('sandybridge', 'clang', '3.9.0', '-march=x86-64 -mcpu=sandybridge'), - ('icelake', 'clang', '6.0.0', '-march=x86-64 -mcpu=icelake'), - ('icelake', 'clang', '8.0.0', '-march=x86-64 -mcpu=icelake-client'), - ('zen2', 'clang', '9.0.0', '-march=x86-64 -mcpu=znver2'), - ('power9le', 'clang', '8.0.0', '-march=ppc64le -mcpu=pwr9'), + ('sandybridge', 'clang', '3.9.0', '-march=sandybridge -mtune=sandybridge'), + ('icelake', 'clang', '6.0.0', '-march=icelake -mtune=icelake'), + ('icelake', 'clang', '8.0.0', + '-march=icelake-client -mtune=icelake-client'), + ('zen2', 'clang', '9.0.0', '-march=znver2 -mtune=znver2'), + ('power9le', 'clang', '8.0.0', '-mcpu=power9 -mtune=power9'), + ('thunderx2', 'clang', '6.0.0', '-mcpu=thunderx2t99'), # Test Intel on Intel CPUs ('sandybridge', 'intel', '17.0.2', '-march=corei7-avx -mtune=corei7-avx'), ('sandybridge', 'intel', '18.0.5', @@ -236,7 +245,7 @@ def test_unsupported_optimization_flags(target_name, compiler, version): target = llnl.util.cpu.targets[target_name] with pytest.raises( llnl.util.cpu.UnsupportedMicroarchitecture, - matches='cannot produce optimized binary' + match='cannot produce optimized binary' ): target.optimization_flags(compiler, version) @@ -269,3 +278,14 @@ def test_version_components(version, expected_number, expected_suffix): number, suffix = llnl.util.cpu.version_components(version) assert number == expected_number assert suffix == expected_suffix + + +def test_invalid_family(): + targets = llnl.util.cpu.targets + multi_parents = Microarchitecture( + name='chimera', parents=[targets['pentium4'], targets['power7']], + vendor='Imagination', features=[], compilers={}, generation=0 + ) + with pytest.raises(AssertionError, + match='a target is expected to belong'): + multi_parents.family diff --git a/lib/spack/spack/test/llnl/util/file_list.py b/lib/spack/spack/test/llnl/util/file_list.py index a43c8b2b44..e4036fe492 100644 --- a/lib/spack/spack/test/llnl/util/file_list.py +++ b/lib/spack/spack/test/llnl/util/file_list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py index 6c26c5fb43..b48abb4fc6 100644 --- a/lib/spack/spack/test/llnl/util/filesystem.py +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -116,13 +116,13 @@ class TestCopyTree: # Make sure we get the right error if we try to copy a parent into # a descendent directory. - with pytest.raises(ValueError, matches="Cannot copy"): + with pytest.raises(ValueError, match="Cannot copy"): with fs.working_dir(str(stage)): fs.copy_tree('source', 'source/sub/directory') # Only point with this check is to make sure we don't try to perform # the copy. - with pytest.raises(IOError, matches="No such file or directory"): + with pytest.raises(IOError, match="No such file or directory"): with fs.working_dir(str(stage)): fs.copy_tree('foo/ba', 'foo/bar') diff --git a/lib/spack/spack/test/llnl/util/lang.py b/lib/spack/spack/test/llnl/util/lang.py index cbc24b51db..b285d8d0b2 100644 --- a/lib/spack/spack/test/llnl/util/lang.py +++ b/lib/spack/spack/test/llnl/util/lang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/link_tree.py b/lib/spack/spack/test/llnl/util/link_tree.py index 6b00afb3fe..bebb3da805 100644 --- a/lib/spack/spack/test/llnl/util/link_tree.py +++ b/lib/spack/spack/test/llnl/util/link_tree.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index ca879cdc0b..63f2b91782 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/llnl/util/log.py b/lib/spack/spack/test/llnl/util/log.py index 5a57c8bb96..1eae1ccf69 100644 --- a/lib/spack/spack/test/llnl/util/log.py +++ b/lib/spack/spack/test/llnl/util/log.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/main.py b/lib/spack/spack/test/main.py new file mode 100644 index 0000000000..c35a6e195b --- /dev/null +++ b/lib/spack/spack/test/main.py @@ -0,0 +1,63 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +import llnl.util.filesystem as fs + +import spack.paths +from spack.main import get_version, main + + +def test_get_version_no_match_git(tmpdir, working_env): + git = str(tmpdir.join("git")) + with open(git, "w") as f: + f.write("""#!/bin/sh +echo v0.13.3 +""") + fs.set_executable(git) + + os.environ["PATH"] = str(tmpdir) + assert spack.spack_version == get_version() + + +def test_get_version_match_git(tmpdir, working_env): + git = str(tmpdir.join("git")) + with open(git, "w") as f: + f.write("""#!/bin/sh +echo v0.13.3-912-g3519a1762 +""") + fs.set_executable(git) + + os.environ["PATH"] = str(tmpdir) + assert "0.13.3-912-3519a1762" == get_version() + + +def test_get_version_no_repo(tmpdir, monkeypatch): + monkeypatch.setattr(spack.paths, "prefix", str(tmpdir)) + assert spack.spack_version == get_version() + + +def test_get_version_no_git(tmpdir, working_env): + os.environ["PATH"] = str(tmpdir) + assert spack.spack_version == get_version() + + +def test_main_calls_get_version(tmpdir, capsys, working_env): + os.environ["PATH"] = str(tmpdir) + main(["-V"]) + assert spack.spack_version == capsys.readouterr()[0].strip() + + +def test_get_version_bad_git(tmpdir, working_env): + bad_git = str(tmpdir.join("git")) + with open(bad_git, "w") as f: + f.write("""#!/bin/sh +exit 1 +""") + fs.set_executable(bad_git) + + os.environ["PATH"] = str(tmpdir) + assert spack.spack_version == get_version() diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index d945593dde..f554fd038c 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index e1b31695e3..08b32f74f1 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -16,7 +16,7 @@ from spack.util.executable import which from llnl.util.filesystem import resolve_link_target_relative_to_the_link -pytestmark = pytest.mark.usefixtures('config', 'mutable_mock_packages') +pytestmark = pytest.mark.usefixtures('config', 'mutable_mock_repo') # paths in repos that shouldn't be in the mirror tarballs. exclude = ['.hg', '.git', '.svn'] diff --git a/lib/spack/spack/test/module_parsing.py b/lib/spack/spack/test/module_parsing.py index 95a65ab33f..bbe18b1ad0 100644 --- a/lib/spack/spack/test/module_parsing.py +++ b/lib/spack/spack/test/module_parsing.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/__init__.py b/lib/spack/spack/test/modules/__init__.py new file mode 100644 index 0000000000..9f87532b85 --- /dev/null +++ b/lib/spack/spack/test/modules/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/common.py b/lib/spack/spack/test/modules/common.py index fdaf898daf..cc2eb61d0b 100644 --- a/lib/spack/spack/test/modules/common.py +++ b/lib/spack/spack/test/modules/common.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/conftest.py b/lib/spack/spack/test/modules/conftest.py index b49d8cd535..4ba25bd073 100644 --- a/lib/spack/spack/test/modules/conftest.py +++ b/lib/spack/spack/test/modules/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py index 34ba63c373..12b6d71aa5 100644 --- a/lib/spack/spack/test/modules/lmod.py +++ b/lib/spack/spack/test/modules/lmod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/modules/tcl.py b/lib/spack/spack/test/modules/tcl.py index 7bb058cd13..70eee24461 100644 --- a/lib/spack/spack/test/modules/tcl.py +++ b/lib/spack/spack/test/modules/tcl.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 5e50006e86..0e5bd5dad5 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index a24a75deac..2bf83c59f2 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index e1a34e8436..221712e5ef 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index e663ee70f4..d9268e7e14 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/package_class.py b/lib/spack/spack/test/package_class.py index 9502be08c9..b3351ffb49 100644 --- a/lib/spack/spack/test/package_class.py +++ b/lib/spack/spack/test/package_class.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -10,14 +10,14 @@ etc.). Only methods like ``possible_dependencies()`` that deal with the static DSL metadata for packages. """ +import pytest import spack.repo -def test_possible_dependencies(mock_packages): - mpileaks = spack.repo.get('mpileaks') +@pytest.fixture +def mpileaks_possible_deps(mock_packages): mpi_names = [spec.name for spec in spack.repo.path.providers_for('mpi')] - - assert mpileaks.possible_dependencies(expand_virtuals=True) == { + possible = { 'callpath': set(['dyninst'] + mpi_names), 'dyninst': set(['libdwarf', 'libelf']), 'fake': set(), @@ -29,6 +29,13 @@ def test_possible_dependencies(mock_packages): 'multi-provider-mpi': set(), 'zmpi': set(['fake']), } + return possible + + +def test_possible_dependencies(mock_packages, mpileaks_possible_deps): + mpileaks = spack.repo.get('mpileaks') + assert (mpileaks.possible_dependencies(expand_virtuals=True) == + mpileaks_possible_deps) assert mpileaks.possible_dependencies(expand_virtuals=False) == { 'callpath': set(['dyninst']), @@ -40,6 +47,15 @@ def test_possible_dependencies(mock_packages): } +def test_possible_dependencies_missing(mock_packages): + md = spack.repo.get("missing-dependency") + missing = {} + md.possible_dependencies(transitive=True, missing=missing) + assert missing["missing-dependency"] == set([ + "this-is-a-missing-dependency" + ]) + + def test_possible_dependencies_with_deptypes(mock_packages): dtbuild1 = spack.repo.get('dtbuild1') @@ -59,3 +75,17 @@ def test_possible_dependencies_with_deptypes(mock_packages): 'dtbuild1': set(['dtlink2']), 'dtlink2': set(), } + + +def test_possible_dependencies_with_multiple_classes( + mock_packages, mpileaks_possible_deps): + pkgs = ['dt-diamond', 'mpileaks'] + expected = mpileaks_possible_deps.copy() + expected.update({ + 'dt-diamond': set(['dt-diamond-left', 'dt-diamond-right']), + 'dt-diamond-left': set(['dt-diamond-bottom']), + 'dt-diamond-right': set(['dt-diamond-bottom']), + 'dt-diamond-bottom': set(), + }) + + assert spack.package.possible_dependencies(*pkgs) == expected diff --git a/lib/spack/spack/test/package_hash.py b/lib/spack/spack/test/package_hash.py index 61da62bb7e..a8a160d96e 100644 --- a/lib/spack/spack/test/package_hash.py +++ b/lib/spack/spack/test/package_hash.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index bbf63881a7..25494e1ab2 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -10,6 +10,7 @@ import re import pytest import spack.fetch_strategy +import spack.package import spack.paths import spack.repo import spack.util.executable as executable @@ -141,7 +142,6 @@ def test_all_packages_use_sha256_checksums(): assert [] == errors -@pytest.mark.xfail def test_api_for_build_and_run_environment(): """Ensure that every package uses the correct API to set build and run environment, and not the old one. @@ -154,7 +154,7 @@ def test_api_for_build_and_run_environment(): failing.append(pkg) msg = ('there are {0} packages using the old API to set build ' - 'and run environment [{1}], for further information see' + 'and run environment [{1}], for further information see ' 'https://github.com/spack/spack/pull/11115') assert not failing, msg.format( len(failing), ','.join(x.name for x in failing) @@ -182,7 +182,24 @@ def test_prs_update_old_api(): if failed: failing.append(name) - msg = 'there are {0} packages still using old APIs in this PR [{1}]' + msg = ('there are {0} packages using the old API to set build ' + 'and run environment [{1}], for further information see ' + 'https://github.com/spack/spack/pull/11115') assert not failing, msg.format( len(failing), ','.join(failing) ) + + +def test_all_dependencies_exist(): + """Make sure no packages have nonexisting dependencies.""" + missing = {} + pkgs = [pkg for pkg in spack.repo.path.all_package_names()] + spack.package.possible_dependencies( + *pkgs, transitive=True, missing=missing) + + lines = [ + "%s: [%s]" % (name, ", ".join(deps)) for name, deps in missing.items() + ] + assert not missing, "These packages have missing dependencies:\n" + ( + "\n".join(lines) + ) diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 186b0d0007..c811f9040e 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -16,6 +16,11 @@ from spack.version import VersionChecksumError import spack.directives +def _generate_content_strip_name(spec): + content = package_content(spec) + return content.replace(spec.package.__class__.__name__, '') + + @pytest.mark.usefixtures('config', 'mock_packages') class TestPackage(object): def test_load_package(self): @@ -53,38 +58,43 @@ class TestPackage(object): assert '_3db' == mod_to_class('3db') def test_content_hash_all_same_but_patch_contents(self): - spec1 = Spec("hash-test1@1.1") - spec2 = Spec("hash-test2@1.1") - spec1.concretize() - spec2.concretize() - content1 = package_content(spec1) - content1 = content1.replace(spec1.package.__class__.__name__, '') - content2 = package_content(spec2) - content2 = content2.replace(spec2.package.__class__.__name__, '') + spec1 = Spec("hash-test1@1.1").concretized() + spec2 = Spec("hash-test2@1.1").concretized() + content1 = _generate_content_strip_name(spec1) + content2 = _generate_content_strip_name(spec2) assert spec1.package.content_hash(content=content1) != \ spec2.package.content_hash(content=content2) def test_content_hash_different_variants(self): - spec1 = Spec("hash-test1@1.2 +variantx") - spec2 = Spec("hash-test2@1.2 ~variantx") - spec1.concretize() - spec2.concretize() - content1 = package_content(spec1) - content1 = content1.replace(spec1.package.__class__.__name__, '') - content2 = package_content(spec2) - content2 = content2.replace(spec2.package.__class__.__name__, '') + spec1 = Spec("hash-test1@1.2 +variantx").concretized() + spec2 = Spec("hash-test2@1.2 ~variantx").concretized() + content1 = _generate_content_strip_name(spec1) + content2 = _generate_content_strip_name(spec2) assert spec1.package.content_hash(content=content1) == \ spec2.package.content_hash(content=content2) + def test_content_hash_cannot_get_details_from_ast(self): + """Packages hash-test1 and hash-test3 would be considered the same + except that hash-test3 conditionally executes a phase based on + a "when" directive that Spack cannot evaluate by examining the + AST. This test ensures that Spack can compute a content hash + for hash-test3. If Spack cannot determine when a phase applies, + it adds it by default, so the test also ensures that the hashes + differ where Spack includes a phase on account of AST-examination + failure. + """ + spec3 = Spec("hash-test1@1.7").concretized() + spec4 = Spec("hash-test3@1.7").concretized() + content3 = _generate_content_strip_name(spec3) + content4 = _generate_content_strip_name(spec4) + assert(spec3.package.content_hash(content=content3) != + spec4.package.content_hash(content=content4)) + def test_all_same_but_archive_hash(self): - spec1 = Spec("hash-test1@1.3") - spec2 = Spec("hash-test2@1.3") - spec1.concretize() - spec2.concretize() - content1 = package_content(spec1) - content1 = content1.replace(spec1.package.__class__.__name__, '') - content2 = package_content(spec2) - content2 = content2.replace(spec2.package.__class__.__name__, '') + spec1 = Spec("hash-test1@1.3").concretized() + spec2 = Spec("hash-test2@1.3").concretized() + content1 = _generate_content_strip_name(spec1) + content2 = _generate_content_strip_name(spec2) assert spec1.package.content_hash(content=content1) != \ spec2.package.content_hash(content=content2) @@ -172,7 +182,7 @@ def test_urls_for_versions(mock_packages, config): assert url == 'http://www.doesnotexist.org/url_override-0.8.1.tar.gz' -def test_url_for_version_with_no_urls(): +def test_url_for_version_with_no_urls(mock_packages, config): pkg = spack.repo.get('git-test') with pytest.raises(spack.package.NoURLError): pkg.url_for_version('1.0') diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index 9c0396ef5e..edad8e29fa 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -19,10 +19,10 @@ import spack.repo import spack.store import spack.binary_distribution as bindist import spack.cmd.buildcache as buildcache +import spack.util.gpg from spack.spec import Spec from spack.paths import mock_gpg_keys_path from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite -from spack.util.executable import ProcessError from spack.relocate import needs_binary_relocation, needs_text_relocation from spack.relocate import strings_contains_installroot from spack.relocate import get_patchelf, relocate_text, relocate_links @@ -31,22 +31,6 @@ from spack.relocate import macho_replace_paths, macho_make_paths_relative from spack.relocate import modify_macho_object, macho_get_paths -@pytest.fixture(scope='function') -def testing_gpg_directory(tmpdir): - old_gpg_path = spack.util.gpg.GNUPGHOME - spack.util.gpg.GNUPGHOME = str(tmpdir.join('gpg')) - yield - spack.util.gpg.GNUPGHOME = old_gpg_path - - -def has_gnupg2(): - try: - spack.util.gpg.Gpg.gpg()('--version', output=os.devnull) - return True - except ProcessError: - return False - - def fake_fetchify(url, pkg): """Fake the URL for a package so it downloads from a file.""" fetcher = FetchStrategyComposite() @@ -54,7 +38,7 @@ def fake_fetchify(url, pkg): pkg.fetcher = fetcher -@pytest.mark.usefixtures('install_mockery', 'testing_gpg_directory') +@pytest.mark.usefixtures('install_mockery', 'mock_gnupghome') def test_buildcache(mock_archive, tmpdir): # tweak patchelf to only do a download spec = Spec("patchelf") @@ -107,7 +91,7 @@ echo $PATH""" buildcache.setup_parser(parser) # Create a private key to sign package with if gpg2 available - if has_gnupg2(): + if spack.util.gpg.Gpg.gpg(): spack.util.gpg.Gpg.create(name='test key 1', expires='0', email='spack@googlegroups.com', comment='Spack test key') @@ -230,7 +214,7 @@ echo $PATH""" stage.destroy() # Remove cached binary specs since we deleted the mirror - bindist._cached_specs = None + bindist._cached_specs = set() def test_relocate_text(tmpdir): diff --git a/lib/spack/spack/test/patch.py b/lib/spack/spack/test/patch.py index b705d01e42..2dc781538a 100644 --- a/lib/spack/spack/test/patch.py +++ b/lib/spack/spack/test/patch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -24,6 +24,7 @@ from spack.spec import Spec foo_sha256 = 'b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c' bar_sha256 = '7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730' baz_sha256 = 'bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c' +biz_sha256 = 'a69b288d7393261e613c276c6d38a01461028291f6e381623acc58139d01f54d' # url patches url1_sha256 = 'abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234' @@ -105,6 +106,20 @@ def test_patch_in_spec(mock_packages, config): tuple(spec.variants['patches']._patches_in_order_of_appearance)) +def test_patch_mixed_versions_subset_constraint(mock_packages, config): + """If we have a package with mixed x.y and x.y.z versions, make sure that + a patch applied to a version range of x.y.z versions is not applied to + an x.y version. + """ + spec1 = Spec('patch@1.0.1') + spec1.concretize() + assert biz_sha256 in spec1.variants['patches'].value + + spec2 = Spec('patch@1.0') + spec2.concretize() + assert biz_sha256 not in spec2.variants['patches'].value + + def test_patch_order(mock_packages, config): spec = Spec('dep-diamond-patch-top') spec.concretize() diff --git a/lib/spack/spack/test/pattern.py b/lib/spack/spack/test/pattern.py index 5481bfc0ea..582ece3d51 100644 --- a/lib/spack/spack/test/pattern.py +++ b/lib/spack/spack/test/pattern.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/permissions.py b/lib/spack/spack/test/permissions.py index b233c7a0fa..fa83eb37ff 100644 --- a/lib/spack/spack/test/permissions.py +++ b/lib/spack/spack/test/permissions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/provider_index.py b/lib/spack/spack/test/provider_index.py index 84ffcc3092..0855ac50ec 100644 --- a/lib/spack/spack/test/provider_index.py +++ b/lib/spack/spack/test/provider_index.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py deleted file mode 100644 index 6875aa4655..0000000000 --- a/lib/spack/spack/test/python_version.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""Check that Spack complies with minimum supported python versions. - -We ensure that all Spack files work with Python2 >= 2.6 and Python3 >= 3.0. - -We'd like to drop 2.6 support at some point, but there are still many HPC -systems that ship with RHEL6/CentOS 6, which have Python 2.6 as the -default version. Once those go away, we can likely drop 2.6 and increase -the minimum supported Python 3 version, as well. -""" -from __future__ import print_function - -import os -import sys -import re - -import pytest - -import llnl.util.tty as tty - -import spack.paths -from spack.paths import lib_path as spack_lib_path - - -# -# This test uses pyqver, by Greg Hewgill, which is a dual-source module. -# That means we need to do different checks depending on whether we're -# running Python 2 or Python 3. -# -if sys.version_info[0] < 3: - import pyqver2 as pyqver - spack_min_supported = (2, 6) - - # Exclude Python 3 versions of dual-source modules when using Python 2 - exclude_paths = [ - # Jinja 2 has some 'async def' functions that are not treated correctly - # by pyqver.py - os.path.join(spack_lib_path, 'external', 'jinja2', 'asyncfilters.py'), - os.path.join(spack_lib_path, 'external', 'jinja2', 'asyncsupport.py'), - os.path.join(spack_lib_path, 'external', 'yaml', 'lib3'), - os.path.join(spack_lib_path, 'external', 'pyqver3.py'), - # Uses importlib - os.path.join(spack_lib_path, 'spack', 'test', 'schema.py') - ] - -else: - import pyqver3 as pyqver - spack_min_supported = (3, 0) - - # Exclude Python 2 versions of dual-source modules when using Python 3 - exclude_paths = [ - # Jinja 2 has some 'async def' functions that are not treated correctly - # by pyqver.py - os.path.join(spack_lib_path, 'external', 'jinja2', 'asyncfilters.py'), - os.path.join(spack_lib_path, 'external', 'jinja2', 'asyncsupport.py'), - os.path.join(spack_lib_path, 'external', 'yaml', 'lib'), - os.path.join(spack_lib_path, 'external', 'pyqver2.py'), - # Uses importlib - os.path.join(spack_lib_path, 'spack', 'test', 'schema.py') - ] - - -def pyfiles(search_paths, exclude=()): - """Generator that yields all the python files in the search paths. - - Args: - search_paths (list of str): list of paths to search for python files - exclude (list of str): file paths to exclude from search - - Yields: - python files in the search path. - """ - # first file is the spack script. - yield spack.paths.spack_script - - # Iterate through the whole spack source tree. - for path in search_paths: - for root, dirnames, filenames in os.walk(path): - for filename in filenames: - realpath = os.path.realpath(os.path.join(root, filename)) - if any(realpath.startswith(p) for p in exclude): - continue - - if re.match(r'^[^.#].*\.py$', filename): - yield os.path.join(root, filename) - - -def check_python_versions(files): - """Check that a set of Python files works with supported Ptyhon versions""" - # This is a dict dict mapping: - # version -> filename -> reasons - # - # Reasons are tuples of (lineno, string), where the string is the - # cause for a version incompatibility. - all_issues = {} - - # Parse files and run pyqver on each file. - for path in files: - with open(path) as pyfile: - full_text = pyfile.read() - versions = pyqver.get_versions(full_text, path) - - for ver, reasons in versions.items(): - if ver <= spack_min_supported: - continue - - # Record issues. Mark exceptions with '# nopyqver' comment - for lineno, cause in reasons: - lines = full_text.split('\n') - if not re.search(r'#\s*nopyqver\s*$', lines[lineno - 1]): - all_issues.setdefault(ver, {})[path] = reasons - - # Print a message if there are are issues - if all_issues: - tty.msg("Spack must remain compatible with Python version %d.%d" - % spack_min_supported) - - # Print out a table showing which files/linenos require which - # python version, and a string describing why. - for v in sorted(all_issues.keys(), reverse=True): - messages = [] - for path in sorted(all_issues[v].keys()): - short_path = path - if path.startswith(spack.paths.prefix): - short_path = path[len(spack.paths.prefix):] - - reasons = [r for r in set(all_issues[v][path]) if r] - for lineno, cause in reasons: - file_line = "%s:%s" % (short_path.lstrip('/'), lineno) - messages.append((file_line, cause)) - - print() - tty.msg("These files require version %d.%d:" % v) - maxlen = max(len(f) for f, prob in messages) - fmt = "%%-%ds%%s" % (maxlen + 3) - print(fmt % ('File', 'Reason')) - print(fmt % ('-' * (maxlen), '-' * 20)) - for msg in messages: - print(fmt % msg) - - # Fail this test if there were issues. - assert not all_issues - - -@pytest.mark.maybeslow -def test_core_module_compatibility(): - """Test that all core spack modules work with supported Python versions.""" - check_python_versions( - pyfiles([spack_lib_path], exclude=exclude_paths)) - - -@pytest.mark.maybeslow -def test_package_module_compatibility(): - """Test that all spack packages work with supported Python versions.""" - check_python_versions(pyfiles([spack.paths.packages_path])) diff --git a/lib/spack/spack/test/relocate.py b/lib/spack/spack/test/relocate.py index f070e150c7..113bdcf66a 100644 --- a/lib/spack/spack/test/relocate.py +++ b/lib/spack/spack/test/relocate.py @@ -1,4 +1,4 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -60,6 +60,15 @@ def test_file_is_relocatable(source_file, is_relocatable): assert spack.relocate.file_is_relocatable(executable) is is_relocatable +@pytest.mark.requires_executables( + 'patchelf', 'strings', 'file' +) +def test_patchelf_is_relocatable(): + patchelf = spack.relocate.get_patchelf() + assert llnl.util.filesystem.is_exe(patchelf) + assert spack.relocate.file_is_relocatable(patchelf) + + @pytest.mark.skipif( platform.system().lower() != 'linux', reason='implementation for MacOS still missing' diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index e171dbca68..2cd1c0fa4a 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -10,14 +10,6 @@ import spack.repo import spack.paths -# Unlike the repo_path fixture defined in conftest, this has a test-level -# scope rather than a session level scope, since we want to edit the -# given RepoPath -@pytest.fixture() -def repo_for_test(): - return spack.repo.RepoPath(spack.paths.mock_packages_path) - - @pytest.fixture() def extra_repo(tmpdir_factory): repo_namespace = 'extra_test_repo' @@ -32,31 +24,31 @@ repo: return spack.repo.Repo(str(repo_dir)) -def test_repo_getpkg(repo_for_test): - repo_for_test.get('a') - repo_for_test.get('builtin.mock.a') +def test_repo_getpkg(mutable_mock_repo): + mutable_mock_repo.get('a') + mutable_mock_repo.get('builtin.mock.a') -def test_repo_multi_getpkg(repo_for_test, extra_repo): - repo_for_test.put_first(extra_repo) - repo_for_test.get('a') - repo_for_test.get('builtin.mock.a') +def test_repo_multi_getpkg(mutable_mock_repo, extra_repo): + mutable_mock_repo.put_first(extra_repo) + mutable_mock_repo.get('a') + mutable_mock_repo.get('builtin.mock.a') -def test_repo_multi_getpkgclass(repo_for_test, extra_repo): - repo_for_test.put_first(extra_repo) - repo_for_test.get_pkg_class('a') - repo_for_test.get_pkg_class('builtin.mock.a') +def test_repo_multi_getpkgclass(mutable_mock_repo, extra_repo): + mutable_mock_repo.put_first(extra_repo) + mutable_mock_repo.get_pkg_class('a') + mutable_mock_repo.get_pkg_class('builtin.mock.a') -def test_repo_pkg_with_unknown_namespace(repo_for_test): +def test_repo_pkg_with_unknown_namespace(mutable_mock_repo): with pytest.raises(spack.repo.UnknownNamespaceError): - repo_for_test.get('unknown.a') + mutable_mock_repo.get('unknown.a') -def test_repo_unknown_pkg(repo_for_test): +def test_repo_unknown_pkg(mutable_mock_repo): with pytest.raises(spack.repo.UnknownPackageError): - repo_for_test.get('builtin.mock.nonexistentpackage') + mutable_mock_repo.get('builtin.mock.nonexistentpackage') @pytest.mark.maybeslow @@ -66,7 +58,7 @@ def test_repo_last_mtime(): assert spack.repo.path.last_mtime() == latest_mtime -def test_repo_invisibles(repo_for_test, extra_repo): +def test_repo_invisibles(mutable_mock_repo, extra_repo): with open(os.path.join(extra_repo.root, 'packages', '.invisible'), 'w'): pass extra_repo.all_package_names() diff --git a/lib/spack/spack/test/s3_fetch.py b/lib/spack/spack/test/s3_fetch.py new file mode 100644 index 0000000000..682f1a2842 --- /dev/null +++ b/lib/spack/spack/test/s3_fetch.py @@ -0,0 +1,29 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import spack.fetch_strategy as spack_fs +import spack.stage as spack_stage + + +def test_s3fetchstrategy_sans_url(): + """Ensure constructor with no URL fails.""" + with pytest.raises(ValueError): + spack_fs.S3FetchStrategy(None) + + +def test_s3fetchstrategy_bad_url(tmpdir): + """Ensure fetch with bad URL fails as expected.""" + testpath = str(tmpdir) + + fetcher = spack_fs.S3FetchStrategy(url='file:///does-not-exist') + assert fetcher is not None + + with spack_stage.Stage(fetcher, path=testpath) as stage: + assert stage is not None + assert fetcher.archive_file is None + with pytest.raises(spack_fs.FetchError): + fetcher.fetch() diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py index 94c732e6ec..ade6b08a97 100644 --- a/lib/spack/spack/test/sbang.py +++ b/lib/spack/spack/test/sbang.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/schema.py b/lib/spack/spack/test/schema.py index 9149b77b37..a6e9986a01 100644 --- a/lib/spack/spack/test/schema.py +++ b/lib/spack/spack/test/schema.py @@ -1,4 +1,4 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -107,7 +107,7 @@ def test_module_suffixes(module_suffixes_schema): 'repos' ]) def test_schema_validation(meta_schema, config_name): - import importlib + import importlib # novm module_name = 'spack.schema.{0}'.format(config_name) module = importlib.import_module(module_name) schema = getattr(module, 'schema') diff --git a/lib/spack/spack/test/spack_yaml.py b/lib/spack/spack/test/spack_yaml.py index 2166d294e9..fb4abbf1d8 100644 --- a/lib/spack/spack/test/spack_yaml.py +++ b/lib/spack/spack/test/spack_yaml.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 628f67948f..419a39968e 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -189,7 +189,7 @@ def test_conditional_dep_with_user_constraints(): assert ('y@3' in spec) -@pytest.mark.usefixtures('mutable_mock_packages') +@pytest.mark.usefixtures('mutable_mock_repo') class TestSpecDag(object): def test_conflicting_package_constraints(self, set_dependency): diff --git a/lib/spack/spack/test/spec_list.py b/lib/spack/spack/test/spec_list.py index e52608ce5c..9bbbc435e2 100644 --- a/lib/spack/spack/test/spec_list.py +++ b/lib/spack/spack/test/spec_list.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index a183742e65..b55fa27ae1 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,11 +6,11 @@ import sys import pytest -from spack.spec import Spec, UnsatisfiableSpecError, SpecError -from spack.spec import substitute_abstract_variants -from spack.spec import SpecFormatSigilError, SpecFormatStringError -from spack.variant import InvalidVariantValueError +from spack.error import SpecError, UnsatisfiableSpecError +from spack.spec import Spec, SpecFormatSigilError, SpecFormatStringError +from spack.variant import InvalidVariantValueError, UnknownVariantError from spack.variant import MultipleValuesInExclusiveVariantError +from spack.variant import substitute_abstract_variants import spack.architecture import spack.directives @@ -981,3 +981,9 @@ class TestSpecSematics(object): def test_target_constraints(self, spec, constraint, expected_result): s = Spec(spec) assert s.satisfies(constraint) is expected_result + + @pytest.mark.regression('13124') + def test_error_message_unknown_variant(self): + s = Spec('mpileaks +unknown') + with pytest.raises(UnknownVariantError, match=r'package has no such'): + s.concretize() diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index a87fa5188d..a5f7b6fdb0 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -17,9 +17,10 @@ from spack.parse import Token from spack.spec import Spec from spack.spec import SpecParseError, RedundantSpecError from spack.spec import AmbiguousHashError, InvalidHashError, NoSuchHashError -from spack.spec import DuplicateArchitectureError, DuplicateVariantError +from spack.spec import DuplicateArchitectureError from spack.spec import DuplicateDependencyError, DuplicateCompilerSpecError from spack.spec import SpecFilenameError, NoSuchSpecFileError +from spack.variant import DuplicateVariantError # Sample output for a complex lexing. diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index 96bed17b78..f9b41df19a 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -68,7 +68,7 @@ def test_concrete_spec(config, mock_packages): check_yaml_round_trip(spec) -def test_yaml_multivalue(): +def test_yaml_multivalue(config, mock_packages): spec = Spec('multivalue_variant foo="bar,baz"') spec.concretize() check_yaml_round_trip(spec) diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index cc4f944867..9cf50b8da2 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -927,8 +927,11 @@ def test_stage_create_replace_path(tmp_build_stage_dir): assert os.path.isdir(stage.path) -def test_cannot_access(): +def test_cannot_access(capsys): """Ensure can_access dies with the expected error.""" - with pytest.raises(SystemExit, matches='Insufficient permissions'): + with pytest.raises(SystemExit): # It's far more portable to use a non-existent filename. spack.stage.ensure_access('/no/such/file') + + captured = capsys.readouterr() + assert 'Insufficient permissions' in str(captured) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index eb0d22ee7c..f435b07db2 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -30,7 +30,7 @@ def test_fetch( secure, mock_svn_repository, config, - mutable_mock_packages + mutable_mock_repo ): """Tries to: diff --git a/lib/spack/spack/test/tengine.py b/lib/spack/spack/test/tengine.py index 5cacc3cf66..01f2d65780 100644 --- a/lib/spack/spack/test/tengine.py +++ b/lib/spack/spack/test/tengine.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/test_activations.py b/lib/spack/spack/test/test_activations.py index 11de2e5431..eaac98f318 100644 --- a/lib/spack/spack/test/test_activations.py +++ b/lib/spack/spack/test/test_activations.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/url_fetch.py b/lib/spack/spack/test/url_fetch.py index 8047d5e26e..71a122455f 100644 --- a/lib/spack/spack/test/url_fetch.py +++ b/lib/spack/spack/test/url_fetch.py @@ -1,8 +1,9 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import collections import os import pytest @@ -10,8 +11,7 @@ from llnl.util.filesystem import working_dir, is_exe import spack.repo import spack.config -from spack.fetch_strategy import FailedDownloadError -from spack.fetch_strategy import from_list_url, URLFetchStrategy +import spack.fetch_strategy as fs from spack.spec import Spec from spack.stage import Stage from spack.version import ver @@ -23,10 +23,30 @@ def checksum_type(request): return request.param +@pytest.fixture +def pkg_factory(): + Pkg = collections.namedtuple( + 'Pkg', ['url_for_version', 'urls', 'url', 'versions'] + ) + + def factory(url, urls): + + def fn(v): + main_url = url or urls.pop(0) + return spack.url.substitute_version(main_url, v) + + return Pkg( + url_for_version=fn, url=url, urls=urls, + versions=collections.defaultdict(dict) + ) + + return factory + + def test_urlfetchstrategy_sans_url(): """Ensure constructor with no URL fails.""" with pytest.raises(ValueError): - with URLFetchStrategy(None): + with fs.URLFetchStrategy(None): pass @@ -34,8 +54,8 @@ def test_urlfetchstrategy_bad_url(tmpdir): """Ensure fetch with bad URL fails as expected.""" testpath = str(tmpdir) - with pytest.raises(FailedDownloadError): - fetcher = URLFetchStrategy(url='file:///does-not-exist') + with pytest.raises(fs.FailedDownloadError): + fetcher = fs.URLFetchStrategy(url='file:///does-not-exist') assert fetcher is not None with Stage(fetcher, path=testpath) as stage: @@ -55,7 +75,7 @@ def test_fetch( secure, checksum_type, config, - mutable_mock_packages + mutable_mock_repo ): """Fetch an archive and make sure we can checksum it.""" mock_archive.url @@ -106,8 +126,8 @@ def test_from_list_url(mock_packages, config, spec, url, digest): """ specification = Spec(spec).concretized() pkg = spack.repo.get(specification) - fetch_strategy = from_list_url(pkg) - assert isinstance(fetch_strategy, URLFetchStrategy) + fetch_strategy = fs.from_list_url(pkg) + assert isinstance(fetch_strategy, fs.URLFetchStrategy) assert os.path.basename(fetch_strategy.url) == url assert fetch_strategy.digest == digest @@ -118,8 +138,8 @@ def test_from_list_url_unspecified(mock_packages, config): spec = Spec('url-list-test @2.0.0').concretized() pkg = spack.repo.get(spec) - fetch_strategy = from_list_url(pkg) - assert isinstance(fetch_strategy, URLFetchStrategy) + fetch_strategy = fs.from_list_url(pkg) + assert isinstance(fetch_strategy, fs.URLFetchStrategy) assert os.path.basename(fetch_strategy.url) == 'foo-2.0.0.tar.gz' assert fetch_strategy.digest is None @@ -128,7 +148,7 @@ def test_nosource_from_list_url(mock_packages, config): """This test confirms BundlePackages do not have list url.""" pkg = spack.repo.get('nosource') - fetch_strategy = from_list_url(pkg) + fetch_strategy = fs.from_list_url(pkg) assert fetch_strategy is None @@ -148,9 +168,26 @@ def test_url_extra_fetch(tmpdir, mock_archive): """Ensure a fetch after downloading is effectively a no-op.""" testpath = str(tmpdir) - fetcher = URLFetchStrategy(mock_archive.url) + fetcher = fs.URLFetchStrategy(mock_archive.url) with Stage(fetcher, path=testpath) as stage: assert fetcher.archive_file is None stage.fetch() assert fetcher.archive_file is not None fetcher.fetch() + + +@pytest.mark.parametrize('url,urls,version,expected', [ + (None, + ['https://ftpmirror.gnu.org/autoconf/autoconf-2.69.tar.gz', + 'https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz'], + '2.62', + ['https://ftpmirror.gnu.org/autoconf/autoconf-2.62.tar.gz', + 'https://ftp.gnu.org/gnu/autoconf/autoconf-2.62.tar.gz']) +]) +def test_candidate_urls(pkg_factory, url, urls, version, expected): + """Tests that candidate urls include mirrors and that they go through + pattern matching and substitution for versions. + """ + pkg = pkg_factory(url, urls) + f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version) + assert f.candidate_urls == expected diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index c557f78405..0651562007 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -60,6 +60,8 @@ from spack.version import Version ('cppad-20170114.gpl', 'cppad-20170114'), # Arch ('pcraster-4.1.0_x86-64', 'pcraster-4.1.0'), + ('dislin-11.0.linux.i586_64', 'dislin-11.0'), + ('PAGIT.V1.01.64bit', 'PAGIT.V1.01'), # OS - linux ('astyle_2.04_linux', 'astyle_2.04'), # OS - unix @@ -72,6 +74,8 @@ from spack.version import Version ('numpy-1.12.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl', 'numpy-1.12.0'), # noqa # PyPI - exe ('PyYAML-3.12.win-amd64-py3.5.exe', 'PyYAML-3.12'), + # Combinations of multiple patterns - bin, release + ('rocketmq-all-4.5.2-bin-release', 'rocketmq-all-4.5.2'), # Combinations of multiple patterns - all ('p7zip_9.04_src_all', 'p7zip_9.04'), # Combinations of multiple patterns - run @@ -85,20 +89,31 @@ from spack.version import Version # Combinations of multiple patterns - darwin ('ghc-7.0.4-x86_64-apple-darwin', 'ghc-7.0.4'), ('ghc-7.0.4-i386-apple-darwin', 'ghc-7.0.4'), + # Combinations of multiple patterns - centos + ('sratoolkit.2.8.2-1-centos_linux64', 'sratoolkit.2.8.2-1'), # Combinations of multiple patterns - arch ('VizGlow_v2.2alpha17-R21November2016-Linux-x86_64-Install', 'VizGlow_v2.2alpha17-R21November2016'), ('jdk-8u92-linux-x64', 'jdk-8u92'), ('cuda_6.5.14_linux_64.run', 'cuda_6.5.14'), + ('Mathematica_12.0.0_LINUX.sh', 'Mathematica_12.0.0'), + ('trf407b.linux64', 'trf407b'), # Combinations of multiple patterns - with ('mafft-7.221-with-extensions-src', 'mafft-7.221'), ('spark-2.0.0-bin-without-hadoop', 'spark-2.0.0'), + ('conduit-v0.3.0-src-with-blt', 'conduit-v0.3.0'), + # Combinations of multiple patterns - rock + ('bitlib-23-2.src.rock', 'bitlib-23-2'), # Combinations of multiple patterns - public ('dakota-6.3-public.src', 'dakota-6.3'), # Combinations of multiple patterns - universal ('synergy-1.3.6p2-MacOSX-Universal', 'synergy-1.3.6p2'), # Combinations of multiple patterns - dynamic ('snptest_v2.5.2_linux_x86_64_dynamic', 'snptest_v2.5.2'), + # Combinations of multiple patterns - other + ('alglib-3.11.0.cpp.gpl', 'alglib-3.11.0'), + ('hpcviewer-2019.08-linux.gtk.x86_64', 'hpcviewer-2019.08'), + ('apache-mxnet-src-1.3.0-incubating', 'apache-mxnet-src-1.3.0'), ]) def test_url_strip_version_suffixes(url, expected): stripped = strip_version_suffixes(url) @@ -109,24 +124,40 @@ def test_url_strip_version_suffixes(url, expected): # No suffix ('rgb-1.0.6', '1.0.6', 'rgb'), ('nauty26r7', '26r7', 'nauty'), + ('PAGIT.V1.01', '1.01', 'PAGIT'), + ('AmpliconNoiseV1.29', '1.29', 'AmpliconNoise'), # Download type - install ('converge_install_2.3.16', '2.3.16', 'converge'), # Download type - src ('jpegsrc.v9b', '9b', 'jpeg'), + ('blatSrc35', '35', 'blat'), + # Download type - open + ('RepeatMasker-open-4-0-7', '4-0-7', 'RepeatMasker'), # Download type - archive ('coinhsl-archive-2014.01.17', '2014.01.17', 'coinhsl'), # Download type - std ('ghostscript-fonts-std-8.11', '8.11', 'ghostscript-fonts'), + # Download type - bin + ('GapCloser-bin-v1.12-r6', '1.12-r6', 'GapCloser'), + # Download type - software + ('orthomclSoftware-v2.0.9', '2.0.9', 'orthomcl'), # Download version - release ('cbench_release_1.3.0.tar.gz', '1.3.0', 'cbench'), # Download version - snapshot ('gts-snapshot-121130', '121130', 'gts'), # Download version - distrib ('zoltan_distrib_v3.83', '3.83', 'zoltan'), + # Download version - latest + ('Platypus-latest', 'N/A', 'Platypus'), + # Download version - complex + ('qt-everywhere-opensource-src-5.7.0', '5.7.0', 'qt'), + # Arch + ('VESTA-x86_64', '3.4.6', 'VESTA'), # VCS - bazaar ('libvterm-0+bzr681', '681', 'libvterm'), # License - gpl - ('PyQt-x11-gpl-4.11.3', '4.11.3', 'PyQt-x11') + ('PyQt-x11-gpl-4.11.3', '4.11.3', 'PyQt'), + ('PyQt4_gpl_x11-4.12.3', '4.12.3', 'PyQt4'), ]) def test_url_strip_name_suffixes(url, version, expected): stripped = strip_name_suffixes(url, version) @@ -182,6 +213,7 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): @pytest.mark.parametrize('name,version,url', [ # Common Repositories - github downloads + # name/archive/ver.ver ('nco', '4.6.2', 'https://github.com/nco/nco/archive/4.6.2.tar.gz'), # name/archive/vver.ver ('vim', '8.0.0134', 'https://github.com/vim/vim/archive/v8.0.0134.tar.gz'), @@ -257,6 +289,15 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): # Common Tarball Formats + # 1st Pass: Simplest case + # Assume name contains no digits and version contains no letters + + # name-ver.ver + ('libpng', '1.6.37', 'http://download.sourceforge.net/libpng/libpng-1.6.37.tar.gz'), + + # 2nd Pass: Version only + # Assume version contains no letters + # ver.ver ('eigen', '3.2.7', 'https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2'), # ver.ver-ver @@ -266,10 +307,17 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): # vver_ver ('luafilesystem', '1_6_3', 'https://github.com/keplerproject/luafilesystem/archive/v1_6_3.tar.gz'), - # No separators + # 3rd Pass: No separator characters are used + # Assume name contains no digits + + # namever ('turbolinux', '702', 'file://{0}/turbolinux702.tar.gz'.format(os.getcwd())), ('nauty', '26r7', 'http://pallini.di.uniroma1.it/nauty26r7.tar.gz'), - # Dashes only + + # 4th Pass: A single separator character is used + # Assume name contains no digits + + # name-name-ver-ver ('Trilinos', '12-10-1', 'https://github.com/trilinos/Trilinos/archive/trilinos-release-12-10-1.tar.gz'), ('panda', '2016-03-07', @@ -278,7 +326,7 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): 'http://gts.sourceforge.net/tarballs/gts-snapshot-121130.tar.gz'), ('cdd', '061a', 'http://www.cs.mcgill.ca/~fukuda/download/cdd/cdd-061a.tar.gz'), - # Only underscores + # name_name_ver_ver ('tinyxml', '2_6_2', 'https://sourceforge.net/projects/tinyxml/files/tinyxml/2.6.2/tinyxml_2_6_2.tar.gz'), ('boost', '1_55_0', @@ -287,9 +335,6 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): 'https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz'), ('tbb', '44_20160413', 'https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160413oss_src.tgz'), - - # Only dots - # name.name.ver.ver ('prank', '150803', 'http://wasabiapp.org/download/prank/prank.source.150803.tgz'), ('jpeg', '9b', 'http://www.ijg.org/files/jpegsrc.v9b.tar.gz'), @@ -302,61 +347,51 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): ('geant', '4.10.01.p03', 'http://geant4.cern.ch/support/source/geant4.10.01.p03.tar.gz'), ('tcl', '8.6.5', 'http://prdownloads.sourceforge.net/tcl/tcl8.6.5-src.tar.gz'), - # Dash and dots + # 5th Pass: Two separator characters are used + # Name may contain digits, version may contain letters # name-name-ver.ver - # digit in name ('m4', '1.4.17', 'https://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz'), - # letter in version ('gmp', '6.0.0a', 'https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2'), - # version starts with 'v' ('LaunchMON', '1.0.2', 'https://github.com/LLNL/LaunchMON/releases/download/v1.0.2/launchmon-v1.0.2.tar.gz'), # name-ver-ver.ver ('libedit', '20150325-3.1', 'http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz'), - - # Dash and unserscores - # name-name-ver_ver ('icu4c', '57_1', 'http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz'), - - # Underscores and dots - # name_name_ver.ver ('superlu_dist', '4.1', 'http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz'), ('pexsi', '0.9.0', 'https://math.berkeley.edu/~linlin/pexsi/download/pexsi_v0.9.0.tar.gz'), # name_name.ver.ver ('fer', '696', 'ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.v696.tar.gz'), - - # Dash dot dah dot - + # name_name_ver-ver + ('Bridger', '2014-12-01', + 'https://downloads.sourceforge.net/project/rnaseqassembly/Bridger_r2014-12-01.tar.gz'), # name-name-ver.ver-ver.ver ('sowing', '1.1.23-p1', 'http://ftp.mcs.anl.gov/pub/petsc/externalpackages/sowing-1.1.23-p1.tar.gz'), ('bib2xhtml', '3.0-15-gf506', 'http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v3.0-15-gf506.tar.gz'), # namever.ver-ver.ver ('go', '1.4-bootstrap-20161024', 'https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz'), - # Underscore dash dot + # 6th Pass: All three separator characters are used + # Name may contain digits, version may contain letters # name_name-ver.ver ('the_silver_searcher', '0.32.0', 'http://geoff.greer.fm/ag/releases/the_silver_searcher-0.32.0.tar.gz'), ('sphinx_rtd_theme', '0.1.10a0', 'https://pypi.python.org/packages/source/s/sphinx_rtd_theme/sphinx_rtd_theme-0.1.10a0.tar.gz'), - - # Dot underscore dot dash dot - # name.name_ver.ver-ver.ver ('TH.data', '1.0-8', 'https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz'), ('XML', '3.98-1.4', 'https://cran.r-project.org/src/contrib/XML_3.98-1.4.tar.gz'), - - # Dash dot underscore dot - # name-name-ver.ver_ver.ver ('pypar', '2.1.5_108', 'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/pypar/pypar-2.1.5_108.tgz'), # name-namever.ver_ver.ver ('STAR-CCM+', '11.06.010_02', 'file://{0}/STAR-CCM+11.06.010_02_linux-x86_64.tar.gz'.format(os.getcwd())), + # name-name_name-ver.ver + ('PerlIO-utf8_strict', '0.002', + 'http://search.cpan.org/CPAN/authors/id/L/LE/LEONT/PerlIO-utf8_strict-0.002.tar.gz'), # Various extensions # .tar.gz @@ -399,18 +434,61 @@ def test_url_parse_offset(name, noffset, ver, voffset, path): # .txz ('kim-api', '2.1.0', 'https://s3.openkim.org/kim-api/kim-api-2.1.0.txz'), - # Weird URLS + # 8th Pass: Query strings - # github.com/repo/name/releases/download/name-vver/name - ('nextflow', '0.20.1', 'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow'), # suffix queries ('swiftsim', '0.3.0', 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0'), - ('swiftsim', '0.3.0', 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'), + ('swiftsim', '0.3.0', + 'https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0'), ('sionlib', '1.7.1', 'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1'), + ('jube2', '2.2.2', 'https://apps.fz-juelich.de/jsc/jube/jube2/download.php?version=2.2.2'), + ('archive', '1.0.0', 'https://code.ornl.gov/eck/papyrus/repository/archive.tar.bz2?ref=v1.0.0'), + ('VecGeom', '0.3.rc', + 'https://gitlab.cern.ch/api/v4/projects/VecGeom%2FVecGeom/repository/archive.tar.gz?sha=v0.3.rc'), + ('parsplice', '1.1', + 'https://gitlab.com/api/v4/projects/exaalt%2Fparsplice/repository/archive.tar.gz?sha=v1.1'), + ('busco', '2.0.1', 'https://gitlab.com/api/v4/projects/ezlab%2Fbusco/repository/archive.tar.gz?sha=2.0.1'), + ('libaec', '1.0.2', + 'https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2'), + ('icet', '2.1.1', + 'https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-2.1.1'), + ('vtk-m', '1.3.0', + 'https://gitlab.kitware.com/api/v4/projects/vtk%2Fvtk-m/repository/archive.tar.gz?sha=v1.3.0'), + ('GATK', '3.8-1-0-gf15c1c3ef', + 'https://software.broadinstitute.org/gatk/download/auth?package=GATK-archive&version=3.8-1-0-gf15c1c3ef'), # stem queries ('slepc', '3.6.2', 'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz'), ('otf', '1.12.5salmon', 'http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz'), + ('eospac', '6.4.0beta.1', + 'http://laws-green.lanl.gov/projects/data/eos/get_file.php?package=eospac&filename=eospac_v6.4.0beta.1_r20171213193219.tgz'), + ('vampirtrace', '5.14.4', + 'http://wwwpub.zih.tu-dresden.de/~mlieber/dcount/dcount.php?package=vampirtrace&get=VampirTrace-5.14.4.tar.gz'), + # (we don't actually look for these, they are picked up + # during the preliminary stem parsing) + ('octopus', '6.0', 'http://octopus-code.org/down.php?file=6.0/octopus-6.0.tar.gz'), + ('cloog', '0.18.1', 'http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz'), + ('libxc', '2.2.2', 'http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz'), + ('cistem', '1.0.0-beta', + 'https://cistem.org/system/tdf/upload3/cistem-1.0.0-beta-source-code.tar.gz?file=1&type=cistem_details&id=37&force=0'), + ('Magics', '4.1.0', + 'https://confluence.ecmwf.int/download/attachments/3473464/Magics-4.1.0-Source.tar.gz?api=v2'), + ('grib_api', '1.17.0', + 'https://software.ecmwf.int/wiki/download/attachments/3473437/grib_api-1.17.0-Source.tar.gz?api=v2'), + ('eccodes', '2.2.0', + 'https://software.ecmwf.int/wiki/download/attachments/45757960/eccodes-2.2.0-Source.tar.gz?api=v2'), + ('SWFFT', '1.0', + 'https://xgitlab.cels.anl.gov/api/v4/projects/hacc%2FSWFFT/repository/archive.tar.gz?sha=v1.0'), + + # 9th Pass: Version in path + + # github.com/repo/name/releases/download/name-vver/name + ('nextflow', '0.20.1', 'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow'), + # ver/name + ('ncbi', '2.2.26', 'ftp://ftp.ncbi.nlm.nih.gov/blast/executables/legacy.NOTSUPPORTED/2.2.26/ncbi.tar.gz'), + + # Other tests for corner cases + # single character name ('R', '3.3.2', 'https://cloud.r-project.org/src/base/R-3/R-3.3.2.tar.gz'), # name starts with digit diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index 281c62e99f..08a3b99a4a 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/__init__.py b/lib/spack/spack/test/util/__init__.py new file mode 100644 index 0000000000..9f87532b85 --- /dev/null +++ b/lib/spack/spack/test/util/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/editor.py b/lib/spack/spack/test/util/editor.py index 2bb30cd1a2..e1495a70cc 100644 --- a/lib/spack/spack/test/util/editor.py +++ b/lib/spack/spack/test/util/editor.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/environment.py b/lib/spack/spack/test/util/environment.py index e34fc95f33..a555c35c9c 100644 --- a/lib/spack/spack/test/util/environment.py +++ b/lib/spack/spack/test/util/environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/executable.py b/lib/spack/spack/test/util/executable.py index 054cdbfccd..26719e94d1 100644 --- a/lib/spack/spack/test/util/executable.py +++ b/lib/spack/spack/test/util/executable.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,6 +8,7 @@ import sys import llnl.util.filesystem as fs import spack.util.executable as ex +from spack.hooks.sbang import filter_shebangs_in_directory def test_read_unicode(tmpdir): @@ -28,6 +29,7 @@ print(u'\\xc3') # make it executable fs.set_executable(script_name) + filter_shebangs_in_directory('.', [script_name]) # read the unicode back in and see whether things work script = ex.Executable('./%s' % script_name) diff --git a/lib/spack/spack/test/util/file_cache.py b/lib/spack/spack/test/util/file_cache.py index 0a82ae9f51..0fec1a5dc9 100644 --- a/lib/spack/spack/test/util/file_cache.py +++ b/lib/spack/spack/test/util/file_cache.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/log_parser.py b/lib/spack/spack/test/util/log_parser.py index 9025eb0741..0f829082b6 100644 --- a/lib/spack/spack/test/util/log_parser.py +++ b/lib/spack/spack/test/util/log_parser.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/prefix.py b/lib/spack/spack/test/util/prefix.py index 1d905eb13b..cee3ad028b 100644 --- a/lib/spack/spack/test/util/prefix.py +++ b/lib/spack/spack/test/util/prefix.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/spack_lock_wrapper.py b/lib/spack/spack/test/util/spack_lock_wrapper.py index c64af82e37..d7819c84cb 100644 --- a/lib/spack/spack/test/util/spack_lock_wrapper.py +++ b/lib/spack/spack/test/util/spack_lock_wrapper.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/spack_yaml.py b/lib/spack/spack/test/util/spack_yaml.py index 5e8c803774..ed7ba789c0 100644 --- a/lib/spack/spack/test/util/spack_yaml.py +++ b/lib/spack/spack/test/util/spack_yaml.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/util_gpg.py b/lib/spack/spack/test/util/util_gpg.py index a0d9d4d807..4a243985db 100644 --- a/lib/spack/spack/test/util/util_gpg.py +++ b/lib/spack/spack/test/util/util_gpg.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/util_string.py b/lib/spack/spack/test/util/util_string.py index 396c6abc0e..bcaa7d6211 100644 --- a/lib/spack/spack/test/util/util_string.py +++ b/lib/spack/spack/test/util/util_string.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/util/util_url.py b/lib/spack/spack/test/util/util_url.py index 24b40ac63c..5d0d9fa4eb 100644 --- a/lib/spack/spack/test/util/util_url.py +++ b/lib/spack/spack/test/util/util_url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -6,6 +6,7 @@ """Test Spack's URL handling utility functions.""" import os import os.path +import spack.paths import spack.util.url as url_util @@ -41,7 +42,7 @@ def test_url_parse(): assert(parsed.netloc == 'path') assert(parsed.path == '/to/resource') - spack_root = os.path.abspath(os.environ['SPACK_ROOT']) + spack_root = spack.paths.spack_root parsed = url_util.parse('$spack') assert(parsed.scheme == 'file') assert(parsed.netloc == '') @@ -56,7 +57,7 @@ def test_url_parse(): def test_url_local_file_path(): - spack_root = os.path.abspath(os.environ['SPACK_ROOT']) + spack_root = spack.paths.spack_root lfp = url_util.local_file_path('/a/b/c.txt') assert(lfp == '/a/b/c.txt') @@ -171,7 +172,7 @@ def test_url_join_local_paths(): 'https://mirror.spack.io/build_cache/my-package') # file:// URL path components are *NOT* canonicalized - spack_root = os.path.abspath(os.environ['SPACK_ROOT']) + spack_root = spack.paths.spack_root join_result = url_util.join('/a/b/c', '$spack') assert(join_result == 'file:///a/b/c/$spack') # not canonicalized diff --git a/lib/spack/spack/test/variant.py b/lib/spack/spack/test/variant.py index 4a76b4824e..d1657b71b7 100644 --- a/lib/spack/spack/test/variant.py +++ b/lib/spack/spack/test/variant.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/verification.py b/lib/spack/spack/test/verification.py index dbc76e6268..009f407186 100644 --- a/lib/spack/spack/test/verification.py +++ b/lib/spack/spack/test/verification.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index b39da0c698..315788f7f3 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -266,6 +266,8 @@ def test_contains(): assert_in('1.3.5-7', '1.2:1.4') assert_not_in('1.1', '1.2:1.4') assert_not_in('1.5', '1.2:1.4') + assert_not_in('1.5', '1.5.1:1.6') + assert_not_in('1.5', '1.5.1:') assert_in('1.4.2', '1.2:1.4') assert_not_in('1.4.2', '1.2:1.4.0') diff --git a/lib/spack/spack/test/views.py b/lib/spack/spack/test/views.py index 52ecb91e73..e1dcec4c5c 100644 --- a/lib/spack/spack/test/views.py +++ b/lib/spack/spack/test/views.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index c80e29b523..ae62301319 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -1,13 +1,16 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for web.py.""" import os +import pytest + +from ordereddict_backport import OrderedDict import spack.paths -from spack.util.web import spider, find_versions_of_archive +import spack.util.web as web_util from spack.version import ver @@ -23,7 +26,7 @@ page_4 = 'file://' + os.path.join(web_data_path, '4.html') def test_spider_0(): - pages, links = spider(root, depth=0) + pages, links = web_util.spider(root, depth=0) assert root in pages assert page_1 not in pages @@ -41,7 +44,7 @@ def test_spider_0(): def test_spider_1(): - pages, links = spider(root, depth=1) + pages, links = web_util.spider(root, depth=1) assert root in pages assert page_1 in pages @@ -60,7 +63,7 @@ def test_spider_1(): def test_spider_2(): - pages, links = spider(root, depth=2) + pages, links = web_util.spider(root, depth=2) assert root in pages assert page_1 in pages @@ -81,7 +84,7 @@ def test_spider_2(): def test_spider_3(): - pages, links = spider(root, depth=3) + pages, links = web_util.spider(root, depth=3) assert root in pages assert page_1 in pages @@ -104,31 +107,36 @@ def test_spider_3(): def test_find_versions_of_archive_0(): - versions = find_versions_of_archive(root_tarball, root, list_depth=0) + versions = web_util.find_versions_of_archive( + root_tarball, root, list_depth=0) assert ver('0.0.0') in versions def test_find_versions_of_archive_1(): - versions = find_versions_of_archive(root_tarball, root, list_depth=1) + versions = web_util.find_versions_of_archive( + root_tarball, root, list_depth=1) assert ver('0.0.0') in versions assert ver('1.0.0') in versions def test_find_versions_of_archive_2(): - versions = find_versions_of_archive(root_tarball, root, list_depth=2) + versions = web_util.find_versions_of_archive( + root_tarball, root, list_depth=2) assert ver('0.0.0') in versions assert ver('1.0.0') in versions assert ver('2.0.0') in versions def test_find_exotic_versions_of_archive_2(): - versions = find_versions_of_archive(root_tarball, root, list_depth=2) + versions = web_util.find_versions_of_archive( + root_tarball, root, list_depth=2) # up for grabs to make this better. assert ver('2.0.0b2') in versions def test_find_versions_of_archive_3(): - versions = find_versions_of_archive(root_tarball, root, list_depth=3) + versions = web_util.find_versions_of_archive( + root_tarball, root, list_depth=3) assert ver('0.0.0') in versions assert ver('1.0.0') in versions assert ver('2.0.0') in versions @@ -137,7 +145,49 @@ def test_find_versions_of_archive_3(): def test_find_exotic_versions_of_archive_3(): - versions = find_versions_of_archive(root_tarball, root, list_depth=3) + versions = web_util.find_versions_of_archive( + root_tarball, root, list_depth=3) assert ver('2.0.0b2') in versions assert ver('3.0a1') in versions assert ver('4.5-rc5') in versions + + +def test_get_header(): + headers = { + 'Content-type': 'text/plain' + } + + # looking up headers should just work like a plain dict + # lookup when there is an entry with the right key + assert(web_util.get_header(headers, 'Content-type') == 'text/plain') + + # looking up headers should still work if there is a fuzzy match + assert(web_util.get_header(headers, 'contentType') == 'text/plain') + + # ...unless there is an exact match for the "fuzzy" spelling. + headers['contentType'] = 'text/html' + assert(web_util.get_header(headers, 'contentType') == 'text/html') + + # If lookup has to fallback to fuzzy matching and there are more than one + # fuzzy match, the result depends on the internal ordering of the given + # mapping + headers = OrderedDict() + headers['Content-type'] = 'text/plain' + headers['contentType'] = 'text/html' + + assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/plain') + del headers['Content-type'] + assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/html') + + # Same as above, but different ordering + headers = OrderedDict() + headers['contentType'] = 'text/html' + headers['Content-type'] = 'text/plain' + + assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/html') + del headers['contentType'] + assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/plain') + + # If there isn't even a fuzzy match, raise KeyError + with pytest.raises(KeyError): + web_util.get_header(headers, 'ContentLength') diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 11b289a0b1..a728c46a71 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -153,13 +153,14 @@ def strip_version_suffixes(path): r'[Ii]nstall', r'all', r'code', - r'src(_0)?', r'[Ss]ources?', r'file', r'full', r'single', - r'public', r'with[a-zA-Z_-]+', + r'rock', + r'src(_0)?', + r'public', r'bin', r'binary', r'run', @@ -174,6 +175,7 @@ def strip_version_suffixes(path): # Download version r'release', + r'bin', r'stable', r'[Ff]inal', r'rel', @@ -189,15 +191,24 @@ def strip_version_suffixes(path): r'ia32', r'intel', r'amd64', + r'linux64', r'x64', + r'64bit', r'x86[_-]64', + r'i586_64', r'x86', r'i[36]86', r'ppc64(le)?', r'armv?(7l|6l|64)', + # Other + r'cpp', + r'gtk', + r'incubating', + # OS r'[Ll]inux(_64)?', + r'LINUX', r'[Uu]ni?x', r'[Ss]un[Oo][Ss]', r'[Mm]ac[Oo][Ss][Xx]?', @@ -208,14 +219,18 @@ def strip_version_suffixes(path): r'[Ww]in(64|32)?', r'[Cc]ygwin(64|32)?', r'[Mm]ingw', + r'centos', # Arch # Needs to come before and after OS, appears in both orders r'ia32', r'intel', r'amd64', + r'linux64', r'x64', + r'64bit', r'x86[_-]64', + r'i586_64', r'x86', r'i[36]86', r'ppc64(le)?', @@ -270,31 +285,41 @@ def strip_name_suffixes(path, version): # name-ver # name_ver # name.ver - r'[._-]v?' + str(version) + '.*', + r'[._-][rvV]?' + str(version) + '.*', # namever - str(version) + '.*', + r'V?' + str(version) + '.*', # Download type r'install', - r'src', + r'[Ss]rc', r'(open)?[Ss]ources?', + r'[._-]open', r'[._-]archive', r'[._-]std', + r'[._-]bin', + r'Software', # Download version r'release', r'snapshot', r'distrib', + r'everywhere', + r'latest', # Arch - r'Linux64', + r'Linux(64)?', + r'x86_64', # VCS r'0\+bzr', # License r'gpl', + + # Needs to come before and after gpl, appears in both orders + r'[._-]x11', + r'gpl', ] for regex in suffix_regexes: @@ -407,7 +432,7 @@ def parse_version_offset(path): # 3. names can contain A-Z, a-z, 0-9, '+', separators # 4. versions can contain A-Z, a-z, 0-9, separators # 5. versions always start with a digit - # 6. versions are often prefixed by a 'v' character + # 6. versions are often prefixed by a 'v' or 'r' character # 7. separators are most reliable to determine name/version boundaries # List of the following format: @@ -450,7 +475,7 @@ def parse_version_offset(path): (r'^[a-zA-Z+-]*(\d[\da-zA-Z-]*)$', stem), # name_name_ver_ver - # e.g. tinyxml_2_6_2, boost_1_55_0, tbb2017_20161128, v1_6_3 + # e.g. tinyxml_2_6_2, boost_1_55_0, tbb2017_20161128 (r'^[a-zA-Z+_]*(\d[\da-zA-Z_]*)$', stem), # name.name.ver.ver @@ -476,6 +501,10 @@ def parse_version_offset(path): # e.g. fer_source.v696 (r'^[a-zA-Z\d+_]+\.v?(\d[\da-zA-Z.]*)$', stem), + # name_ver-ver + # e.g. Bridger_r2014-12-01 + (r'^[a-zA-Z\d+]+_r?(\d[\da-zA-Z-]*)$', stem), + # name-name-ver.ver-ver.ver # e.g. sowing-1.1.23-p1, bib2xhtml-v3.0-15-gf506, 4.6.3-alpha04 (r'^(?:[a-zA-Z\d+-]+-)?v?(\d[\da-zA-Z.-]*)$', stem), @@ -507,19 +536,17 @@ def parse_version_offset(path): # e.g. STAR-CCM+11.06.010_02 (r'^[a-zA-Z+-]+(\d[\da-zA-Z._]*)$', stem), + # name-name_name-ver.ver + # e.g. PerlIO-utf8_strict-0.002 + (r'^[a-zA-Z\d+_-]+-v?(\d[\da-zA-Z.]*)$', stem), + # 7th Pass: Specific VCS # bazaar # e.g. libvterm-0+bzr681 (r'bzr(\d[\da-zA-Z._-]*)$', stem), - # 8th Pass: Version in path - - # github.com/repo/name/releases/download/vver/name - # e.g. https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow - (r'github\.com/[^/]+/[^/]+/releases/download/[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)/', path), # noqa - - # 9th Pass: Query strings + # 8th Pass: Query strings # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0 (r'\?sha=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix), @@ -528,13 +555,24 @@ def parse_version_offset(path): (r'\?ref=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix), # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1 - (r'\?version=v?(\d[\da-zA-Z._-]*)$', suffix), + # e.g. https://software.broadinstitute.org/gatk/download/auth?package=GATK-archive&version=3.8-1-0-gf15c1c3ef + (r'[?&]version=v?(\d[\da-zA-Z._-]*)$', suffix), # e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz - (r'\?filename=[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem), + # e.g. http://laws-green.lanl.gov/projects/data/eos/get_file.php?package=eospac&filename=eospac_v6.4.0beta.1_r20171213193219.tgz + (r'[?&]filename=[a-zA-Z\d+-]+[_-]v?(\d[\da-zA-Z.]*)', stem), # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz - (r'\?package=[a-zA-Z\d+-]+&get=[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem), # noqa + (r'&get=[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem), # noqa + + # 9th Pass: Version in path + + # github.com/repo/name/releases/download/vver/name + # e.g. https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow + (r'github\.com/[^/]+/[^/]+/releases/download/[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)/', path), # noqa + + # e.g. ftp://ftp.ncbi.nlm.nih.gov/blast/executables/legacy.NOTSUPPORTED/2.2.26/ncbi.tar.gz + (r'(\d[\da-zA-Z._-]*)/[^/]+$', path), ] for i, version_regex in enumerate(version_regexes): @@ -662,6 +700,9 @@ def parse_name_offset(path, v=None): # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz (r'\?package=([A-Za-z\d+-]+)', stem), + # ?package=name-version + (r'\?package=([A-Za-z\d]+)', suffix), + # download.php # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1 (r'([^/]+)/download.php$', path), diff --git a/lib/spack/spack/user_environment.py b/lib/spack/spack/user_environment.py new file mode 100644 index 0000000000..4c9fdc3d67 --- /dev/null +++ b/lib/spack/spack/user_environment.py @@ -0,0 +1,91 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import sys +import os + +import spack.util.prefix as prefix +import spack.util.environment as environment +import spack.build_environment as build_env + +#: Environment variable name Spack uses to track individually loaded packages +spack_loaded_hashes_var = 'SPACK_LOADED_HASHES' + + +def prefix_inspections(platform): + """Get list of prefix inspections for platform + + Arguments: + platform (string): the name of the platform to consider. The platform + determines what environment variables Spack will use for some + inspections. + + Returns: + A dictionary mapping subdirectory names to lists of environment + variables to modify with that directory if it exists. + """ + inspections = { + 'bin': ['PATH'], + 'lib': ['LD_LIBRARY_PATH', 'LIBRARY_PATH'], + 'lib64': ['LD_LIBRARY_PATH', 'LIBRARY_PATH'], + 'man': ['MANPATH'], + 'share/man': ['MANPATH'], + 'share/aclocal': ['ACLOCAL_PATH'], + 'include': ['CPATH'], + 'lib/pkgconfig': ['PKG_CONFIG_PATH'], + 'lib64/pkgconfig': ['PKG_CONFIG_PATH'], + '': ['CMAKE_PREFIX_PATH'] + } + + if platform == 'darwin': + for subdir in ('lib', 'lib64'): + inspections[subdir].append('DYLD_LIBRARY_PATH') + + return inspections + + +def unconditional_environment_modifications(view): + """List of environment (shell) modifications to be processed for view. + + This list does not depend on the specs in this environment""" + env = environment.EnvironmentModifications() + + for subdir, vars in prefix_inspections(sys.platform).items(): + full_subdir = os.path.join(view.root, subdir) + for var in vars: + env.prepend_path(var, full_subdir) + + return env + + +def environment_modifications_for_spec(spec, view=None): + """List of environment (shell) modifications to be processed for spec. + + This list is specific to the location of the spec or its projection in + the view.""" + spec = spec.copy() + if view: + spec.prefix = prefix.Prefix(view.view().get_projection_for_spec(spec)) + + # generic environment modifications determined by inspecting the spec + # prefix + env = environment.inspect_path( + spec.prefix, + prefix_inspections(spec.platform), + exclude=environment.is_system_path + ) + + # Let the extendee/dependency modify their extensions/dependents + # before asking for package-specific modifications + env.extend( + build_env.modifications_from_dependencies( + spec, context='run' + ) + ) + + # Package specific modifications + build_env.set_module_variables_for_package(spec.package) + spec.package.setup_run_environment(env) + + return env diff --git a/lib/spack/spack/util/__init__.py b/lib/spack/spack/util/__init__.py index 94f8ac4d9e..9f87532b85 100644 --- a/lib/spack/spack/util/__init__.py +++ b/lib/spack/spack/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index 0ad80457ec..d617954ab1 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py index 523878106e..566e99da21 100644 --- a/lib/spack/spack/util/crypto.py +++ b/lib/spack/spack/util/crypto.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/debug.py b/lib/spack/spack/util/debug.py index 8fcb50973f..1ff74f24cf 100644 --- a/lib/spack/spack/util/debug.py +++ b/lib/spack/spack/util/debug.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/editor.py b/lib/spack/spack/util/editor.py index ce5010d99f..00a095252e 100644 --- a/lib/spack/spack/util/editor.py +++ b/lib/spack/spack/util/editor.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index bfd6300ec8..248a6d3c8c 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index e383a533af..ce30e18f42 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/file_cache.py b/lib/spack/spack/util/file_cache.py index 0227edf155..9342f9772b 100644 --- a/lib/spack/spack/util/file_cache.py +++ b/lib/spack/spack/util/file_cache.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/file_permissions.py b/lib/spack/spack/util/file_permissions.py index c732aad0ab..d94b74f51e 100644 --- a/lib/spack/spack/util/file_permissions.py +++ b/lib/spack/spack/util/file_permissions.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index a7d1a3d8fa..29b2add852 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -1,13 +1,17 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import re +import spack.error import spack.paths -from spack.util.executable import Executable +import spack.version +from spack.util.executable import which +_gnupg_version_re = r"^gpg \(GnuPG\) (.*)$" GNUPGHOME = spack.paths.gpg_path @@ -28,15 +32,39 @@ def parse_keys_output(output): class Gpg(object): + _gpg = None + @staticmethod def gpg(): # TODO: Support loading up a GPG environment from a built gpg. - gpg = Executable('gpg2') - if not os.path.exists(GNUPGHOME): - os.makedirs(GNUPGHOME) - os.chmod(GNUPGHOME, 0o700) - gpg.add_default_env('GNUPGHOME', GNUPGHOME) - return gpg + if Gpg._gpg is None: + gpg = which('gpg2', 'gpg') + + if not gpg: + raise SpackGPGError("Spack requires gpg version 2 or higher.") + + # ensure that the version is actually >= 2 if we find 'gpg' + if gpg.name == 'gpg': + output = gpg('--version', output=str) + match = re.search(_gnupg_version_re, output, re.M) + + if not match: + raise SpackGPGError("Couldn't determine version of gpg") + + v = spack.version.Version(match.group(1)) + if v < spack.version.Version('2'): + raise SpackGPGError("Spack requires GPG version >= 2") + + # make the GNU PG path if we need to + # TODO: does this need to be in the spack directory? + # we should probably just use GPG's regular conventions + if not os.path.exists(GNUPGHOME): + os.makedirs(GNUPGHOME) + os.chmod(GNUPGHOME, 0o700) + gpg.add_default_env('GNUPGHOME', GNUPGHOME) + + Gpg._gpg = gpg + return Gpg._gpg @classmethod def create(cls, **kwargs): @@ -112,3 +140,7 @@ class Gpg(object): cls.gpg()('--list-public-keys') if signing: cls.gpg()('--list-secret-keys') + + +class SpackGPGError(spack.error.SpackError): + """Class raised when GPG errors are detected.""" diff --git a/lib/spack/spack/util/imp/__init__.py b/lib/spack/spack/util/imp/__init__.py index e06b83492a..27e2b00f9f 100644 --- a/lib/spack/spack/util/imp/__init__.py +++ b/lib/spack/spack/util/imp/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/imp/imp_importer.py b/lib/spack/spack/util/imp/imp_importer.py index 523926e469..810e1d01a3 100644 --- a/lib/spack/spack/util/imp/imp_importer.py +++ b/lib/spack/spack/util/imp/imp_importer.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/imp/importlib_importer.py b/lib/spack/spack/util/imp/importlib_importer.py index 33c50cb601..c230f4c50a 100644 --- a/lib/spack/spack/util/imp/importlib_importer.py +++ b/lib/spack/spack/util/imp/importlib_importer.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -7,7 +7,7 @@ ``importlib`` is only fully implemented in Python 3. """ -from importlib.machinery import SourceFileLoader +from importlib.machinery import SourceFileLoader # novm class PrependFileLoader(SourceFileLoader): diff --git a/lib/spack/spack/util/lock.py b/lib/spack/spack/util/lock.py index 59202528e2..b748aa056a 100644 --- a/lib/spack/spack/util/lock.py +++ b/lib/spack/spack/util/lock.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/log_parse.py b/lib/spack/spack/util/log_parse.py index 467f9eb45f..4b01c18c91 100644 --- a/lib/spack/spack/util/log_parse.py +++ b/lib/spack/spack/util/log_parse.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/module_cmd.py b/lib/spack/spack/util/module_cmd.py index d203670769..0edf7e6102 100644 --- a/lib/spack/spack/util/module_cmd.py +++ b/lib/spack/spack/util/module_cmd.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 618c21e8bd..1cc6d6109f 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/package_hash.py b/lib/spack/spack/util/package_hash.py index 2a3ee80fd5..f689aa9710 100644 --- a/lib/spack/spack/util/package_hash.py +++ b/lib/spack/spack/util/package_hash.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -69,8 +69,17 @@ class TagMultiMethods(ast.NodeVisitor): if node.decorator_list: dec = node.decorator_list[0] if isinstance(dec, ast.Call) and dec.func.id == 'when': - cond = dec.args[0].s - nodes.append((node, self.spec.satisfies(cond, strict=True))) + try: + cond = dec.args[0].s + nodes.append( + (node, self.spec.satisfies(cond, strict=True))) + except AttributeError: + # In this case the condition for the 'when' decorator is + # not a string literal (for example it may be a Python + # variable name). Therefore the function is added + # unconditionally since we don't know whether the + # constraint applies or not. + nodes.append((node, None)) else: nodes.append((node, None)) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 20b1d8b41f..9d5413c609 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py index 8ba6c54d71..f467e2d18e 100644 --- a/lib/spack/spack/util/pattern.py +++ b/lib/spack/spack/util/pattern.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/prefix.py b/lib/spack/spack/util/prefix.py index b4c0a63ce8..804790c1d3 100644 --- a/lib/spack/spack/util/prefix.py +++ b/lib/spack/spack/util/prefix.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/s3.py b/lib/spack/spack/util/s3.py index ee6b3d56cf..c9ed22f72c 100644 --- a/lib/spack/spack/util/s3.py +++ b/lib/spack/spack/util/s3.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/spack_json.py b/lib/spack/spack/util/spack_json.py index 4a966de46c..01f773058b 100644 --- a/lib/spack/spack/util/spack_json.py +++ b/lib/spack/spack/util/spack_json.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py index 1d92d59ad2..46e8e35543 100644 --- a/lib/spack/spack/util/spack_yaml.py +++ b/lib/spack/spack/util/spack_yaml.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/string.py b/lib/spack/spack/util/string.py index 1c28dbe6c3..7284cef2d3 100644 --- a/lib/spack/spack/util/string.py +++ b/lib/spack/spack/util/string.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/util/url.py b/lib/spack/spack/util/url.py index 7ac12e7b81..ab5503229f 100644 --- a/lib/spack/spack/util/url.py +++ b/lib/spack/spack/util/url.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,6 +9,7 @@ Utility functions for parsing, formatting, and manipulating URLs. import itertools import os.path +import re from six import string_types import six.moves.urllib.parse as urllib_parse @@ -69,8 +70,7 @@ def parse(url, scheme='file'): if scheme == 'file': path = spack.util.path.canonicalize_path(netloc + path) - while path.startswith('//'): - path = path[1:] + path = re.sub(r'^/+', '/', path) netloc = '' return urllib_parse.ParseResult(scheme=scheme, diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 1fe58d6415..4fb8c5a591 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -15,16 +15,13 @@ import ssl import sys import traceback -from itertools import product - -import six from six.moves.urllib.request import urlopen, Request from six.moves.urllib.error import URLError import multiprocessing.pool try: # Python 2 had these in the HTMLParser package. - from HTMLParser import HTMLParser, HTMLParseError + from HTMLParser import HTMLParser, HTMLParseError # novm except ImportError: # In Python 3, things moved to html.parser from html.parser import HTMLParser @@ -50,30 +47,6 @@ from spack.util.compression import ALLOWED_ARCHIVE_TYPES # Timeout in seconds for web requests _timeout = 10 -# See docstring for standardize_header_names() -_separators = ('', ' ', '_', '-') -HTTP_HEADER_NAME_ALIASES = { - "Accept-ranges": set( - ''.join((A, 'ccept', sep, R, 'anges')) - for A, sep, R in product('Aa', _separators, 'Rr')), - - "Content-length": set( - ''.join((C, 'ontent', sep, L, 'ength')) - for C, sep, L in product('Cc', _separators, 'Ll')), - - "Content-type": set( - ''.join((C, 'ontent', sep, T, 'ype')) - for C, sep, T in product('Cc', _separators, 'Tt')), - - "Date": set(('Date', 'date')), - - "Last-modified": set( - ''.join((L, 'ast', sep, M, 'odified')) - for L, sep, M in product('Ll', _separators, 'Mm')), - - "Server": set(('Server', 'server')) -} - class LinkParser(HTMLParser): """This parser just takes an HTML page and strips out the hrefs on the @@ -107,7 +80,7 @@ if sys.version_info[0] < 3: Process = NonDaemonProcess else: - class NonDaemonContext(type(multiprocessing.get_context())): + class NonDaemonContext(type(multiprocessing.get_context())): # novm Process = NonDaemonProcess class NonDaemonPool(multiprocessing.pool.Pool): @@ -155,7 +128,7 @@ def read_from_url(url, accept_content_type=None): warn_no_ssl_cert_checking() else: # User wants SSL verification, and it *can* be provided. - context = ssl.create_default_context() + context = ssl.create_default_context() # novm else: # User has explicitly indicated that they do not want SSL # verification. @@ -173,7 +146,7 @@ def read_from_url(url, accept_content_type=None): req.get_method = lambda: "HEAD" resp = _urlopen(req, timeout=_timeout, context=context) - content_type = resp.headers.get('Content-type') + content_type = get_header(resp.headers, 'Content-type') # Do the real GET request when we know it's just HTML. req.get_method = lambda: "GET" @@ -185,7 +158,7 @@ def read_from_url(url, accept_content_type=None): ERROR=str(err))) if accept_content_type and not is_web_url: - content_type = response.headers.get('Content-type') + content_type = get_header(response.headers, 'Content-type') reject_content_type = ( accept_content_type and ( @@ -208,9 +181,8 @@ def warn_no_ssl_cert_checking(): "your Python to enable certificate verification.") -def push_to_url(local_file_path, remote_path, **kwargs): - keep_original = kwargs.get('keep_original', True) - +def push_to_url( + local_file_path, remote_path, keep_original=True, extra_args=None): remote_url = url_util.parse(remote_path) verify_ssl = spack.config.get('config:verify_ssl') @@ -235,7 +207,8 @@ def push_to_url(local_file_path, remote_path, **kwargs): os.remove(local_file_path) elif remote_url.scheme == 's3': - extra_args = kwargs.get('extra_args', {}) + if extra_args is None: + extra_args = {} remote_path = remote_url.path while remote_path.startswith('/'): @@ -290,16 +263,31 @@ def remove_url(url): if url.scheme == 's3': s3 = s3_util.create_s3_session(url) - s3.delete_object(Bucket=url.s3_bucket, Key=url.path) + s3.delete_object(Bucket=url.netloc, Key=url.path) return # Don't even try for other URL schemes. -def _list_s3_objects(client, url, num_entries, start_after=None): +def _iter_s3_contents(contents, prefix): + for entry in contents: + key = entry['Key'] + + if not key.startswith('/'): + key = '/' + key + + key = os.path.relpath(key, prefix) + + if key == '.': + continue + + yield key + + +def _list_s3_objects(client, bucket, prefix, num_entries, start_after=None): list_args = dict( - Bucket=url.netloc, - Prefix=url.path, + Bucket=bucket, + Prefix=prefix[1:], MaxKeys=num_entries) if start_after is not None: @@ -311,21 +299,19 @@ def _list_s3_objects(client, url, num_entries, start_after=None): if result['IsTruncated']: last_key = result['Contents'][-1]['Key'] - iter = (key for key in - ( - os.path.relpath(entry['Key'], url.path) - for entry in result['Contents'] - ) - if key != '.') + iter = _iter_s3_contents(result['Contents'], prefix) return iter, last_key def _iter_s3_prefix(client, url, num_entries=1024): key = None + bucket = url.netloc + prefix = re.sub(r'^/*', '/', url.path) + while True: contents, key = _list_s3_objects( - client, url, num_entries, start_after=key) + client, bucket, prefix, num_entries, start_after=key) for x in contents: yield x @@ -577,106 +563,34 @@ def find_versions_of_archive(archive_urls, list_url=None, list_depth=0): return versions -def standardize_header_names(headers): - """Replace certain header names with standardized spellings. - - Standardizes the spellings of the following header names: - - Accept-ranges - - Content-length - - Content-type - - Date - - Last-modified - - Server - - Every name considered is translated to one of the above names if the only - difference between the two is how the first letters of each word are - capitalized; whether words are separated; or, if separated, whether they - are so by a dash (-), underscore (_), or space ( ). Header names that - cannot be mapped as described above are returned unaltered. - - For example: The standard spelling of "Content-length" would be substituted - for any of the following names: - - Content-length - - content_length - - contentlength - - content_Length - - contentLength - - content Length - - ... and any other header name, such as "Content-encoding", would not be - altered, regardless of spelling. - - If headers is a string, then it (or an appropriate substitute) is returned. - - If headers is a non-empty tuple, headers[0] is a string, and there exists a - standardized spelling for header[0] that differs from it, then a new tuple - is returned. This tuple has the same elements as headers, except the first - element is the standardized spelling for headers[0]. - - If headers is a sequence, then a new list is considered, where each element - is its corresponding element in headers, but mapped as above if a string or - tuple. This new list is returned if at least one of its elements differ - from their corrsponding element in headers. - - If headers is a mapping, then a new dict is considered, where the key in - each item is the key of its corresponding item in headers, mapped as above - if a string or tuple. The value is taken from the corresponding item. If - the keys of multiple items in headers map to the same key after being - standardized, then the value for the resulting item is undefined. The new - dict is returned if at least one of its items has a key that differs from - that of their corresponding item in headers, or if the keys of multiple - items in headers map to the same key after being standardized. - - In all other cases headers is returned unaltered. - """ - if isinstance(headers, six.string_types): - for standardized_spelling, other_spellings in ( - HTTP_HEADER_NAME_ALIASES.items()): - if headers in other_spellings: - if headers == standardized_spelling: - return headers - return standardized_spelling - return headers - - if isinstance(headers, tuple): - if not headers: - return headers - old = headers[0] - if isinstance(old, six.string_types): - new = standardize_header_names(old) - if old is not new: - return (new,) + headers[1:] - return headers - - try: - changed = False - new_dict = {} - for key, value in headers.items(): - if isinstance(key, (tuple, six.string_types)): - old_key, key = key, standardize_header_names(key) - changed = changed or key is not old_key - - new_dict[key] = value +def get_header(headers, header_name): + """Looks up a dict of headers for the given header value. - return new_dict if changed else headers - except (AttributeError, TypeError, ValueError): - pass + Looks up a dict of headers, [headers], for a header value given by + [header_name]. Returns headers[header_name] if header_name is in headers. + Otherwise, the first fuzzy match is returned, if any. - try: - changed = False - new_list = [] - for item in headers: - if isinstance(item, (tuple, six.string_types)): - old_item, item = item, standardize_header_names(item) - changed = changed or item is not old_item + This fuzzy matching is performed by discarding word separators and + capitalization, so that for example, "Content-length", "content_length", + "conTENtLength", etc., all match. In the case of multiple fuzzy-matches, + the returned value is the "first" such match given the underlying mapping's + ordering, or unspecified if no such ordering is defined. - new_list.append(item) + If header_name is not in headers, and no such fuzzy match exists, then a + KeyError is raised. + """ - return new_list if changed else headers - except TypeError: - pass + def unfuzz(header): + return re.sub(r'[ _-]', '', header).lower() - return headers + try: + return headers[header_name] + except KeyError: + unfuzzed_header_name = unfuzz(header_name) + for header, value in headers.items(): + if unfuzz(header) == unfuzzed_header_name: + return value + raise class SpackWebError(spack.error.SpackError): diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index 7eea243b06..3915fe00fa 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -21,7 +21,7 @@ import spack.directives import spack.error as error try: - from collections.abc import Sequence + from collections.abc import Sequence # novm except ImportError: from collections import Sequence @@ -600,7 +600,9 @@ def substitute_abstract_variants(spec): for name, v in spec.variants.items(): if name in spack.directives.reserved_names: continue - pkg_variant = spec.package_class.variants[name] + pkg_variant = spec.package_class.variants.get(name, None) + if not pkg_variant: + raise UnknownVariantError(spec, [name]) new_variant = pkg_variant.make_variant(v._original_value) pkg_variant.validate_or_raise(new_variant, spec.package_class) spec.variants.substitute(new_variant) @@ -778,12 +780,13 @@ class DuplicateVariantError(error.SpecError): class UnknownVariantError(error.SpecError): """Raised when an unknown variant occurs in a spec.""" - - def __init__(self, pkg, variants): + def __init__(self, spec, variants): self.unknown_variants = variants - super(UnknownVariantError, self).__init__( - 'Package {0} has no variant {1}!'.format(pkg, comma_or(variants)) - ) + variant_str = 'variant' if len(variants) == 1 else 'variants' + msg = ('trying to set {0} "{1}" in package "{2}", but the package' + ' has no such {0} [happened during concretization of {3}]') + msg = msg.format(variant_str, comma_or(variants), spec.name, spec.root) + super(UnknownVariantError, self).__init__(msg) class InconsistentValidationError(error.SpecError): diff --git a/lib/spack/spack/verify.py b/lib/spack/spack/verify.py index 88b89cf17c..08e6270a4a 100644 --- a/lib/spack/spack/verify.py +++ b/lib/spack/spack/verify.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 0bb5a767e7..bb7a79772e 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) |
