New package: SpECTRE numerical relativity code (#28399)

* New package: SpECTRE numerical relativity code

3 files changed, 369 insertions, 0 deletions

diff --git a/var/spack/repos/builtin/packages/spectre/build-testing-pre-2022.01.03.patch b/var/spack/repos/builtin/packages/spectre/build-testing-pre-2022.01.03.patch
new file mode 100644
index 0000000000..4550de6be4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/spectre/build-testing-pre-2022.01.03.patch
@@ -0,0 +1,84 @@
+From 3c6f3c3e7b025b52d29d663e1e79314dd2a07c55 Mon Sep 17 00:00:00 2001
+From: Nils Leif Fischer <nils.fischer@aei.mpg.de>
+Date: Fri, 14 Jan 2022 11:25:36 +0100
+Subject: [PATCH] Add standard CMake `BUILD_TESTING` option
+
+---
+ CMakeLists.txt                                 | 18 +++++++++++-------
+ cmake/SpectreSetupPythonPackage.cmake          |  4 +++-
+ .../ReduceCceWorldtube/CMakeLists.txt          |  4 +++-
+ 3 files changed, 17 insertions(+), 9 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index f7646195de7..99716332a6e 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -97,7 +97,6 @@ include(SetupLIBXSMM)
+ 
+ include(SetupBlaze)
+ include(SetupBrigand)
+-include(SetupCatch)
+ include(SetupGoogleBenchmark)
+ include(SetupGsl)
+ include(SetupHdf5)
+@@ -132,11 +131,14 @@ include(SetupSphinx)
+ include(CodeCoverageDetection)
+ include(SpectreAddLibraries)
+ 
+-enable_testing(true)
+-include(SetupPypp)
+-include(SpectreAddTestLibs)
+-include(SpectreAddCatchTests)
+-include(AddInputFileTests)
++include(CTest)
++if(BUILD_TESTING)
++  include(SetupCatch)
++  include(SetupPypp)
++  include(SpectreAddTestLibs)
++  include(SpectreAddCatchTests)
++  include(AddInputFileTests)
++endif()
+ 
+ include_directories(${CMAKE_SOURCE_DIR}/external)
+ include_directories(${CMAKE_SOURCE_DIR}/src)
+@@ -149,7 +151,9 @@ spectre_include_directories(${CMAKE_BINARY_DIR}/src/Parallel)
+ 
+ add_subdirectory(external)
+ add_subdirectory(src)
+-add_subdirectory(tests)
++if(BUILD_TESTING)
++  add_subdirectory(tests)
++endif()
+ 
+ include(PrintUsefulCMakeInfo)
+ 
+diff --git a/cmake/SpectreSetupPythonPackage.cmake b/cmake/SpectreSetupPythonPackage.cmake
+index 2b2466e6337..c81a9256f59 100644
+--- a/cmake/SpectreSetupPythonPackage.cmake
++++ b/cmake/SpectreSetupPythonPackage.cmake
+@@ -187,7 +187,9 @@ function(SPECTRE_PYTHON_ADD_MODULE MODULE_NAME)
+       LINK_FLAGS "${PY_LIB_LINK_FLAGS}"
+       )
+     set(SPECTRE_PYTHON_MODULE_IMPORT "from ._${ARG_LIBRARY_NAME} import *")
+-    add_dependencies(test-executables ${ARG_LIBRARY_NAME})
++    if(BUILD_TESTING)
++      add_dependencies(test-executables ${ARG_LIBRARY_NAME})
++    endif()
+     add_dependencies(all-pybindings ${ARG_LIBRARY_NAME})
+   endif(BUILD_PYTHON_BINDINGS AND NOT "${ARG_SOURCES}" STREQUAL "")
+ 
+diff --git a/src/Executables/ReduceCceWorldtube/CMakeLists.txt b/src/Executables/ReduceCceWorldtube/CMakeLists.txt
+index 3224b71c7e6..eca8f597b12 100644
+--- a/src/Executables/ReduceCceWorldtube/CMakeLists.txt
++++ b/src/Executables/ReduceCceWorldtube/CMakeLists.txt
+@@ -26,4 +26,6 @@ set_target_properties(
+   PROPERTIES LINK_FLAGS "-nomain-module -nomain"
+   )
+ 
+-add_dependencies(test-executables ${EXECUTABLE})
++if(BUILD_TESTING)
++  add_dependencies(test-executables ${EXECUTABLE})
++endif()
+-- 
+2.34.1
+
diff --git a/var/spack/repos/builtin/packages/spectre/install-pre-2022.01.03.patch b/var/spack/repos/builtin/packages/spectre/install-pre-2022.01.03.patch
new file mode 100644
index 0000000000..7cf0f28d06
--- /dev/null
+++ b/var/spack/repos/builtin/packages/spectre/install-pre-2022.01.03.patch
@@ -0,0 +1,59 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index f7646195de7..5cffd634909 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -33,6 +33,11 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.19)
+   cmake_policy(SET CMP0110 NEW)
+ endif ()
+ 
++# Disable `make install` depending on `make all` since we want to control what
++# we install more closely. With this setting, and targets marked as `OPTIONAL`,
++# only targets that were built will be installed.
++set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY ON)
++
+ set(CMAKE_VERBOSE_MAKEFILE OFF)
+ 
+ include(SpectreGetGitHash)
+diff --git a/cmake/AddSpectreExecutable.cmake b/cmake/AddSpectreExecutable.cmake
+index 790f3223a44..e6da837e02e 100644
+--- a/cmake/AddSpectreExecutable.cmake
++++ b/cmake/AddSpectreExecutable.cmake
+@@ -38,6 +38,7 @@ function(add_spectre_executable TARGET_NAME)
+     PRIVATE
+     SpectreFlags
+     )
++  install(TARGETS ${TARGET_NAME} OPTIONAL)
+ endfunction()
+ 
+ # A function to add a SpECTRE executable that uses Charm++
+diff --git a/cmake/SpectreSetupPythonPackage.cmake b/cmake/SpectreSetupPythonPackage.cmake
+index 2b2466e6337..fca7f3a9e4d 100644
+--- a/cmake/SpectreSetupPythonPackage.cmake
++++ b/cmake/SpectreSetupPythonPackage.cmake
+@@ -1,6 +1,11 @@
+ # Distributed under the MIT License.
+ # See LICENSE.txt for details.
+ 
++set(SPECTRE_PYTHON_INSTALL_LIBDIR
++  "lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages"
++  CACHE STRING "Location where the Python package is installed. Defaults to \
++CMAKE_INSTALL_PREFIX/lib/pythonX.Y/site-packages/.")
++
+ option(SPECTRE_PYTHON_TEST_TIMEOUT_FACTOR
+   "Multiply timeout for Python tests by this factor"
+   1)
+@@ -44,6 +49,12 @@ configure_file(
+   "${CMAKE_BINARY_DIR}/tmp/LoadPython.sh"
+   "${CMAKE_BINARY_DIR}/bin/LoadPython.sh")
+ 
++# Install the SpECTRE Python package to the user-specified location.
++install(
++  DIRECTORY ${SPECTRE_PYTHON_PREFIX}
++  DESTINATION ${SPECTRE_PYTHON_INSTALL_LIBDIR}
++  )
++
+ add_custom_target(all-pybindings)
+ 
+ # Add a python module, either with or without python bindings and with
+-- 
+2.34.1
diff --git a/var/spack/repos/builtin/packages/spectre/package.py b/var/spack/repos/builtin/packages/spectre/package.py
new file mode 100644
index 0000000000..ba1d911fca
--- /dev/null
+++ b/var/spack/repos/builtin/packages/spectre/package.py
@@ -0,0 +1,226 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Spectre(CMakePackage):
+    """The SpECTRE numerical relativity code.
+
+    SpECTRE is an open-source code for multi-scale, multi-physics problems in
+    astrophysics and gravitational physics. In the future, we hope that it can
+    be applied to problems across discipline boundaries in fluid dynamics,
+    geoscience, plasma physics, nuclear physics, and engineering. It runs at
+    petascale and is designed for future exascale computers.
+
+    SpECTRE is being developed in support of our collaborative Simulating
+    eXtreme Spacetimes (SXS) research program into the multi-messenger
+    astrophysics of neutron star mergers, core-collapse supernovae, and
+    gamma-ray bursts."""
+
+    homepage = "https://spectre-code.com"
+    url = "https://github.com/sxs-collaboration/spectre/archive/v2021.12.15.tar.gz"
+    git = "https://github.com/sxs-collaboration/spectre.git"
+
+    maintainers = ['nilsleiffischer']
+
+    generator = 'Ninja'
+
+    version('develop', branch='develop')
+    version('2022.01.03', sha256='872a0d152c19864ad543ddcc585ce30baaad4185c2617c13463d780175cbde5f')
+    version('2021.12.15', sha256='4bfe9e27412e263ffdc6fcfcb84011f16d34a9fdd633ad7fc84a34c898f67e5c')
+
+    # Configuration variants
+    variant('executables',
+            values=any_combination_of(
+                # CCE
+                'CharacteristicExtract', 'ReduceCceWorldtube',
+                # Elliptic / initial data
+                'SolvePoisson1D', 'SolvePoisson2D', 'SolvePoisson3D',
+                'SolveElasticity2D', 'SolveElasticity3D', 'SolveXcts',
+                # Tools
+                'ExportCoordinates1D', 'ExportCoordinates2D',
+                'ExportCoordinates3D',
+            ),
+            description="Executables to install")
+    variant('python', default=False, description="Build Python bindings")
+    variant('doc', default=False, description="Build documentation")
+    # TODO: support installation of executables with shared libs
+    # variant('shared',
+    #         default=False,
+    #         description="Build shared libraries instead of static")
+    variant('memory_allocator',
+            values=('system', 'jemalloc'),
+            multi=False,
+            default='system',
+            description="Which memory allocator to use")
+    variant('formaline',
+            default=True,
+            description=("Write the source tree into simulation output files "
+                         "to increase reproducibility of results"))
+    variant('profiling',
+            default=False,
+            description="Enable options to make profiling SpECTRE easier")
+
+    # Compiler support
+    conflicts('%gcc@:6')
+    conflicts('%clang@:7')
+    conflicts('%apple-clang@:10')
+
+    # Build dependencies
+    depends_on('cmake@3.12:', type='build')
+    depends_on('ninja', type='build')
+    depends_on('python@2.7:', type='build')
+
+    # Link dependencies
+    depends_on('charmpp@6.10.2:')
+    depends_on('blaze@3.8')
+    depends_on('boost@1.60:+math+program_options')
+    depends_on('brigand@master')
+    depends_on('gsl')
+    depends_on('hdf5')
+    depends_on('jemalloc', when='memory_allocator=jemalloc')
+    depends_on('libsharp~mpi~openmp')
+    depends_on('libxsmm@1.16.1:')
+    depends_on('blas')
+    depends_on('lapack')
+    depends_on('yaml-cpp@0.6:')
+
+    # Test dependencies
+    depends_on('catch2@2.8:', type='test')
+    depends_on('py-numpy@1.10:', type='test')
+    depends_on('py-scipy', type='test')
+    depends_on('py-h5py', type='test')
+
+    # Python bindings
+    with when('+python'):
+        extends('python')
+        depends_on('python@3.7:', type=('build', 'run'))
+        depends_on('py-pybind11@2.6:', type='build')
+        depends_on('py-numpy@1.10:', type=('build', 'run'))
+        depends_on('py-scipy', type=('build', 'run'))
+        depends_on('py-matplotlib', type=('build', 'run'))
+        depends_on('py-h5py', type=('build', 'run'))
+
+    # Docs
+    with when('+doc'):
+        depends_on('doxygen', type='build')
+        depends_on('py-beautifulsoup4', type='build')
+        depends_on('py-pybtex', type='build')
+
+    # These patches backport updates to the SpECTRE build system to earlier
+    # releases, to support installing them with Spack. In particular, we try to
+    # support releases associated with published papers, so their results are
+    # reproducible.
+    # - Backport installation of targets, based on upstream patch:
+    #   https://github.com/sxs-collaboration/spectre/commit/fe3514117c8205dbf18c4d42ec17712e67d33251
+    patch('install-pre-2022.01.03.patch', when='@:2022.01.03')
+    # - Backport experimental support for Charm++ v7+
+    patch(
+        'https://github.com/sxs-collaboration/spectre/commit/a2203824ef38ec79a247703ae8cd215befffe391.patch',
+        sha256='eb6094028530d9f28cb9c91a90b4af908cc537c8525fb4c81b11c74fd0354932',
+        when='@:2022.01.03 ^charmpp@7.0.0:')
+    # - Backport IWYU toggle to avoid CMake configuration issues
+    patch(
+        'https://github.com/sxs-collaboration/spectre/commit/cffeba1bc24bf7c00ec8bac710f02d3db36fa111.patch',
+        sha256='912877d4f553adff8b6df8264c50600c1e6d5a9c3ad18be0b86c9d801c07699c',
+        when='@:2022.01.03')
+    # - Backport patch for Boost 1.77
+    patch(
+        'https://github.com/sxs-collaboration/spectre/commit/001fc190a6ec73ad6c19ada9444d04a2320f2b96.patch',
+        sha256='bf539feb01d01e178889828dbbe5e990e8ee58c9e971d8634845c70a7cfb42a9',
+        when='@:2022.01.03 ^boost@1.77:')
+    # - Backport patch for Python 3.10 in tests
+    patch(
+        'https://github.com/sxs-collaboration/spectre/commit/82ff2c39cdae0ecc1e42bdf4564506a4ca869818.patch',
+        sha256='5a5a3abf102e92812933e7318daabe2ca0a5a00d81d9663731c527e5dc6c8ced',
+        when='@:2022.01.03 ^python@3.10:')
+    # - Backport patch for hdf5+mpi
+    patch(
+        'https://github.com/sxs-collaboration/spectre/commit/eb887635f5e2b394ae2c7e96170e9d907eb315cf.patch',
+        sha256='eb50b31af79d1e6b6535503bc30a9c5efd2ce36bd3638a2b3ab02af44bac6de3',
+        when='@:2022.01.03 ^hdf5+mpi')
+    # - Backport `BUILD_TESTING` toggle, based on upstream patch:
+    #   https://github.com/sxs-collaboration/spectre/commit/79bed6cad6e95efadf48a5846f389e90801202d4
+    patch('build-testing-pre-2022.01.03.patch', when='@:2022.01.03')
+    # - Backport `PYTHONPATH` in CTest environment
+    patch(
+        'https://github.com/sxs-collaboration/spectre/commit/ada1d15d5963bd22581dd8966599e1529a99645d.patch',
+        sha256='160d55bb2537ea8f3937cea59a9a0fd56a2bfef856bb7fd8e9dceb504c04836c',
+        when='@:2022.01.03')
+    # - Backport executable name CTest labels
+    patch(
+        'https://github.com/sxs-collaboration/spectre/commit/1b61e62a27b02b658cc6a74c4d46af1f5b5d0a4d.patch',
+        sha256='07be176ca4dda74a2dd8e71c31dab638a9f3567c3a58eb7fddbfde001646fb8c',
+        when='@:2022.01.03')
+
+    def cmake_args(self):
+        args = [
+            self.define('CHARM_ROOT', self.spec['charmpp'].prefix),
+            # self.define_from_variant('BUILD_SHARED_LIBS', 'shared'),
+            self.define('Python_EXECUTABLE', self.spec['python'].command.path),
+            self.define_from_variant('BUILD_PYTHON_BINDINGS', 'python'),
+            self.define('BUILD_TESTING', self.run_tests),
+            self.define('USE_GIT_HOOKS', False),
+            self.define('USE_IWYU', False),
+            self.define_from_variant('USE_FORMALINE', 'formaline'),
+            self.define_from_variant('MEMORY_ALLOCATOR').upper(),
+            self.define_from_variant('ENABLE_PROFILING', 'profiling'),
+            # TODO: Fix PCH builds to reduce compile time
+            self.define('USE_PCH', False),
+        ]
+        # Allow for more time on slower machines
+        if self.run_tests:
+            if self.spec.satisfies('@:2022.01.03'):
+                args.extend([
+                    self.define('SPECTRE_INPUT_FILE_TEST_TIMEOUT_FACTOR', '10'),
+                    self.define('SPECTRE_UNIT_TEST_TIMEOUT_FACTOR', '10'),
+                    self.define('SPECTRE_PYTHON_TEST_TIMEOUT_FACTOR', '10'),
+                ])
+            else:
+                args.append(self.define('SPECTRE_TEST_TIMEOUT_FACTOR', '10'))
+        return args
+
+    @property
+    def build_targets(self):
+        spec = self.spec
+        targets = list(self.spec.variants['executables'].value)
+        if 'none' in targets:
+            targets.remove('none')
+        if '+python' in spec:
+            targets.append('all-pybindings')
+        if '+doc' in spec:
+            targets.append('doc')
+        if self.run_tests:
+            targets.append('unit-tests')
+        if len(targets) == 0:
+            raise InstallError("Specify at least one target to build. See "
+                               "'spack info spectre' for available targets.")
+        return targets
+
+    @run_after('install')
+    def install_docs(self):
+        if '+doc' in self.spec:
+            with working_dir(self.build_directory):
+                install_tree(join_path('docs', 'html'), self.prefix.docs)
+
+    @property
+    def archive_files(self):
+        # Archive the `BuildInfo.txt` file for debugging builds
+        return super(Spectre, self).archive_files + [
+            join_path(self.build_directory, 'BuildInfo.txt')
+        ]
+
+    def check(self):
+        with working_dir(self.build_directory):
+            # The test suite contains a lot of tests. We select only those
+            # related to the targets that were specified.
+            # - Unit tests
+            ctest('--output-on-failure', '-L', 'unit')
+            # - Input file tests for the specified executables
+            for executable in self.spec.variants['executables'].value:
+                if executable == 'none':
+                    continue
+                ctest('--output-on-failure', '-L', executable)