From 745804582be4c8ef25ba9b0cb7dee62aa6e4b8a7 Mon Sep 17 00:00:00 2001 From: Gregor Daiß Date: Tue, 21 Apr 2020 23:03:46 +0200 Subject: Add new package: sgpp (#15961) * sgpp: Added recipe * sgpp: Removed maintainer and updated patch * sgpp: Added more conflicts * sgpp: Added conflicts for older combigrid versions * sgpp: Added one more maintainer * sgpp: Add version 3.3.0 and sane defaults * sgpp: Better description and c++11 conflicts * sgpp: Updated maintainers * sgpp: Fix flake8 errors * sgpp: Fix dependencies and minor issues - Added py-setuptools dependency - Fixed zlib link dependency - Added git url - Using spec.satisfies where appropriate * sgpp: Added testing to package * sgpp: Remove simd variant and use spec instead * sgpp: Remove java variant * sgpp: Small bugfixes * sgpp: Add datadriven patch and patch explanations --- .../sgpp/avx512_datadriven_compilation.patch | 21 +++ .../repos/builtin/packages/sgpp/directory.patch | 22 +++ .../repos/builtin/packages/sgpp/fix-setup-py.patch | 160 ++++++++++++++++ var/spack/repos/builtin/packages/sgpp/ocl.patch | 13 ++ var/spack/repos/builtin/packages/sgpp/package.py | 205 +++++++++++++++++++++ 5 files changed, 421 insertions(+) create mode 100644 var/spack/repos/builtin/packages/sgpp/avx512_datadriven_compilation.patch create mode 100644 var/spack/repos/builtin/packages/sgpp/directory.patch create mode 100644 var/spack/repos/builtin/packages/sgpp/fix-setup-py.patch create mode 100644 var/spack/repos/builtin/packages/sgpp/ocl.patch create mode 100644 var/spack/repos/builtin/packages/sgpp/package.py diff --git a/var/spack/repos/builtin/packages/sgpp/avx512_datadriven_compilation.patch b/var/spack/repos/builtin/packages/sgpp/avx512_datadriven_compilation.patch new file mode 100644 index 0000000000..268cba6358 --- /dev/null +++ b/var/spack/repos/builtin/packages/sgpp/avx512_datadriven_compilation.patch @@ -0,0 +1,21 @@ +diff --git a/datadriven/src/sgpp/datadriven/operation/hash/OperationMultiEvalStreaming/OperationMultiEvalStreaming_multImpl.cpp b/datadriven/src/sgpp/datadriven/operation/hash/OperationMultiEvalStreaming/OperationMultiEvalStreaming_multImpl.cpp +index 7fafd43d4..ea15ba137 100644 +--- a/datadriven/src/sgpp/datadriven/operation/hash/OperationMultiEvalStreaming/OperationMultiEvalStreaming_multImpl.cpp ++++ b/datadriven/src/sgpp/datadriven/operation/hash/OperationMultiEvalStreaming/OperationMultiEvalStreaming_multImpl.cpp +@@ -295,6 +295,7 @@ void OperationMultiEvalStreaming::multImpl( + _mm512_extload_pd(A, _MM_UPCONV_PD_NONE, _MM_BROADCAST_1X8, _MM_HINT_NONE) + #define _mm512_max_pd(A, B) _mm512_gmax_pd(A, B) + #define _mm512_set1_epi64(A) _mm512_set_1to8_epi64(A) ++#define _mm512_set1_pd(A) _mm512_set_1to8_pd(A) + #endif + #if defined(__AVX512F__) + #define _mm512_broadcast_sd(A) _mm512_broadcastsd_pd(_mm_load_sd(A)) +@@ -409,7 +410,7 @@ void OperationMultiEvalStreaming::multImpl( + eval_11 = _mm512_castsi512_pd(_mm512_and_epi64(abs2Mask, _mm512_castpd_si512(eval_11))); + #endif + +- __m512d one = _mm512_set_1to8_pd(1.0); ++ __m512d one = _mm512_set1_pd(1.0); + + eval_0 = _mm512_sub_pd(one, eval_0); + eval_1 = _mm512_sub_pd(one, eval_1); diff --git a/var/spack/repos/builtin/packages/sgpp/directory.patch b/var/spack/repos/builtin/packages/sgpp/directory.patch new file mode 100644 index 0000000000..f1b2286423 --- /dev/null +++ b/var/spack/repos/builtin/packages/sgpp/directory.patch @@ -0,0 +1,22 @@ +diff --git a/SConstruct b/SConstruct +index 98c1a56ac..30baef5e5 100644 +--- a/SConstruct ++++ b/SConstruct +@@ -293,7 +293,7 @@ Parameters are: + for line in vars.GenerateHelpText(env).splitlines()])) + + # add trailing slashes were required and if not present +-BUILD_DIR = Dir(os.path.join("lib", "sgpp")) ++BUILD_DIR = Dir(os.path.join("lib")) + Export("BUILD_DIR") + PYSGPP_PACKAGE_PATH = Dir(os.path.join("lib")) + Export("PYSGPP_PACKAGE_PATH") +@@ -550,7 +550,7 @@ env.SideEffect("sideEffectFinalSteps", exampleTargetList) + ######################################################################### + + installLibSGpp = env.Alias("install-lib-sgpp", +- env.Install(os.path.join(env.get("LIBDIR"), "sgpp"), ++ env.Install(os.path.join(env.get("LIBDIR")), + libraryTargetList)) + + headerFinalDestList = [] diff --git a/var/spack/repos/builtin/packages/sgpp/fix-setup-py.patch b/var/spack/repos/builtin/packages/sgpp/fix-setup-py.patch new file mode 100644 index 0000000000..e201a5dff0 --- /dev/null +++ b/var/spack/repos/builtin/packages/sgpp/fix-setup-py.patch @@ -0,0 +1,160 @@ +diff --git a/setup.py b/setup.py +index af1ec083e..d971f87c5 100644 +--- a/setup.py ++++ b/setup.py +@@ -1,5 +1,5 @@ +-#!/usr/bin/env python +-# Copyright (C) 2008-today The SG++ Project ++#!/usr/bin/env python ++# Copyright (C) 2008-today The SG++ project + # This file is part of the SG++ project. For conditions of distribution and + # use, please see the copyright notice provided with SG++ or at + # sgpp.sparsegrids.org +@@ -8,17 +8,26 @@ + # script creates a pysgpp lib in the site-packages of + # python. Furthermore, it collects all the relevant python code, + # located in each module under the path /python and +-# copies it to the site-package folder of pysgpp using the following +-# scheme: /pysgpp-/extensions//. ++# copies it to the site-package folder of pysgpp under the extensions namespace ++# pysgpp.extensions + + import os + import shutil +-from setuptools import setup ++from setuptools import setup, find_packages ++ ++ ++try: ++ from wheel.bdist_wheel import bdist_wheel as _bdist_wheel ++ class bdist_wheel(_bdist_wheel): ++ def finalize_options(self): ++ _bdist_wheel.finalize_options(self) ++ self.root_is_pure = False ++except ImportError: ++ bdist_wheel = None + + # path to pysgpp lib + libpath = os.path.join("lib", "pysgpp") ++extensionspath = os.path.join(libpath, "extensions") + + # list of all available modules -> all folders in the root directory + moduleFolders = [filename for filename in os.listdir(".") +@@ -29,16 +38,44 @@ pythonModuleFolders = [(moduleFolder, os.path.join(moduleFolder, "python")) + for moduleFolder in moduleFolders + if os.path.exists(os.path.join(moduleFolder, "python"))] + +-# create the data file list such that it can be used by setuptools +-dataFiles = [] ++try: ++ os.mkdir(extensionspath) ++except FileExistsError as e: ++ pass ++ ++# create list of extension scripts ++extFiles = [] + for moduleFolder, srcdir in pythonModuleFolders: +- basepath = os.path.join("pysgpp", "extensions", moduleFolder) ++ basepath = os.path.join(extensionspath, moduleFolder) ++ try: ++ os.mkdir(basepath) ++ except FileExistsError as e: ++ pass + for root, dirs, files in os.walk(srcdir): +- if '.svn' in dirs: +- dirs.remove('.svn') ++ if '.git' in dirs: ++ dirs.remove('.git') ++ ++ extFiles += [os.path.join(root, f) for f in files if ".py" in f] ++ ++## ++# copy extension python files to new layout ++# pysgpp ++# --extensions ++# ----modulename ++# ------*.py ++## + +- dataFiles += [(root.replace(srcdir, basepath), +- [os.path.join(root, f) for f in files])] ++for f in extFiles: ++ dest = os.path.join(extensionspath,f) ++ dest = dest.replace(os.sep + "python", "") ++ try: ++ shutil.copy2(f, dest) ++ except FileNotFoundError as e: ++ os.mkdir(os.path.dirname(dest)) ++ shutil.copy2(f, dest) ++ except shutil.SameFileError as e: ++ pass ++ + + # write init file for pysgpp + initFile = os.path.join(libpath, "__init__.py") +@@ -50,8 +87,8 @@ import sys + sys.path.append(os.path.dirname(__file__)) + + # import pysgpp_swig and extensions +-from pysgpp_swig import * +-import pysgpp.extensions ++from .pysgpp_swig import * ++from . import extensions + """) + + if len(moduleFolders) > 0: +@@ -59,9 +96,13 @@ if len(moduleFolders) > 0: + initFile = os.path.join("__init__.py") + with open(initFile, "w") as f: + for moduleFolder, _ in pythonModuleFolders: +- f.write("import %s\n" % moduleFolder) ++ f.write("from . import %s\n" % moduleFolder) + +- dataFiles += [(os.path.join("pysgpp", "extensions"), [initFile])] ++ try: ++ shutil.copy2(initFile, os.path.join(extensionspath, initFile)) ++ except shutil.SameFileError as e: ++ pass ++ + + # if the current system is windows we need to rename the dll to pyd + dllLibs = [filename for filename in os.listdir(libpath) +@@ -74,20 +115,27 @@ for dllLib in dllLibs: + + # setup pysgpp + setup(name='pysgpp', +- version="1.0.0", +- url='sgpp.sparsegrids.org', +- author="Fabian Franzelin", +- description='', +- license='', ++ version="0.0.0", ++ url='https://github.com/SGpp/SGpp', ++ author="Dirk.Pflueger@ipvs.uni-stuttgart.de", ++ description='''The sparse grids toolkit SG++ ++ SG++ is a collection of numerical algorithms for sparse grids. It ++ contains modules for interpolation, quadrature, data mining ++ (regression, classification, clustering), optimization, PDEs, and ++ more. SG++ implements algorithms for spatially adaptive grids and ++ also provides a module for the combination technique. Many of the ++ implemented algorithms are also available as a high-performance ++ version, often orders of magnitude faster than standard ++ implementations.''', ++ license='BSD-style license', + long_description="README", +- platforms='any', + zip_safe=False, + package_dir={'': 'lib'}, +- packages=['pysgpp'], +- package_data={'pysgpp': ['*.so', '*.lib', '*.pyd']}, +- data_files=dataFiles ++ packages=find_packages(where='lib', include=['pysgpp', 'pysgpp.extensions*']), ++ package_data={'pysgpp': ['_pysgpp_swig.so', '*.lib', '*.pyd']}, + ) + + # cleanup + if len(moduleFolders) > 0 and os.path.exists(initFile): + os.remove(initFile) ++ shutil.rmtree(extensionspath, ignore_errors=True) diff --git a/var/spack/repos/builtin/packages/sgpp/ocl.patch b/var/spack/repos/builtin/packages/sgpp/ocl.patch new file mode 100644 index 0000000000..fbdca6ff18 --- /dev/null +++ b/var/spack/repos/builtin/packages/sgpp/ocl.patch @@ -0,0 +1,13 @@ +diff --git a/datadriven/examplesOCL/multiEvalPerformance.cpp b/datadriven/examplesOCL/multiEvalPerformance.cpp +index 60c281353..373e8509d 100644 +--- a/datadriven/examplesOCL/multiEvalPerformance.cpp ++++ b/datadriven/examplesOCL/multiEvalPerformance.cpp +@@ -17,7 +17,7 @@ int main(int argc, char** argv) { + std::string fileName = "debugging.arff"; + + sgpp::datadriven::ARFFTools arffTools; +- sgpp::datadriven::Dataset dataset = arffTools.readARFFFromeFile(fileName); ++ sgpp::datadriven::Dataset dataset = arffTools.readARFFFromFile(fileName); + + // sgpp::base::DataVector *classes = dataset.getClasses(); + sgpp::base::DataMatrix& trainingData = dataset.getData(); diff --git a/var/spack/repos/builtin/packages/sgpp/package.py b/var/spack/repos/builtin/packages/sgpp/package.py new file mode 100644 index 0000000000..7e5b907374 --- /dev/null +++ b/var/spack/repos/builtin/packages/sgpp/package.py @@ -0,0 +1,205 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Sgpp(SConsPackage): + """SGpp is a library and framework for sparse grids in different flavors. + SGpp supports both hierarchical spatially-adaptive sparse grids and the + dimensionally-adaptive sparse grid combination technique.""" + + homepage = "https://sgpp.sparsegrids.org" + url = "https://github.com/SGpp/SGpp/archive/v3.2.0.tar.gz" + git = "https://github.com/SGpp/SGpp.git" + + maintainers = ['G-071', 'leiterrl', 'pfluegdk'] + + # Versions with Python 3 bindings: + version('master', branch='master') + version('3.3.0', sha256='ca4d5b79f315b425ce69b04940c141451a76848bf1bd7b96067217304c68e2d4') + version('3.2.0', sha256='dab83587fd447f92ed8546eacaac6b8cbe65b8db5e860218c0fa2e42f776962d') + # Versions with Python 2 bindings: + version('3.1.0', sha256='6b46bc5b3966e92567d6754130666bdffb7be1d1d2c1b427d7ce964b8eaab526') + version('3.0.0', sha256='4dd9049e664abd7db78c355fea5e192167812f443115d4bf686a51bb1e9bda9c') + + # Patches with bugfixes that are necessary to build old SGpp versions + # with spack. Patches are submitted upstream, but need to applied + # for versions too old to include them as they will not be + # backported for old releases: + + # Patch that ensures libraries will actually + # be copied into prefix/lib upon installation + # (otherwise it would be prefix/lib/sgpp) + # Fixed in SGpp in PR https://github.com/SGpp/SGpp/pull/222 + patch('directory.patch', when='@:3.2.0') + # Fix faulty setup.py introduced in 3.2.0 + # Fixed in SGpp in version 3.3.0 + patch('fix-setup-py.patch', when='@3.2.0') + # Fix compilation issue with opencl introduced in 3.2.0 + # Fixed in SGpp in PR https://github.com/SGpp/SGpp/pull/219 + patch('ocl.patch', when='@3.2.0+opencl') + # Fixes compilation with AVX512 and datadriven + # Fixed in SGpp in PR https://github.com/SGpp/SGpp/pull/229 + patch('avx512_datadriven_compilation.patch', when='@:3.3.0+datadriven') + + variant('python', default=True, + description='Provide Python bindings for SGpp') + variant('optimization', default=True, + description='Builds the optimization module of SGpp') + variant('pde', default=True, + description='Builds the datadriven module of SGpp') + variant('quadrature', default=True, + description='Builds the datadriven module of SGpp') + variant('datadriven', default=False, + description='Builds the datadriven module of SGpp') + variant('misc', default=False, + description='Builds the misc module of SGpp') + variant('combigrid', default=False, + description='Builds the combigrid module of SGpp') + variant('solver', default=True, + description='Builds the solver module of SGpp') + variant('opencl', default=False, + description='Enables support for OpenCL accelerated operations') + variant('mpi', default=False, + description='Enables support for MPI-distributed operations') + + # Java variant deactivated due to spack issue #987 + # variant('java', default=False, + # description='Provide Java bindings for SGpp') + # depends_on('swig@3:', when='+java', type=('build')) + # extends('openjdk', when='+java') + + # Mandatory dependencies + depends_on('scons@2.5.1', when='@:3.1.0', type=('build')) + depends_on('scons@3:', when='@3.2.0:', type=('build')) + depends_on('zlib', type=('link')) + # Python dependencies + extends('python', when='+python') + depends_on('py-setuptools', when='+python', type=('build')) + # Python 3 support was added in version 3.2.0 + depends_on('python@2.7:2.8', when='@:3.1.0+python', type=('build', 'run')) + depends_on('python@3:', when='@3.2.0:+python', type=('build', 'run')) + depends_on('swig@3:', when='+python', type=('build')) + # Python libraries (version depends on whether we use Python 2 or 3) + depends_on('py-numpy@:1.16', when='@:3.1.0+python', type=('build', 'run')) + depends_on('py-numpy@1.17:', when='@3.2.0:+python', type=('build', 'run')) + depends_on('py-scipy@:1.2.3', when='@:3.1.0+python', type=('build', 'run')) + depends_on('py-scipy@1.3.0:', when='@3.2.0:+python', type=('build', 'run')) + # OpenCL dependency + depends_on('opencl@1.1:', when='+opencl', type=('build', 'run')) + # MPI dependency + depends_on('mpi', when='+mpi', type=('build', 'run')) + # Testing requires boost test + depends_on('boost+test', type=('test')) + + # Compiler with C++11 support is required + conflicts('%gcc@:4.8.4', msg='Compiler with c++11 support is required!') + conflicts('%clang@:3.2', msg='Compiler with c++11 support is required!') + conflicts('%intel@:14', msg='Compiler with c++11 support is required!') + # Solver python bindings are actually using the pde module at one point: + conflicts('-pde', when='+python+solver') + # some modules depend on each other (notably datadriven and misc) + conflicts('+pde', when='-solver') + # Datadriven module requirements + conflicts('+datadriven', when='-solver') + conflicts('+datadriven', when='-optimization') + conflicts('+datadriven', when='-pde') + # Misc module requirements + conflicts('+misc', when='-datadriven') + conflicts('+misc', when='-solver') + conflicts('+misc', when='-optimization') + conflicts('+misc', when='-pde') + conflicts('+misc', when='@:3.1.0', + msg='The misc module was introduced in version 3.2.0') + # Combigrid module requirements (for 3.2.0 or older) + # newer combigrids have no dependencies + conflicts('+combigrid', when='@:3.2.0~optimization') + conflicts('+combigrid', when='@:3.2.0~pde') + conflicts('+combigrid', when='@:3.2.0~solver') + conflicts('+combigrid', when='@:3.2.0~quadrature') + + def build_args(self, spec, prefix): + # Testing parameters + if self.run_tests: + self.args = ['COMPILE_BOOST_TESTS=1', + 'RUN_BOOST_TESTS=1'] + if ('+python' in spec): + self.args.append('RUN_PYTHON_TESTS=1') + if spec.satisfies('@:3.2.0'): + self.args.append('RUN_CPPLINT=1') + else: # argument was renamed after 3.2.0 + self.args.append('CHECK_STYLE=1') + else: + self.args = ['COMPILE_BOOST_TESTS=0', + 'RUN_BOOST_TESTS=0', + 'RUN_PYTHON_TESTS=0'] + if spec.satisfies('@:3.2.0'): + self.args.append('RUN_CPPLINT=0') + else: # argument was renamed after 3.2.0 + self.args.append('CHECK_STYLE=0') + + # Install direction + self.args.append('PREFIX={0}'.format(prefix)) + + # Generate swig bindings? + self.args.append('SG_PYTHON={0}'.format( + '1' if '+python' in spec else '0')) + + # Java variant deactivated due to spack issue #987 + # self.args.append('SG_JAVA={0}'.format( + # '1' if '+java' in spec else '0')) + self.args.append('SG_JAVA=0') + + # Which modules to build? + self.args.append('SG_OPTIMIZATION={0}'.format( + '1' if '+optimization' in spec else '0')) + self.args.append('SG_QUADRATURE={0}'.format( + '1' if '+quadrature' in spec else '0')) + self.args.append('SG_PDE={0}'.format( + '1' if '+pde' in spec else '0')) + self.args.append('SG_DATADRIVEN={0}'.format( + '1' if '+datadriven' in spec else '0')) + self.args.append('SG_COMBIGRID={0}'.format( + '1' if '+combigrid' in spec else '0')) + self.args.append('SG_SOLVER={0}'.format( + '1' if '+solver' in spec else '0')) + + # Misc flag did not exist in older versions + if spec.satisfies('@3.2.0:'): + self.args.append('SG_MISC={0}'.format( + '1' if '+misc' in spec else '0')) + + # SIMD scons parameter (pick according to simd spec) + if 'avx512' in self.spec.target: + self.args.append('ARCH=avx512') + elif 'avx2' in self.spec.target: + self.args.append('ARCH=avx2') + elif 'avx' in self.spec.target: + self.args.append('ARCH=avx') + elif 'fma4' in self.spec.target: + self.args.append('ARCH=fma4') + elif 'sse42' in self.spec.target: + self.args.append('ARCH=sse42') + elif 'sse3' in self.spec.target: + self.args.append('ARCH=sse3') + + # OpenCL Flags + self.args.append('USE_OCL={0}'.format( + '1' if '+opencl' in spec else '0')) + + # Get the mpicxx compiler from the Spack spec + # (makes certain we use the one from spack): + if ('+mpi' in spec): + self.args.append('CXX={0}'.format( + self.spec['mpi'].mpicxx)) + + return self.args + + def install_args(self, spec, prefix): + # Everything is already built, time to install our python bindings: + if '+python' in spec: + setup_py('install', '--prefix={0}'.format(prefix)) + return self.args -- cgit v1.2.3-70-g09d2