From 81668c524bcea8cb60a6a8efd16b576c757f1dbf Mon Sep 17 00:00:00 2001 From: Matthias Wolf Date: Wed, 4 Jul 2018 14:24:39 +0200 Subject: Parquet: new packages (#8601) * parquet: new package Includes the following dependencies: * arrow * flatbuffers Changes for compilation: * snappy * thrift * zstd * parquet: improve recipes (including dependencies) * arrow: remove unused import in recipe * parquet: fix line length for flake8/py27 * parquet, arrow: fix py26 format strings * Address review comments. * arrow: simplify recipe --- var/spack/repos/builtin/packages/arrow/package.py | 74 ++++++++++++++++++++++ .../repos/builtin/packages/flatbuffers/package.py | 36 +++++++++++ .../repos/builtin/packages/parquet/package.py | 52 +++++++++++++++ var/spack/repos/builtin/packages/snappy/package.py | 7 ++ var/spack/repos/builtin/packages/thrift/package.py | 9 +++ var/spack/repos/builtin/packages/zstd/package.py | 6 ++ 6 files changed, 184 insertions(+) create mode 100644 var/spack/repos/builtin/packages/arrow/package.py create mode 100644 var/spack/repos/builtin/packages/flatbuffers/package.py create mode 100644 var/spack/repos/builtin/packages/parquet/package.py (limited to 'var') diff --git a/var/spack/repos/builtin/packages/arrow/package.py b/var/spack/repos/builtin/packages/arrow/package.py new file mode 100644 index 0000000000..9fcc344a32 --- /dev/null +++ b/var/spack/repos/builtin/packages/arrow/package.py @@ -0,0 +1,74 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Arrow(CMakePackage): + """A cross-language development platform for in-memory data. + + This package contains the C++ bindings. + """ + + homepage = "http://arrow.apache.org" + url = "https://github.com/apache/arrow/archive/apache-arrow-0.9.0.tar.gz" + + version('0.9.0', 'ebbd36c362b9e1d398ca612f6d2531ec') + version('0.8.0', '56436f6f61ccc68686b7e0ea30bf4d09') + + depends_on('boost@1.60:') + depends_on('cmake@3.2.0:', type='build') + depends_on('flatbuffers@1.8.0 build_type=Release') # only Release contains flatc + depends_on('rapidjson') + depends_on('snappy~shared') + depends_on('zlib+pic') + depends_on('zstd+pic') + + variant('build_type', default='Release', + description='CMake build type', + values=('Debug', 'FastDebug', 'Release')) + + root_cmakelists_dir = 'cpp' + + def patch(self): + """Prevent `-isystem /usr/include` from appearing, since this confuses gcc. + """ + filter_file(r'(include_directories\()SYSTEM ', + r'\1', + 'cpp/cmake_modules/ThirdpartyToolchain.cmake') + + def cmake_args(self): + args = [ + "-DARROW_USE_SSE=ON", + "-DARROW_BUILD_SHARED=ON", + "-DARROW_BUILD_STATIC=OFF", + "-DARROW_BUILD_TESTS=OFF", + "-DARROW_WITH_BROTLI=OFF", + "-DARROW_WITH_LZ4=OFF", + ] + for dep in ('flatbuffers', 'rapidjson', 'snappy', 'zlib', 'zstd'): + args.append("-D{0}_HOME={1}".format(dep.upper(), + self.spec[dep].prefix)) + args.append("-DZLIB_LIBRARIES={0}".format(self.spec['zlib'].libs)) + return args diff --git a/var/spack/repos/builtin/packages/flatbuffers/package.py b/var/spack/repos/builtin/packages/flatbuffers/package.py new file mode 100644 index 0000000000..0f8eecbf60 --- /dev/null +++ b/var/spack/repos/builtin/packages/flatbuffers/package.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Flatbuffers(CMakePackage): + """Memory Efficient Serialization Library + """ + + homepage = "http://google.github.io/flatbuffers/" + url = "https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz" + + version('1.9.0', '8be7513bf960034f6873326d09521a4b') + version('1.8.0', '276cab8303c4189cbe3b8a70e0515d65') diff --git a/var/spack/repos/builtin/packages/parquet/package.py b/var/spack/repos/builtin/packages/parquet/package.py new file mode 100644 index 0000000000..91e74a97a4 --- /dev/null +++ b/var/spack/repos/builtin/packages/parquet/package.py @@ -0,0 +1,52 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Parquet(CMakePackage): + """C++ bindings for the Apache Parquet columnar data format. + """ + + homepage = "https://github.com/apache/parquet-cpp" + url = "https://github.com/apache/parquet-cpp/archive/apache-parquet-cpp-1.4.0.tar.gz" + + version('1.4.0', '3a3659e65052ef5a76fb88e4922283b9') + + depends_on('arrow') + depends_on('boost') + depends_on('cmake@3.2.0:', type='build') + depends_on('pkgconfig', type='build') + depends_on('thrift+pic') + + variant('build_type', default='Release', + description='CMake build type', + values=('Debug', 'FastDebug', 'Release')) + + def cmake_args(self): + args = ['-DPARQUET_USE_SSE=OFF', '-DPARQUET_BUILD_TESTS=OFF'] + for dep in ('arrow', 'thrift'): + args.append("-D{0}_HOME={1}".format(dep.upper(), + self.spec[dep].prefix)) + return args diff --git a/var/spack/repos/builtin/packages/snappy/package.py b/var/spack/repos/builtin/packages/snappy/package.py index 89ea5b0055..cdcd192056 100644 --- a/var/spack/repos/builtin/packages/snappy/package.py +++ b/var/spack/repos/builtin/packages/snappy/package.py @@ -34,6 +34,7 @@ class Snappy(CMakePackage): version('1.1.7', 'ee9086291c9ae8deb4dac5e0b85bf54a') variant('shared', default=True, description='Build shared libraries') + variant('pic', default=True, description='Build position independent code') def cmake_args(self): spec = self.spec @@ -47,6 +48,12 @@ class Snappy(CMakePackage): return args + def flag_handler(self, name, flags): + flags = list(flags) + if '+pic' in self.spec and name in ('cflags', 'cxxflags'): + flags.append(self.compiler.pic_flag) + return (None, None, flags) + @run_after('install') def install_pkgconfig(self): mkdirp(self.prefix.lib.pkgconfig) diff --git a/var/spack/repos/builtin/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py index 48841db49a..262aa9b106 100644 --- a/var/spack/repos/builtin/packages/thrift/package.py +++ b/var/spack/repos/builtin/packages/thrift/package.py @@ -38,6 +38,7 @@ class Thrift(Package): homepage = "http://thrift.apache.org" url = "http://apache.mirrors.ionfish.org/thrift/0.9.2/thrift-0.9.2.tar.gz" + version('0.11.0', '0be59730ebce071eceaf6bfdb8d3a20e') version('0.10.0', '795c5dd192e310ffff38cfd9430d6b29') version('0.9.3', '88d667a8ae870d5adeca8cb7d6795442') version('0.9.2', '89f63cc4d0100912f4a1f8a9dee63678') @@ -45,6 +46,8 @@ class Thrift(Package): # Currently only support for c-family and python variant('c', default=True, description="Build support for C-family languages") + variant('pic', default=True, + description='Build position independent code') variant('python', default=True, description="Build support for python") @@ -63,6 +66,11 @@ class Thrift(Package): depends_on('zlib', when='+c') depends_on('libevent', when='+c') + def setup_environment(self, spack_env, run_env): + if '+pic' in self.spec: + spack_env.append_flags('CFLAGS', self.compiler.pic_flag) + spack_env.append_flags('CXXFLAGS', self.compiler.pic_flag) + def install(self, spec, prefix): env['PY_PREFIX'] = prefix env['JAVA_HOME'] = spec['java'].prefix @@ -73,6 +81,7 @@ class Thrift(Package): options.append('--with-boost=%s' % spec['boost'].prefix) options.append('--enable-tests=no') + options.append('--with-nodejs=no') options.append('--with-c=%s' % ('yes' if '+c' in spec else 'no')) options.append('--with-python=%s' % ('yes' if '+python' in spec else 'no')) diff --git a/var/spack/repos/builtin/packages/zstd/package.py b/var/spack/repos/builtin/packages/zstd/package.py index 82fe0a0e4a..dea374f931 100644 --- a/var/spack/repos/builtin/packages/zstd/package.py +++ b/var/spack/repos/builtin/packages/zstd/package.py @@ -36,5 +36,11 @@ class Zstd(MakefilePackage): version('1.3.0', '888660a850e33c2dcc7c4f9d0b04d347') version('1.1.2', '4c57a080d194bdaac83f2d3251fc7ffc') + variant('pic', default=True, description='Build position independent code') + + def setup_environment(self, spack_env, run_env): + if '+pic' in self.spec: + spack_env.append_flags('CFLAGS', self.compiler.pic_flag) + def install(self, spec, prefix): make('install', 'PREFIX={0}'.format(prefix)) -- cgit v1.2.3-70-g09d2