From b11767de489426fbe6354b367bc5631b6a11c479 Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Fri, 11 Feb 2022 10:10:31 +0100 Subject: eccodes: update package (#28849) * eccodes: modernize the package * eccodes: add patch fixing location of NetCDF * eccodes: add variant 'shared' * eccodes: enable building with NAG compiler * eccodes: add property 'libs' * eccodes: add variant 'definitions' * eccodes: add variant 'samples' * eccodes: add variant 'tools' --- .../repos/builtin/packages/eccodes/package.py | 310 +++++++++++++++++---- 1 file changed, 258 insertions(+), 52 deletions(-) diff --git a/var/spack/repos/builtin/packages/eccodes/package.py b/var/spack/repos/builtin/packages/eccodes/package.py index 0791559a0b..0a70db94f7 100644 --- a/var/spack/repos/builtin/packages/eccodes/package.py +++ b/var/spack/repos/builtin/packages/eccodes/package.py @@ -3,8 +3,35 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -# -from spack import * + +_definitions = { + # German Meteorological Service (Deutscher Wetterdienst, DWD): + 'edzw': { + 'conflicts': {'when': '@:2.19.1,2.22.0,2.24.0:'}, + 'resources': [ + { + 'when': '@2.20.0', + 'url': 'http://opendata.dwd.de/weather/lib/grib/eccodes_definitions.edzw-2.20.0-1.tar.gz', + 'sha256': 'a92932f8a13c33cba65d3a33aa06c7fb4a37ed12a78e9abe2c5e966402b99af4' + }, + { + 'when': '@2.21.0', + 'url': 'http://opendata.dwd.de/weather/lib/grib/eccodes_definitions.edzw-2.21.0-3.tar.bz2', + 'sha256': '046f1f6450abb3b44c31dee6229f4aab06ca0d3576e27e93e05ccb7cd6e2d9d9' + }, + { + 'when': '@2.22.1', + 'url': 'http://opendata.dwd.de/weather/lib/grib/eccodes_definitions.edzw-2.22.1-1.tar.bz2', + 'sha256': 'be73102a0dcabb236bacd2a70c7b5475f673fda91b49e34df61bef0fa5ad3389' + }, + { + 'when': '@2.23.0', + 'url': 'http://opendata.dwd.de/weather/lib/grib/eccodes_definitions.edzw-2.23.0-4.tar.bz2', + 'sha256': 'c5db32861c7d23410aed466ffef3ca661410d252870a3949442d3ecb176aa338' + } + ] + } +} class Eccodes(CMakePackage): @@ -28,6 +55,7 @@ class Eccodes(CMakePackage): version('2.5.0', sha256='18ab44bc444168fd324d07f7dea94f89e056f5c5cd973e818c8783f952702e4e') version('2.2.0', sha256='1a4112196497b8421480e2a0a1164071221e467853486577c4f07627a702f4c3') + variant('tools', default=False, description='Build the command line tools') variant('netcdf', default=False, description='Enable GRIB to NetCDF conversion tool') variant('jp2k', default='openjpeg', values=('openjpeg', 'jasper', 'none'), @@ -45,6 +73,21 @@ class Eccodes(CMakePackage): variant('python', default=False, description='Enable the Python 2 interface') variant('fortran', default=False, description='Enable the Fortran support') + variant('shared', default=True, + description='Build shared versions of the libraries') + + variant('definitions', + values=disjoint_sets( + ('auto',), + ('default',) + tuple(_definitions.keys()), + ).with_default('auto'), + description="List of definitions to install") + + variant('samples', + values=disjoint_sets( + ('auto',), ('default',), + ).with_default('auto'), + description="List of samples to install") depends_on('netcdf-c', when='+netcdf') # Cannot be built with openjpeg@2.0.x. @@ -66,9 +109,26 @@ class Eccodes(CMakePackage): depends_on('cmake@3.6:', type='build') depends_on('cmake@3.12:', when='@2.19:', type='build') + conflicts('+openmp', when='+pthreads', msg='Cannot enable both POSIX threads and OMP') + conflicts('+netcdf', when='~tools', + msg='Cannot enable the NetCDF conversion tool ' + 'when the command line tools are disabled') + + conflicts('~tools', when='@:2.18.0', + msg='The command line tools can be disabled ' + 'only starting version 2.19.0') + + for center, definitions in _definitions.items(): + kwargs = definitions.get('conflicts', None) + if kwargs: + conflicts('definitions={0}'.format(center), **kwargs) + for kwargs in definitions.get('resources', []): + resource(name=center, destination='spack-definitions', + placement='definitions.{0}'.format(center), **kwargs) + # Enforce linking against the specified JPEG2000 backend, see also # https://github.com/ecmwf/eccodes/commit/2c10828495900ff3d80d1e570fe96c1df16d97fb patch('openjpeg_jasper.patch', when='@:2.16') @@ -76,6 +136,140 @@ class Eccodes(CMakePackage): # CMAKE_INSTALL_RPATH must be a semicolon-separated list. patch('cmake_install_rpath.patch', when='@:2.10') + # Fix a bug preventing cmake from finding NetCDF: + patch('https://github.com/ecmwf/ecbuild/commit/3916c7d22575c45166fcc89edcbe02a6e9b81aa2.patch', + sha256='857454528b666c52eb36ef3aa5d40ae018981b44e129bb8df3c2d3d560e3fa03', + when='@:2.4.0+netcdf') + + @when('%nag+fortran') + def patch(self): + # A number of Fortran source files assume that the kinds of integer and + # real variables are specified in bytes. However, the NAG compiler + # accepts such code only with an additional compiler flag -kind=byte. + # We do not simply add the flag because all user applications would + # have to be compiled with this flag too, which goes against one of the + # purposes of using the NAG compiler: make sure the code does not + # contradict the Fortran standards. The following logic could have been + # implemented as regular patch files, which would, however, be quite + # large. We would also have to introduce several versions of each patch + # file to support different versions of the package. + + patch_kind_files = ['fortran/eccodes_f90_head.f90', + 'fortran/eccodes_f90_tail.f90', + 'fortran/grib_f90_head.f90', + 'fortran/grib_f90_tail.f90', + 'fortran/grib_types.f90'] + + patch_unix_ext_files = [] + + if self.run_tests: + patch_kind_files.extend([ + 'examples/F90/grib_print_data.f90', + 'examples/F90/grib_print_data_static.f90', + # Files that need patching only when the extended regression + # tests are enabled, which we disable unconditionally: + # 'examples/F90/bufr_attributes.f90', + # 'examples/F90/bufr_expanded.f90', + # 'examples/F90/bufr_get_keys.f90', + # 'examples/F90/bufr_read_scatterometer.f90', + # 'examples/F90/bufr_read_synop.f90', + # 'examples/F90/bufr_read_temp.f90', + # 'examples/F90/bufr_read_tempf.f90', + # 'examples/F90/bufr_read_tropical_cyclone.f90', + # 'examples/F90/grib_clone.f90', + # 'examples/F90/grib_get_data.f90', + # 'examples/F90/grib_nearest.f90', + # 'examples/F90/grib_precision.f90', + # 'examples/F90/grib_read_from_file.f90', + # 'examples/F90/grib_samples.f90', + # 'examples/F90/grib_set_keys.f90' + ]) + + patch_unix_ext_files.extend([ + 'examples/F90/bufr_ecc-1284.f90', + 'examples/F90/grib_set_data.f90', + 'examples/F90/grib_set_packing.f90', + # Files that need patching only when the extended regression + # tests are enabled, which we disable unconditionally: + # 'examples/F90/bufr_copy_data.f90', + # 'examples/F90/bufr_get_string_array.f90', + # 'examples/F90/bufr_keys_iterator.f90', + # 'examples/F90/get_product_kind.f90', + # 'examples/F90/grib_count_messages_multi.f90' + ]) + + kwargs = {'string': False, 'backup': False, 'ignore_absent': True} + + # Return the kind and not the size: + filter_file(r'(^\s*kind_of_double\s*=\s*)(\d{1,2})(\s*$)', + '\\1kind(real\\2)\\3', + 'fortran/grib_types.f90', **kwargs) + filter_file(r'(^\s*kind_of_\w+\s*=\s*)(\d{1,2})(\s*$)', + '\\1kind(x\\2)\\3', + 'fortran/grib_types.f90', **kwargs) + + # Replace integer kinds: + for size, r in [(2, 4), (4, 9), (8, 18)]: + filter_file(r'(^\s*integer\((?:kind=)?){0}(\).*)'.format(size), + '\\1selected_int_kind({0})\\2'.format(r), + *patch_kind_files, **kwargs) + + # Replace real kinds: + for size, p, r in [(4, 6, 37), (8, 15, 307)]: + filter_file(r'(^\s*real\((?:kind=)?){0}(\).*)'.format(size), + '\\1selected_real_kind({0}, {1})\\2'.format(p, r), + *patch_kind_files, **kwargs) + + # Enable getarg and exit subroutines: + filter_file(r'(^\s*program\s+\w+)(\s*$)', + '\\1; use f90_unix_env; use f90_unix_proc\\2', + *patch_unix_ext_files, **kwargs) + + @property + def libs(self): + libraries = [] + + query_parameters = self.spec.last_query.extra_parameters + + if 'shared' in query_parameters: + shared = True + elif 'static' in query_parameters: + shared = False + else: + shared = '+shared' in self.spec + + # Return Fortran library if requested: + return_fortran = 'fortran' in query_parameters + # Return C library if either requested or the Fortran library is not + # requested (to avoid overlinking) or the static libraries are + # requested: + return_c = 'c' in query_parameters or not (return_fortran and shared) + # Return MEMFS library only if enabled and the static libraries are + # requested: + return_memfs = '+memfs' in self.spec and not shared + + if return_fortran: + libraries.append('libeccodes_f90') + + if return_c: + libraries.append('libeccodes') + + if return_memfs: + libraries.append('libeccodes_memfs') + + libs = find_libraries( + libraries, root=self.prefix, shared=shared, recursive=True + ) + + if libs and len(libs) == len(libraries): + return libs + + msg = 'Unable to recursively locate {0} {1} libraries in {2}' + raise spack.error.NoLibrariesError( + msg.format('shared' if shared else 'static', + self.spec.name, + self.spec.prefix)) + @run_before('cmake') def check_fortran(self): if '+fortran' in self.spec and self.compiler.fc is None: @@ -83,69 +277,81 @@ class Eccodes(CMakePackage): 'Fortran interface requires a Fortran compiler!') def cmake_args(self): - var_opt_list = [ - ('+pthreads', 'ECCODES_THREADS'), - ('+openmp', 'ECCODES_OMP_THREADS'), - ('+memfs', 'MEMFS'), - ('+python', - 'PYTHON2' if self.spec.satisfies('@2.20.0:') else 'PYTHON'), - ('+fortran', 'FORTRAN')] - - args = ['-DENABLE_%s=%s' % (opt, 'ON' if var in self.spec else 'OFF') - for var, opt in var_opt_list] - - args.extend( - ['-DENABLE_%s=%s' % (opt, 'ON' if self.run_tests else 'OFF') - for opt in ['TESTS', - # Examples are not installed and are - # just part of the test suite. - 'EXAMPLES']]) - - # Unconditionally disable the extended regression testing, - # which requires data downloads. - args.append('-DENABLE_EXTRA_TESTS=OFF') + jp2k = self.spec.variants['jp2k'].value - if '+netcdf' in self.spec: - args.extend(['-DENABLE_NETCDF=ON', - # Prevent overriding by environment variable - # HDF5_ROOT. - '-DHDF5_ROOT=' + self.spec['hdf5'].prefix, - # Prevent possible overriding by environment variables - # NETCDF_ROOT, NETCDF_DIR, and NETCDF_PATH. - '-DNETCDF_PATH=' + self.spec['netcdf-c'].prefix]) - else: - args.append('-DENABLE_NETCDF=OFF') + args = [ + self.define_from_variant('ENABLE_BUILD_TOOLS', 'tools'), + self.define_from_variant('ENABLE_NETCDF', 'netcdf'), + self.define('ENABLE_JPG', jp2k != 'none'), + self.define('ENABLE_JPG_LIBJASPER', jp2k == 'jasper'), + self.define('ENABLE_JPG_LIBOPENJPEG', jp2k == 'openjpeg'), + self.define_from_variant('ENABLE_PNG', 'png'), + self.define_from_variant('ENABLE_AEC', 'aec'), + self.define_from_variant('ENABLE_ECCODES_THREADS', 'pthreads'), + self.define_from_variant('ENABLE_ECCODES_OMP_THREADS', 'openmp'), + self.define_from_variant('ENABLE_MEMFS', 'memfs'), + self.define_from_variant( + 'ENABLE_PYTHON{0}'.format( + '2' if self.spec.satisfies('@2.20.0:') else ''), + 'python'), + self.define_from_variant('ENABLE_FORTRAN', 'fortran'), + self.define('BUILD_SHARED_LIBS', + 'BOTH' if '+shared' in self.spec else 'OFF'), + self.define('ENABLE_TESTS', self.run_tests), + # Examples are not installed and are just part of the test suite: + self.define('ENABLE_EXAMPLES', self.run_tests), + # Unconditionally disable the extended regression tests, since they + # download additional data (~134MB): + self.define('ENABLE_EXTRA_TESTS', False) + ] - jp2k = self.spec.variants['jp2k'].value - args.append('-DENABLE_JPG=' + - ('OFF' if jp2k == 'none' else 'ON')) - args.append('-DENABLE_JPG_LIBJASPER=' + - ('ON' if jp2k == 'jasper' else 'OFF')) - args.append('-DENABLE_JPG_LIBOPENJPEG=' + - ('ON' if jp2k == 'openjpeg' else 'OFF')) + if '+netcdf' in self.spec: + args.extend([ + # Prevent possible overriding by environment variables + # NETCDF_ROOT, NETCDF_DIR, and NETCDF_PATH: + self.define('NETCDF_PATH', self.spec['netcdf-c'].prefix), + # Prevent overriding by environment variable HDF5_ROOT: + self.define('HDF5_ROOT', self.spec['hdf5'].prefix)]) if jp2k == 'openjpeg': - args.append('-DOPENJPEG_PATH=' + self.spec['openjpeg'].prefix) + args.append(self.define('OPENJPEG_PATH', + self.spec['openjpeg'].prefix)) if '+png' in self.spec: - args.extend(['-DENABLE_PNG=ON', - '-DZLIB_ROOT=' + self.spec['zlib'].prefix]) - else: - args.append('-DENABLE_PNG=OFF') + args.append(self.define('ZLIB_ROOT', self.spec['zlib'].prefix)) if '+aec' in self.spec: - args.extend(['-DENABLE_AEC=ON', - # Prevent overriding by environment variables - # AEC_DIR and AEC_PATH. - '-DAEC_DIR=' + self.spec['libaec'].prefix]) - else: - args.append('-DENABLE_AEC=OFF') + # Prevent overriding by environment variables AEC_DIR and AEC_PATH: + args.append(self.define('AEC_DIR', self.spec['libaec'].prefix)) if '^python' in self.spec: - args.append('-DPYTHON_EXECUTABLE:FILEPATH=' + python.path) + args.append(self.define('PYTHON_EXECUTABLE', python.path)) + + definitions = self.spec.variants['definitions'].value + + if 'auto' not in definitions: + args.append(self.define('ENABLE_INSTALL_ECCODES_DEFINITIONS', + 'default' in definitions)) + + samples = self.spec.variants['samples'].value + + if 'auto' not in samples: + args.append(self.define('ENABLE_INSTALL_ECCODES_SAMPLES', + 'default' in samples)) return args + @run_after('install') + def install_extra_definitions(self): + noop = set(['auto', 'none', 'default']) + for center in self.spec.variants['definitions'].value: + if center not in noop: + center_dir = 'definitions.{0}'.format(center) + install_tree( + join_path(self.stage.source_path, + 'spack-definitions', center_dir), + join_path(self.prefix.share.eccodes, center_dir)) + def check(self): # https://confluence.ecmwf.int/display/ECC/ecCodes+installation with working_dir(self.build_directory): -- cgit v1.2.3-70-g09d2