xios: I/O management for climate models (#6796)

* xios: XML-IO-SERVER library for IO management of climate models

* xios: install to prefix directory

* xios: add a test case

* xios: define versions to use cache and mirror directories

* xios: use proper svn URL for version 910

* xios: package url must be defined, otherwise uninstall fails

* xios: add trunk version to allow building latest revision

* xios: enable parallel build

* xios: allow installation with mpif90 compiler wrappers

* xios: use mpi variants of dependencies

* xios: allow user to specify MPI launcher via MPIRUN environment variable

* xios: prefer spack versions of blitz and boost (needed for gcc@7)

* xios: enable builds with clang+gfortran on OSX

* xios: use numeric versions for releases, @develop for trunk

* xios: different patches required for @1.0 and @develop

* xios: patch function for clang works across revisions

* xios: link blitz library

* xios: workaround bug 17782 in llvm

* xios: whitespace adjustments for flake8

* xios: avoid bare except clause

* xios: netcdf-fortran inherits ^netcdf+mpi from xios dependencies

4 files changed, 289 insertions, 0 deletions

diff --git a/var/spack/repos/builtin/packages/xios/bld_extern_1.0.patch b/var/spack/repos/builtin/packages/xios/bld_extern_1.0.patch
new file mode 100644
index 0000000000..cf2aeaaa4e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/xios/bld_extern_1.0.patch
@@ -0,0 +1,27 @@
+--- a/bld.cfg	2017-12-23 11:41:14.000000000 +1100
++++ b/bld.cfg	2017-12-23 15:34:53.914024629 +1100
+@@ -24,9 +24,6 @@
+ 
+ search_src           true
+ src::zzz .
+-src::date $PWD/extern/boost/src/date_time
+-src::blitz $PWD/extern/blitz/src
+-src::netcdf $PWD/extern/netcdf4
+ bld::lib xios
+ bld::target libxios.a
+ #bld::target generate_fortran_interface.exe 
+@@ -37,11 +34,11 @@
+ bld::tool::fc        %FCOMPILER
+ bld::tool::fpp       %FPP
+ bld::tool::cpp       %CPP
+-bld::tool::cppflags  %CBASE_INC -I${PWD}/extern/src_netcdf -I${PWD}/extern/boost/include -I${PWD}/extern/rapidxml/include -I${PWD}/extern/blitz/include 
+-bld::tool::fppflags  %BASE_INC -I${PWD}/extern/boost/include -I${PWD}/extern/rapidxml/include 
++bld::tool::cppflags  %CBASE_INC -I${PWD}/extern/rapidxml/include
++bld::tool::fppflags  %BASE_INC -I${PWD}/extern/rapidxml/include 
+ bld::tool::ld        %LINKER
+ bld::tool::ldflags   %LD_FLAGS 
+-bld::tool::cflags    %CFLAGS %CBASE_INC -I${PWD}/extern/src_netcdf -I${PWD}/extern/boost/include -I${PWD}/extern/rapidxml/include -I${PWD}/extern/blitz/include 
++bld::tool::cflags    %CFLAGS %CBASE_INC -I${PWD}/extern/rapidxml/include
+ bld::tool::fflags    %FFLAGS %FBASE_INC 
+ bld::tool::cppkeys   %CPP_KEY
+ bld::tool::fppkeys   %CPP_KEY
diff --git a/var/spack/repos/builtin/packages/xios/bld_extern_1.x.patch b/var/spack/repos/builtin/packages/xios/bld_extern_1.x.patch
new file mode 100644
index 0000000000..fe52da1192
--- /dev/null
+++ b/var/spack/repos/builtin/packages/xios/bld_extern_1.x.patch
@@ -0,0 +1,27 @@
+--- a/bld.cfg	2017-12-23 15:21:01.458603785 +1100
++++ b/bld.cfg	2017-12-23 15:36:36.667684136 +1100
+@@ -24,9 +24,6 @@
+ 
+ search_src           true
+ src::zzz .
+-src::date $PWD/extern/boost/src/date_time
+-src::blitz $PWD/extern/blitz/src
+-src::netcdf $PWD/extern/netcdf4
+ src::remap $PWD/extern/remap/src
+ bld::lib xios
+ bld::target libxios.a 
+@@ -46,11 +43,11 @@
+ bld::tool::fc        %FCOMPILER
+ bld::tool::fpp       %FPP
+ bld::tool::cpp       %CPP
+-bld::tool::cppflags  %CBASE_INC -I${PWD}/extern/src_netcdf -I${PWD}/extern/boost/include -I${PWD}/extern/rapidxml/include -I${PWD}/extern/blitz/include 
+-bld::tool::fppflags  %BASE_INC -I${PWD}/extern/boost/include -I${PWD}/extern/rapidxml/include 
++bld::tool::cppflags  %CBASE_INC -I${PWD}/extern/rapidxml/include
++bld::tool::fppflags  %BASE_INC -I${PWD}/extern/rapidxml/include
+ bld::tool::ld        %LINKER
+ bld::tool::ldflags   %LD_FLAGS 
+-bld::tool::cflags    %CFLAGS %CBASE_INC -I${PWD}/extern/src_netcdf -I${PWD}/extern/boost/include -I${PWD}/extern/rapidxml/include -I${PWD}/extern/blitz/include 
++bld::tool::cflags    %CFLAGS %CBASE_INC -I${PWD}/extern/rapidxml/include
+ bld::tool::fflags    %FFLAGS %FBASE_INC 
+ bld::tool::cppkeys   %CPP_KEY
+ bld::tool::fppkeys   %CPP_KEY
diff --git a/var/spack/repos/builtin/packages/xios/llvm_bug_17782.patch b/var/spack/repos/builtin/packages/xios/llvm_bug_17782.patch
new file mode 100644
index 0000000000..9eb5c21950
--- /dev/null
+++ b/var/spack/repos/builtin/packages/xios/llvm_bug_17782.patch
@@ -0,0 +1,11 @@
+--- a/src/duration.cpp	2017-12-28 10:45:09.000000000 +1100
++++ b/src/duration.cpp	2017-12-28 11:01:40.000000000 +1100
+@@ -43,7 +43,7 @@
+       StdIStream& operator>>(StdIStream& in , CDuration& duration)
+       {
+         duration = NoneDu;
+-        double v = 1.0;
++        int v = 1;
+         char   c = '/';
+         bool   invalidUnit = false;
+         CDuration sentinel = NoneDu;
diff --git a/var/spack/repos/builtin/packages/xios/package.py b/var/spack/repos/builtin/packages/xios/package.py
new file mode 100644
index 0000000000..a3df5e9ca8
--- /dev/null
+++ b/var/spack/repos/builtin/packages/xios/package.py
@@ -0,0 +1,224 @@
+##############################################################################
+# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+#
+from spack import *
+import os
+
+
+class Xios(Package):
+    """XML-IO-SERVER library for IO management of climate models."""
+
+    homepage = "https://forge.ipsl.jussieu.fr/ioserver/wiki"
+    url = "https://forge.ipsl.jussieu.fr/ioserver"
+
+    version('1.0', revision=910, 
+            svn='http://forge.ipsl.jussieu.fr/ioserver/svn/XIOS/branchs/xios-1.0')
+    version('develop', svn='http://forge.ipsl.jussieu.fr/ioserver/svn/XIOS/trunk')
+
+    variant('mode', values=('debug', 'dev', 'prod'), default='dev',
+            description='Build for debugging, development or production')
+    # NOTE: oasis coupler could be supported with a variant
+
+    # Use spack versions of blitz and netcdf for compatibility
+    # with recent compilers and optimised platform libraries:
+    patch('bld_extern_1.0.patch', when='@:1.0')
+    patch('bld_extern_1.x.patch', when='@1.1:')
+
+    # Workaround bug #17782 in llvm, where reading a double
+    # followed by a character is broken (e.g. duration '1d'):
+    patch('llvm_bug_17782.patch', when='@1.1: %clang')
+
+    depends_on('netcdf+mpi')
+    depends_on('netcdf-fortran')
+    depends_on('hdf5+mpi')
+    depends_on('mpi')
+    depends_on('boost')
+    depends_on('blitz')
+    depends_on('perl', type='build')
+    depends_on('perl-uri-escape', type='build')
+    depends_on('gmake', type='build')
+
+    @when('%clang')
+    def patch(self):
+        """Fix type references that are ambiguous for clang."""
+        for dirpath, dirnames, filenames in os.walk('src'):
+            for filename in filenames:
+                filepath = os.path.join(dirpath, filename)
+                # Use boost definition of type shared_ptr:
+                filter_file(r'([^:/])shared_ptr<',
+                            r'\1boost::shared_ptr<', filepath)
+                # Use type long for position in output stream:
+                filter_file(r'oss.tellp\(\) *- *startPos',
+                            r'(long)oss.tellp() - startPos', filepath)
+
+    def xios_env(self):
+        file = join_path('arch', 'arch-SPACK.env')
+        touch(file)
+
+    def xios_path(self):
+        file = join_path('arch', 'arch-SPACK.path')
+        spec = self.spec
+        paths = {'NETCDF_INC_DIR': spec['netcdf'].prefix.include,
+                 'NETCDF_LIB_DIR': spec['netcdf'].prefix.lib,
+                 'HDF5_INC_DIR': spec['hdf5'].prefix.include,
+                 'HDF5_LIB_DIR': spec['hdf5'].prefix.lib}
+        text = r"""
+NETCDF_INCDIR="-I {NETCDF_INC_DIR}"
+NETCDF_LIBDIR="-L {NETCDF_LIB_DIR}"
+NETCDF_LIB="-lnetcdff -lnetcdf"
+
+MPI_INCDIR=""
+MPI_LIBDIR=""
+MPI_LIB=""
+
+HDF5_INCDIR="-I {HDF5_INC_DIR}"
+HDF5_LIBDIR="-L {HDF5_LIB_DIR}"
+HDF5_LIB="-lhdf5_hl -lhdf5"
+
+OASIS_INCDIR=""
+OASIS_LIBDIR=""
+OASIS_LIB=""
+"""
+        with open(file, 'w') as f:
+            f.write(text.format(**paths))
+
+    def xios_fcm(self):
+        file = join_path('arch', 'arch-SPACK.fcm')
+        spec = self.spec
+        param = dict()
+        param['MPICXX'] = spec['mpi'].mpicxx
+        param['MPIFC'] = spec['mpi'].mpifc
+        param['CC'] = self.compiler.cc
+        param['FC'] = self.compiler.fc
+        param['BOOST_INC_DIR'] = spec['boost'].prefix.include
+        param['BOOST_LIB_DIR'] = spec['boost'].prefix.lib
+        param['BLITZ_INC_DIR'] = spec['blitz'].prefix.include
+        param['BLITZ_LIB_DIR'] = spec['blitz'].prefix.lib
+        if spec.satisfies('%clang platform=darwin'):
+            param['LIBCXX'] = '-lc++'
+        else:
+            param['LIBCXX'] = '-lstdc++'
+
+        if any(map(spec.satisfies, ('%gcc', '%intel', '%clang'))):
+            text = r"""
+%CCOMPILER      {MPICXX}
+%FCOMPILER      {MPIFC}
+%LINKER         {MPIFC}
+
+%BASE_CFLAGS    -ansi -w -D_GLIBCXX_USE_CXX11_ABI=0 \
+                -I{BOOST_INC_DIR} -I{BLITZ_INC_DIR}
+%PROD_CFLAGS    -O3 -DBOOST_DISABLE_ASSERTS
+%DEV_CFLAGS     -g -O2
+%DEBUG_CFLAGS   -g
+
+%BASE_FFLAGS    -D__NONE__
+%PROD_FFLAGS    -O3
+%DEV_FFLAGS     -g -O2
+%DEBUG_FFLAGS   -g
+
+%BASE_INC       -D__NONE__
+%BASE_LD        -L{BOOST_LIB_DIR} -L{BLITZ_LIB_DIR} -lblitz {LIBCXX}
+
+%CPP            {CC} -E
+%FPP            {CC} -E -P -x c
+%MAKE           gmake
+""".format(**param)
+        elif spec.satisfies('%cce'):
+            # In the CC compiler prior to cce/8.3.7,
+            # optimisation must be reduced to avoid a bug,
+            # as reported by Mike Rezny at the UK Met Office:
+            if spec.satisfies('%cce@8.3.7:'):
+                param.update({'CC_OPT_DEV': '-O2', 'CC_OPT_PROD': '-O3'})
+            else:
+                param.update({'CC_OPT_DEV': '-O1', 'CC_OPT_PROD': '-O1'})
+
+            text = r"""
+%CCOMPILER      {MPICXX}
+%FCOMPILER      {MPIFC}
+%LINKER         {MPIFC}
+
+%BASE_CFLAGS    -DMPICH_SKIP_MPICXX -h msglevel_4 -h zero -h gnu \
+                -I{BOOST_INC_DIR} -I{BLITZ_INC_DIR}
+%PROD_CFLAGS    {CC_OPT_PROD} -DBOOST_DISABLE_ASSERTS
+%DEV_CFLAGS     {CC_OPT_DEV}
+%DEBUG_CFLAGS   -g
+
+%BASE_FFLAGS    -em -m 4 -e0 -eZ
+%PROD_FFLAGS    -O3
+%DEV_FFLAGS     -G2
+%DEBUG_FFLAGS   -g
+
+%BASE_INC       -D__NONE__
+%BASE_LD        -D__NONE__ -L{BOOST_LIB_DIR} -L{BLITZ_LIB_DIR} -lblitz
+
+%CPP            cpp
+%FPP            cpp -P -CC
+%MAKE           gmake
+""".format(**param)
+        else:
+            raise InstallError('Unsupported compiler.')
+
+        with open(file, 'w') as f:
+            f.write(text)
+
+    def install(self, spec, prefix):
+        env['CC'] = spec['mpi'].mpicc
+        env['CXX'] = spec['mpi'].mpicxx
+        env['F77'] = spec['mpi'].mpif77
+        env['FC'] = spec['mpi'].mpifc
+
+        options = ['--full',
+                   '--%s' % spec.variants['mode'].value,
+                   '--arch', 'SPACK',
+                   '--netcdf_lib', 'netcdf4_par',
+                   '--job', str(make_jobs)]
+
+        self.xios_env()
+        self.xios_path()
+        self.xios_fcm()
+
+        make_xios = Executable('./make_xios')
+        make_xios(*options)
+
+        mkdirp(spec.prefix)
+        install_tree('bin', spec.prefix.bin)
+        install_tree('lib', spec.prefix.lib)
+        install_tree('inc', spec.prefix.include)
+        install_tree('etc', spec.prefix.etc)
+        install_tree('cfg', spec.prefix.cfg)
+
+    @run_after('install')
+    @on_package_attributes(run_tests=True)
+    def check_build(self):
+        mpirun = os.getenv('MPIRUN')
+        if mpirun is None:
+            mpirun = 'mpiexec'
+        mpiexec = Executable(mpirun)
+        with working_dir('inputs'):
+            try:
+                mpiexec('-n', '2', join_path('..', 'bin', 'test_client.exe'))
+            except Exception:
+                raise InstallError(
+                    'Test failed; defining MPIRUN variable may help.')