From dd5f4cb251f77a4d4122a4eb26f44beb68bebc92 Mon Sep 17 00:00:00 2001 From: 健美猫 Date: Tue, 25 Apr 2017 19:40:23 +0800 Subject: A new package: CNTK. (#3578) --- var/spack/repos/builtin/packages/cntk/build.patch | 401 +++++++++++++++++++++ .../packages/cntk/kaldireader-openblas.patch | 32 ++ var/spack/repos/builtin/packages/cntk/package.py | 117 ++++++ .../repos/builtin/packages/cntk1bitsgd/package.py | 44 +++ .../builtin/packages/multiverso/cmake-143187.patch | 28 ++ .../repos/builtin/packages/multiverso/package.py | 48 +++ 6 files changed, 670 insertions(+) create mode 100644 var/spack/repos/builtin/packages/cntk/build.patch create mode 100644 var/spack/repos/builtin/packages/cntk/kaldireader-openblas.patch create mode 100644 var/spack/repos/builtin/packages/cntk/package.py create mode 100644 var/spack/repos/builtin/packages/cntk1bitsgd/package.py create mode 100644 var/spack/repos/builtin/packages/multiverso/cmake-143187.patch create mode 100644 var/spack/repos/builtin/packages/multiverso/package.py diff --git a/var/spack/repos/builtin/packages/cntk/build.patch b/var/spack/repos/builtin/packages/cntk/build.patch new file mode 100644 index 0000000000..0b1e46e911 --- /dev/null +++ b/var/spack/repos/builtin/packages/cntk/build.patch @@ -0,0 +1,401 @@ +From 6ee89684a8e7b509f977bab00573c73563df4bc6 Mon Sep 17 00:00:00 2001 +From: Jianwen WEI +Date: Wed, 5 Apr 2017 16:52:54 +0800 +Subject: [PATCH 1/1] Refine the build process to allow reuse pre-installed + packages: multiverso 1bit-sgd kaldi openfst. + +--- + Makefile | 69 +++++++++++++++--------------- + configure | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- + 2 files changed, 178 insertions(+), 35 deletions(-) + +diff --git a/Makefile b/Makefile +index 0586fd94b..bbd75e08c 100644 +--- a/Makefile ++++ b/Makefile +@@ -25,14 +25,18 @@ + # If not specified, GPU will not be enabled + # CUB_PATH= path to NVIDIA CUB installation, so $(CUB_PATH)/cub/cub.cuh exists + # defaults to /usr/local/cub-1.4.1 +-# CUDNN_PATH= path to NVIDIA cuDNN installation so $(CUDNN_PATH)/cuda/include/cudnn.h exists ++# CUDNN_PATH= path to NVIDIA cuDNN installation so $(CUDNN_PATH)/include/cudnn.h exists + # CuDNN version needs to be 5.0 or higher. + # KALDI_PATH= Path to Kaldi + # If not specified, Kaldi plugins will not be built ++# OPENFST_PATH= Path to OpenFST ++# OpenFST is required by Kaldi Reader. + # OPENCV_PATH= path to OpenCV 3.1.0 installation, so $(OPENCV_PATH) exists + # defaults to /usr/local/opencv-3.1.0 + # PROTOBUF_PATH= path to Protocol Buffers 3.1.0 installation, so $(PROTOBUF_PATH) exists + # defaults to /usr/local/protobuf-3.1.0 ++# MULTIVERSO_PATH= path to Multiverso installation or source code, so $(MULTIVERSO_PATH) exists ++# defaults to Source/Multiverso + # LIBZIP_PATH= path to libzip installation, so $(LIBZIP_PATH) exists + # defaults to /usr/local/ + # BOOST_PATH= path to Boost installation, so $(BOOST_PATH)/include/boost/test/unit_test.hpp +@@ -149,8 +153,8 @@ ifdef CUDA_PATH + + # Set up cuDNN if needed + ifdef CUDNN_PATH +- INCLUDEPATH += $(CUDNN_PATH)/cuda/include +- LIBPATH += $(CUDNN_PATH)/cuda/lib64 ++ INCLUDEPATH += $(CUDNN_PATH)/include ++ LIBPATH += $(CUDNN_PATH)/lib64 + LIBS_LIST += cudnn + COMMON_FLAGS +=-DUSE_CUDNN + endif +@@ -188,16 +192,11 @@ ifeq ("$(MATHLIB)","openblas") + CPPFLAGS += -DUSE_OPENBLAS + endif + +- ++# Set up kaldi and openfst for Kaldi2Reader if needed + ifdef KALDI_PATH +- ########## Copy includes and defines from $(KALDI_PATH)/src/kaldi.mk ########## +- FSTROOT = $(KALDI_PATH)/tools/openfst +- ATLASINC = $(KALDI_PATH)/tools/ATLAS/include +- +- INCLUDEPATH += $(KALDI_PATH)/src $(ATLASINC) $(FSTROOT)/include ++ INCLUDEPATH += $(KALDI_PATH)/include $(OPENFST_PATH)/incldue + CPPFLAGS += -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_ATLAS -DHAVE_OPENFST_GE_10400 +- +- KALDI_LIBPATH += $(KALDI_PATH)/src/lib ++ KALDI_LIBPATH += $(KALDI_PATH)/lib $(OPENFST_PATH)/lib + KALDI_LIBS_LIST := kaldi-util kaldi-matrix kaldi-base kaldi-hmm kaldi-cudamatrix kaldi-nnet kaldi-lat + KALDI_LIBS := $(addprefix -l,$(KALDI_LIBS_LIST)) + endif +@@ -316,7 +315,7 @@ SRC += $(PP_SRC) + $(PERF_PROFILER_LIB): $(PP_OBJ) + @echo $(SEPARATOR) + @echo creating $@ for $(ARCH) with build type $(BUILDTYPE) +- @mkdir -p $(dir $@) ++ -@mkdir -p $(dir $@) + $(CXX) $(LDFLAGS) -shared $(patsubst %,$(RPATH)%, $(ORIGINDIR)) -o $@ $^ + + +@@ -920,17 +919,13 @@ endif + + ifeq ("$(CNTK_ENABLE_1BitSGD)","true") + +-ifeq (,$(wildcard Source/1BitSGD/*.h)) +- $(error Build with 1bit-SGD was requested but cannot find the code. Please check https://github.com/Microsoft/CNTK/wiki/Enabling-1bit-SGD for instructions) +-endif +- +- INCLUDEPATH += $(SOURCEDIR)/1BitSGD ++ INCLUDEPATH += $(ONEBITSGD_PATH) + + COMMON_FLAGS += -DCNTK_PARALLEL_TRAINING_SUPPORT + # temporarily adding to 1bit, need to work with others to fix it + endif + +- ++ + ######################################## + # ASGD(multiverso) setup + ######################################## +@@ -938,13 +933,9 @@ endif + + ifeq ("$(CNTK_ENABLE_ASGD)","true") + +-ifeq (,$(wildcard Source/Multiverso/include/multiverso/*.h)) +- $(error Build with Multiverso was requested but cannot find the code. Please check https://github.com/Microsoft/CNTK/wiki/Multiple-GPUs-and-machines#24-data-parallel-asgd to learn more.) +-endif +- + lMULTIVERSO:=-lmultiverso + +-INCLUDEPATH += $(SOURCEDIR)/Multiverso/include ++INCLUDEPATH += $(MULTIVERSO_PATH)/include + COMMON_FLAGS += -DASGD_PARALLEL_SUPPORT + + MULTIVERSO_LIB:=$(LIBDIR)/libmultiverso.so +@@ -958,11 +949,16 @@ MULTIVERSO_CMAKE_BUILDTYPE=Debug + endif + + # TODO need to align Multiverso OpenMP with the one we use (libiomp). For now, disabled. +-$(MULTIVERSO_LIB): ++$(MULTIVERSO_LIB): ++ifneq ("$(wildcard $(MULTIVERSO_PATH)/lib/libmultiverso.so)","") ++ # reuse pre-installed multiverso library to $(LIBDIR) then skip building $(MULTIVERSO_LIB) when possible ++ @mkdir -p $(LIBDIR) ++ cp $(MULTIVERSO_PATH)/lib/libmultiverso.so $(LIBDIR) ++else + @echo "Build Multiverso lib" + @mkdir -p $(LIBDIR) + @mkdir -p $(BINDIR) +- @mkdir -p $(SOURCEDIR)/Multiverso/build/$(BUILDTYPE) ++ @mkdir -p $(MULTIVERSO_PATH)/build/$(BUILDTYPE) + @cmake -DCMAKE_VERBOSE_MAKEFILE=TRUE \ + -DCMAKE_CXX_COMPILER=$(CXX) \ + -DOpenMP_CXX_FLAGS="" \ +@@ -976,17 +972,22 @@ $(MULTIVERSO_LIB): + -DCMAKE_BUILD_TYPE=$(MULTIVERSO_CMAKE_BUILDTYPE) \ + -B./Source/Multiverso/build/$(BUILDTYPE) -H./Source/Multiverso + @make VERBOSE=1 -C ./Source/Multiverso/build/$(BUILDTYPE) -j multiverso ++endif + + UNITTEST_MULTIVERSO_SRC = \ +- $(SOURCEDIR)/Multiverso/Test/unittests/test_array.cpp \ +- $(SOURCEDIR)/Multiverso/Test/unittests/test_blob.cpp \ +- $(SOURCEDIR)/Multiverso/Test/unittests/test_kv.cpp \ +- $(SOURCEDIR)/Multiverso/Test/unittests/test_message.cpp \ +- $(SOURCEDIR)/Multiverso/Test/unittests/test_multiverso.cpp \ +- $(SOURCEDIR)/Multiverso/Test/unittests/test_node.cpp \ +- $(SOURCEDIR)/Multiverso/Test/unittests/test_sync.cpp \ +- +-UNITTEST_MULTIVERSO_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(UNITTEST_MULTIVERSO_SRC)) ++ $(MULTIVERSO_PATH)/Test/unittests/test_array.cpp \ ++ $(MULTIVERSO_PATH)/Test/unittests/test_blob.cpp \ ++ $(MULTIVERSO_PATH)/Test/unittests/test_kv.cpp \ ++ $(MULTIVERSO_PATH)/Test/unittests/test_message.cpp \ ++ $(MULTIVERSO_PATH)/Test/unittests/test_multiverso.cpp \ ++ $(MULTIVERSO_PATH)/Test/unittests/test_node.cpp \ ++ $(MULTIVERSO_PATH)/Test/unittests/test_sync.cpp \ ++ ++UNITTEST_MULTIVERSO_OBJ := $(patsubst $(MULTIVERSO_PATH)/Test/unittests/%.cpp, $(OBJDIR)/Source/Multiverso/Test/unittests/%.o, $(UNITTEST_MULTIVERSO_SRC)) ++ ++$(OBJDIR)/Source/Multiverso/Test/unittests/%.o: $(MULTIVERSO_PATH)/Test/unittests/%.cpp ++ -@ mkdir -p $(OBJDIR)/Source/Multiverso/Test/unittests ++ $(CXX) -c $< -o $@ $(COMMON_FLAGS) $(CPPFLAGS) $(CXXFLAGS) $(INCLUDEPATH:%=-I%) + + UNITTEST_MULTIVERSO := $(BINDIR)/multiversotests + +diff --git a/configure b/configure +index 2c2e5cee7..c9c0072c9 100755 +--- a/configure ++++ b/configure +@@ -41,6 +41,14 @@ boost_check=include/boost/test/unit_test.hpp + protobuf_path= + protobuf_check=lib/libprotobuf.a + ++# Multiverso library ++multiverso_path= ++multiverso_check=include/multiverso/multiverso.h ++ ++# 1bitsgd library ++onebitsgd_path= ++onebitsgd_check=BlockMomentumSGD.h ++ + # MPI library + mpi_path= + mpi_check=include/mpi.h +@@ -49,6 +57,10 @@ have_kaldi=no + kaldi_path= + kaldi_check=src/kaldi.mk + ++have_openfst=no ++openfst_path= ++openfst_check=include/fst/fst.h ++ + have_buildtype=no + buildtype= + default_buildtype=release +@@ -67,7 +79,7 @@ cub_check=cub/cub.cuh + + have_cudnn=no + cudnn_path= +-cudnn_check=cuda/include/cudnn.h ++cudnn_check=include/cudnn.h + + have_opencv=no + opencv_path= +@@ -112,6 +124,7 @@ default_boost="boost-1.60.0" + default_cudas="cuda-8.0 cuda-7.5" + default_nccls="nccl" + default_kaldis="kaldi-trunk kaldi-c024e8aa" ++default_openfst="openfst-1.6" + default_gdk_includes="include/nvidia/gdk" + default_gdk_nvml_libs="src/gdk/nvml/lib" + default_cubs="cub-1.4.1" +@@ -121,6 +134,8 @@ default_protobuf="protobuf-3.1.0" + default_libzips="libzip-1.1.2" + default_swig="swig-3.0.10" + default_mpi="mpi" ++default_multiverso=Source/Multiverso ++default_onebitsgd=Source/1BitSGD + + function default_paths () + { +@@ -178,6 +193,16 @@ function find_protobuf () + find_dir "$default_protobuf" "$protobuf_check" + } + ++function find_multiverso () ++{ ++ find_dir "$default_multiverso" "$multiverso_check" ++} ++ ++function find_onebitsgd () ++{ ++ find_dir "$default_onebitsgd" "$onebitsgd_check" ++} ++ + function find_nccl () + { + find_dir "$default_nccls" "$nccl_check" +@@ -193,6 +218,11 @@ function find_kaldi () + find_dir "$default_kaldis" "$kaldi_check" + } + ++function find_openfst () ++{ ++ find_dir "$default_openfst" "$openfst_check" ++} ++ + function find_gdk_include () + { + find_dir "$default_gdk_includes" "$gdk_include_check" +@@ -352,11 +382,14 @@ function show_help () + echo " --with-openblas[=directory] (experimental) $(show_default $(find_openblas))" + echo " --with-buildtype=(debug|release) $(show_default $default_buildtype)" + echo " --with-kaldi[=directory] $(show_default $(find_kaldi))" ++ echo " --with-openfst[=directory] $(show_default $(find_openfst))" + echo " --with-opencv[=directory] $(show_default $(find_opencv))" + echo " --with-libzip[=directory] $(show_default $(find_libzip))" + echo " --with-code-coverage[=(yes|no)] $(show_default ${default_use_code_coverage})" + echo " --with-boost[=directory] $(show_default $(find_boost))" + echo " --with-protobuf[=directory] $(show_default $(find_protobuf))" ++ echo " --with-multiverso[=directory] $(show_default $(find_multiverso))" ++ echo " --with-1bitsgd[=directory] $(show_default $(find_onebitsgd))" + echo " --with-py-versions=(space-separated list of 27, 34, 35)" + echo " --with-py27-path[=directory] $(show_default $(find_python 27))" + echo " --with-py34-path[=directory] $(show_default $(find_python 34))" +@@ -751,6 +784,46 @@ do + fi + fi + ;; ++ --with-multiverso*) ++ if test x$optarg = x ++ then ++ multiverso_path=$(find_multiverso) ++ if test x$multiverso_path = x ++ then ++ echo "Cannot find Multiverso directory" ++ echo "Please specify a value for --with-multiverso" ++ exit 1 ++ fi ++ else ++ if test $(check_dir $optarg $multiverso_check) = yes ++ then ++ multiverso_path=$optarg ++ else ++ echo "Invalid Multiverso directory $optarg" ++ exit 1 ++ fi ++ fi ++ ;; ++ --with-1bitsgd*) ++ if test x$optarg = x ++ then ++ onebitsgd_path=$(find_onebitsgd) ++ if test x$onebitsgd_path = x ++ then ++ echo "Cannot find 1BitSGD directory" ++ echo "Please specify a value for --with-1bitsgd" ++ exit 1 ++ fi ++ else ++ if test $(check_dir $optarg $onebitsgd_check) = yes ++ then ++ onebitsgd_path=$optarg ++ else ++ echo "Invalid 1BitSGD directory $optarg" ++ exit 1 ++ fi ++ fi ++ ;; + --with-buildtype*) + have_buildtype=yes + case $optarg in +@@ -784,6 +857,27 @@ do + fi + fi + ;; ++ --with-openfst*) ++ have_openfst=yes ++ if test x$optarg = x ++ then ++ openfst_path=$(find_openfst) ++ if test x$openfst_path = x ++ then ++ echo "Cannot find openfst directory" ++ echo "Please specify a value for --with-openfst" ++ exit 1 ++ fi ++ else ++ if test $(check_dir $optarg $openfst_check) ++ then ++ openfst_path=$optarg ++ else ++ echo "Invalid openfst directory $optarg" ++ exit 1 ++ fi ++ fi ++ ;; + --with-opencv*) + have_opencv=yes + if test x$optarg = x +@@ -1049,6 +1143,48 @@ then + fi + fi + ++if test x$kaldi_path != x && test x$openfst_path = x ++then ++ openfst_path=$(find_openfst) ++ if test x$openfst_path = x ++ then ++ echo CNTK KaldiReader requires OpenFST library. Please check ++ echo https://github.com/Microsoft/CNTK/blob/master/Source/Readers/KaldiReaderReadme ++ echo to learn more. ++ exit 1 ++ else ++ echo Found OpenFST library at $openfst_path ++ fi ++fi ++ ++if test $enable_1bitsgd = yes && test x$onebitsgd_path = x ++then ++ onebitsgd_path=$(find_onebitsgd) ++ if test x$onebitsgd_path = x ++ then ++ echo Cannot locate 1BitSGD library. See ++ echo https://github.com/Microsoft/CNTK/wiki/Enabling-1bit-SGD ++ echo for installation instructions. ++ exit 1 ++ else ++ echo Found 1BitSGD library at $onebitsgd_path ++ fi ++fi ++ ++if test $enable_asgd = yes && test x$multiverso_path = x ++then ++ multiverso_path=$(find_multiverso) ++ if test x$multiverso_path = x ++ then ++ echo Build with Multiverso was requested but cannot find the code. Please check ++ echo https://github.com/Microsoft/CNTK/wiki/Multiple-GPUs-and-machines#24-data-parallel-asgd ++ echo to learn more. ++ exit 1 ++ else ++ echo Found Multiverso library at $multiverso_path ++ fi ++fi ++ + if test x$mpi_path = x + then + mpi_path=$(find_mpi) +@@ -1123,6 +1259,12 @@ fi + if test x$protobuf_path != x; then + echo PROTOBUF_PATH=$protobuf_path >> $config + fi ++if test x$multiverso_path != x; then ++ echo MULTIVERSO_PATH=$multiverso_path >> $config ++fi ++if test x$onebitsgd_path != x; then ++ echo ONEBITSGD_PATH=$onebitsgd_path >> $config ++fi + if test x$mpi_path != x; then + echo MPI_PATH=$mpi_path >> $config + fi +-- +2.11.1 + diff --git a/var/spack/repos/builtin/packages/cntk/kaldireader-openblas.patch b/var/spack/repos/builtin/packages/cntk/kaldireader-openblas.patch new file mode 100644 index 0000000000..75bd4146e7 --- /dev/null +++ b/var/spack/repos/builtin/packages/cntk/kaldireader-openblas.patch @@ -0,0 +1,32 @@ +From a5cdb4fdce55813b79e3ee6300f282ba6d7b6b85 Mon Sep 17 00:00:00 2001 +From: Jianwen WEI +Date: Fri, 14 Apr 2017 12:40:44 +0800 +Subject: [PATCH 1/1] Replace ATLAS with OpenBLAS in KaldiReader. + +--- + Makefile | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/Makefile b/Makefile +index bbd75e08c..909457963 100644 +--- a/Makefile ++++ b/Makefile +@@ -189,13 +189,13 @@ ifeq ("$(MATHLIB)","openblas") + INCLUDEPATH += $(OPENBLAS_PATH)/include + LIBPATH += $(OPENBLAS_PATH)/lib + LIBS_LIST += openblas m pthread +- CPPFLAGS += -DUSE_OPENBLAS ++ CPPFLAGS += -DUSE_OPENBLAS -DHAVE_OPENBLAS + endif + + # Set up kaldi and openfst for Kaldi2Reader if needed + ifdef KALDI_PATH + INCLUDEPATH += $(KALDI_PATH)/include $(OPENFST_PATH)/incldue +- CPPFLAGS += -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_ATLAS -DHAVE_OPENFST_GE_10400 ++ CPPFLAGS += -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_OPENFST_GE_10400 + KALDI_LIBPATH += $(KALDI_PATH)/lib $(OPENFST_PATH)/lib + KALDI_LIBS_LIST := kaldi-util kaldi-matrix kaldi-base kaldi-hmm kaldi-cudamatrix kaldi-nnet kaldi-lat + KALDI_LIBS := $(addprefix -l,$(KALDI_LIBS_LIST)) +-- +2.11.1 + diff --git a/var/spack/repos/builtin/packages/cntk/package.py b/var/spack/repos/builtin/packages/cntk/package.py new file mode 100644 index 0000000000..801c69dae4 --- /dev/null +++ b/var/spack/repos/builtin/packages/cntk/package.py @@ -0,0 +1,117 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Cntk(Package): + """The Microsoft Cognitive Toolkit is a unified deep-learning toolkit + that describes neural networks as a series of computational steps + via a directed graph.""" + + homepage = "https://www.microsoft.com/en-us/research/product/cognitive-toolkit" + url = "https://github.com/Microsoft/CNTK/archive/v2.0.beta15.0.tar.gz" + + version('master', git='https://github.com/Microsoft/CNTK.git', branch='master') + version('2.0.rc1', url='https://github.com/Microsoft/CNTK/archive/v2.0.rc1.tar.gz') + + variant('opencv', default=False, description="Enable OpenCV support.") + variant('kaldi', default=False, description="Enable Kaldi support.") + variant('asgd', default=True, description="Enable DataParallelASGD powered by Multiverso.") + variant('1bitsgd', default=False, description="Enable 1bitsgd support.") + variant('cuda', default=False, description="Enable CUDA support.") + variant('debug', default=False, description="Debug build.") + + depends_on('libzip') + depends_on('openblas') + depends_on('mpi') + depends_on('boost') + depends_on('protobuf~shared') + # CNTK depends on kaldi@c02e8. + # See https://github.com/Microsoft/CNTK/blob/master/Tools/docker/CNTK-CPUOnly-Image/Dockerfile#L105-L125 + depends_on('kaldi@c024e8', when='+kaldi') + depends_on('opencv', when='+opencv') + depends_on('cuda', when='+cuda') + depends_on('cub@1.4.1', when='+cuda') + depends_on('cudnn@5.1', when='+cuda') + depends_on('nccl', when='+cuda') + depends_on('cntk1bitsgd@c8b77d', when='+1bitsgd') + depends_on('multiverso@143187', when='+asgd') + + # Patch CNTN's build process to use libs installed outside CNTK source tree + # multiverso, kaldi, openfst + patch('build.patch') + # Patch to fix BLAS inconsistency between CNTK and KaldiReader + patch('kaldireader-openblas.patch') + + def install(self, spec, prefix): + args = [] + + args.append('--with-mpi=' + spec['mpi'].prefix) + args.append('--with-openblas=' + spec['openblas'].prefix) + args.append('--with-libzip=' + spec['libzip'].prefix) + args.append('--with-boost=' + spec['boost'].prefix) + args.append('--with-protobuf=' + spec['protobuf'].prefix) + + if '+debug' in spec: + args.append('--with-buildtype=debug') + else: + args.append('--with-buildtype=release') + + if '+1bitsgd' in spec: + args.append('--1bitsgd=yes') + args.append('--with-1bitsgd={0}/include' + .format(spec['cntk1bitsgd'].prefix)) + + if '+asgd' in spec: + args.append('--asgd=yes') + args.append('--with-multiverso={0}' + .format(spec['multiverso'].prefix)) + else: + args.append('--asgd=no') + + if '+opencv' in spec: + args.append('--with-opencv=' + spec['opencv'].prefix) + + if '+kaldi' in spec: + args.append('--with-kaldi=' + spec['kaldi'].prefix) + args.append('--with-openfst=' + spec['openfst'].prefix) + + if '+cuda' in spec: + args.append('--cuda=yes') + args.append('--with-cuda={0}'.format(spec['cuda'].prefix)) + args.append('--with-cub={0}' + .format(spec['cub'].prefix.include)) + args.append('--with-cudnn={0}' + .format(spec['cudnn'].prefix)) + args.append('--with-nccl={0}'.format(spec['nccl'].prefix)) + + configure(*args) + + make() + + install_tree('bin', prefix.bin) + install_tree('lib', prefix.lib) + install_tree('Examples', join_path(prefix, 'Examples')) + install_tree('Tutorials', join_path(prefix, 'Tutorials')) diff --git a/var/spack/repos/builtin/packages/cntk1bitsgd/package.py b/var/spack/repos/builtin/packages/cntk1bitsgd/package.py new file mode 100644 index 0000000000..7cc3c7030a --- /dev/null +++ b/var/spack/repos/builtin/packages/cntk1bitsgd/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +from os import listdir + + +class Cntk1bitsgd(Package): + """CNTK1bitSGD is the header-only + 1-bit stochastic gradient descent (1bit-SGD) library for + the Computational Network Toolkit (CNTK).""" + + homepage = "https://github.com/CNTK-components/CNTK1bitSGD" + + version('master', git='https://github.com/CNTK-components/CNTK1bitSGD.git') + version('c8b77d', git='https://github.com/CNTK-components/CNTK1bitSGD.git', + commit='c8b77d6e325a4786547b27624890276c1483aed1') + + def install(self, spec, prefix): + mkdirp(prefix.include) + for file in listdir('.'): + if file.endswith('.h'): + install(file, prefix.include) diff --git a/var/spack/repos/builtin/packages/multiverso/cmake-143187.patch b/var/spack/repos/builtin/packages/multiverso/cmake-143187.patch new file mode 100644 index 0000000000..5f2847f019 --- /dev/null +++ b/var/spack/repos/builtin/packages/multiverso/cmake-143187.patch @@ -0,0 +1,28 @@ +From 478bcfecadafb09aa73bacab35d932851b323142 Mon Sep 17 00:00:00 2001 +From: Jianwen WEI +Date: Mon, 10 Apr 2017 15:29:50 +0800 +Subject: [PATCH 1/1] Patch 1431875. + +--- + CMakeLists.txt | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 6a824f1..100b21c 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -30,9 +30,10 @@ ADD_SUBDIRECTORY(Test/unittests) + ADD_SUBDIRECTORY(Applications/WordEmbedding) + ADD_SUBDIRECTORY(Applications/LogisticRegression) + +-if(USE_INSTALL_MULTIVERSO) ++if(INSTALL_MULTIVERSO) + install (DIRECTORY ${PROJECT_SOURCE_DIR}/include/multiverso DESTINATION include) +-endif(USE_INSTALL_MULTIVERSO) ++ install (DIRECTORY ${PROJECT_SOURCE_DIR}/Test DESTINATION .) ++endif(INSTALL_MULTIVERSO) + + + # uninstall target +-- +2.11.1 diff --git a/var/spack/repos/builtin/packages/multiverso/package.py b/var/spack/repos/builtin/packages/multiverso/package.py new file mode 100644 index 0000000000..1537202905 --- /dev/null +++ b/var/spack/repos/builtin/packages/multiverso/package.py @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Multiverso(CMakePackage): + """Multiverso is a parameter server based framework for + training machine learning models on big data with numbers of machines.""" + + homepage = "https://github.com/Microsoft/Multiverso" + url = "https://github.com/Microsoft/Multiverso/archive/v0.2.tar.gz" + + version('master', git='https://github.com/Microsoft/Multiverso.git', + branch='master') + version('143187', git='https://github.com/Microsoft/Multiverso.git', + commit='143187575d1cfa410100037b8aea2e767e0af637') + version('0.2', '483ca7524fea14a311389e421f2bc098') + + depends_on('mpi') + depends_on('boost') + + patch('cmake-143187.patch', when='@143187') + + def cmake_args(self): + spec = self.spec + return ['-DBOOST_ROOT:PATH=%s' % spec['boost'].prefix] -- cgit v1.2.3-70-g09d2