From ec9959b152ca0b5d11621c1d1e5acc4bba2a8db1 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Fri, 22 Jul 2016 17:05:01 -0500 Subject: R extension dependencies with compiler wrapper This commit introduces a mechanism to insure that R package dependencies are built with the Spack compiler wrapper. A copy of Makeconf is made before `filter_compilers` is called. This is then pointed to by the R_MAKEVARS_SITE environment variable set up in `setup_dependent_environment`. With this the normal compilers are used outside of spack and the spack wrapper compilers are used inside of spack. This commit also standardizes on the `join_path` call. It also sets the commented build command to reflect what is actually used with the newer string formatting. --- var/spack/repos/builtin/packages/R/package.py | 40 +++++++++++++++++---------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index ad06c2ca48..554adc7793 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -22,10 +22,9 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os - from spack import * from spack.util.environment import * +import shutil class R(Package): @@ -74,6 +73,10 @@ class R(Package): depends_on('pcre') depends_on('jdk') + @property + def etcdir(self): + return join_path(prefix, 'rlib', 'R', 'etc') + def install(self, spec, prefix): rlibdir = join_path(prefix, 'rlib') configure_args = ['--prefix=%s' % prefix, @@ -88,6 +91,12 @@ class R(Package): make() make('install') + # Make a copy of Makeconf because it will be needed to properly build R + # dependencies in Spack. + src_makeconf = join_path(self.etcdir, 'Makeconf') + dst_makeconf = join_path(self.etcdir, 'Makeconf.spack') + shutil.copy(src_makeconf, dst_makeconf) + self.filter_compilers(spec, prefix) def filter_compilers(self, spec, prefix): @@ -98,18 +107,16 @@ class R(Package): cc and c++. We want them to be bound to whatever compiler they were built with.""" - etcdir = join_path(prefix, 'rlib', 'R', 'etc') - kwargs = {'ignore_absent': True, 'backup': False, 'string': True} - filter_file(env['CC'], self.compiler.cc, - join_path(etcdir, 'Makeconf'), **kwargs) + filter_file(env['CC'], self.compiler.cc, + join_path(self.etcdir, 'Makeconf'), **kwargs) filter_file(env['CXX'], self.compiler.cxx, - join_path(etcdir, 'Makeconf'), **kwargs) + join_path(self.etcdir, 'Makeconf'), **kwargs) filter_file(env['F77'], self.compiler.f77, - join_path(etcdir, 'Makeconf'), **kwargs) + join_path(self.etcdir, 'Makeconf'), **kwargs) filter_file(env['FC'], self.compiler.fc, - join_path(etcdir, 'Makeconf'), **kwargs) + join_path(self.etcdir, 'Makeconf'), **kwargs) # ======================================================================== # Set up environment to make install easy for R extensions. @@ -117,7 +124,7 @@ class R(Package): @property def r_lib_dir(self): - return os.path.join('rlib', 'R', 'library') + return join_path('rlib', 'R', 'library') def setup_dependent_environment(self, spack_env, run_env, extension_spec): # Set R_LIBS to include the library dir for the @@ -125,15 +132,17 @@ class R(Package): r_libs_path = [] for d in extension_spec.traverse(deptype=nolink, deptype_query='run'): if d.package.extends(self.spec): - r_libs_path.append(os.path.join(d.prefix, self.r_lib_dir)) + r_libs_path.append(join_path(d.prefix, self.r_lib_dir)) r_libs_path = ':'.join(r_libs_path) spack_env.set('R_LIBS', r_libs_path) + spack_env.set('R_MAKEVARS_SITE', + join_path(self.etcdir, 'Makeconf.spack')) # For run time environment set only the path for extension_spec and # prepend it to R_LIBS if extension_spec.package.extends(self.spec): - run_env.prepend_path('R_LIBS', os.path.join( + run_env.prepend_path('R_LIBS', join_path( extension_spec.prefix, self.r_lib_dir)) def setup_environment(self, spack_env, run_env): @@ -147,13 +156,14 @@ class R(Package): def setup_dependent_package(self, module, ext_spec): """Called before R modules' install() methods. In most cases, extensions will only need to have one line: - R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % - self.stage.source_path)""" + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path)""" + # R extension builds can have a global R executable function module.R = Executable(join_path(self.spec.prefix.bin, 'R')) # Add variable for library directry - module.r_lib_dir = os.path.join(ext_spec.prefix, self.r_lib_dir) + module.r_lib_dir = join_path(ext_spec.prefix, self.r_lib_dir) # Make the site packages directory for extensions, if it does not exist # already. -- cgit v1.2.3-70-g09d2 From 8dc26bbcd957a7a00c4a370c434136e59b52aa87 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 30 Jul 2016 15:07:18 -0500 Subject: Have R extensions build in parallel. Set `MAKEFLAGS` so R extensions can be built in parallel if that is set in spack. --- var/spack/repos/builtin/packages/R/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index 554adc7793..e880a3aa66 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -139,6 +139,10 @@ class R(Package): spack_env.set('R_MAKEVARS_SITE', join_path(self.etcdir, 'Makeconf.spack')) + # Use the number of make_jobs set in spack. The make program will + # determine how many jobs can actually be started. + spack_env.set('MAKEFLAGS', '-j{0}'.format(make_jobs)) + # For run time environment set only the path for extension_spec and # prepend it to R_LIBS if extension_spec.package.extends(self.spec): -- cgit v1.2.3-70-g09d2