From 16530f84be04b8466769b9b74fe920d67006b988 Mon Sep 17 00:00:00 2001 From: Tamara Dahlgren <35777542+tldahlgren@users.noreply.github.com> Date: Wed, 12 Jun 2019 17:07:47 -0700 Subject: Update remaining packages to use Stage.source_path (#11662) #11528 updated Stage to always store a Package's source in a fixed directory accessible via `Stage.source_path` This left behind a number of packages which were expecting to access the source code via `Stage.path`. This Updates those packages to use `Stage.source_path` instead. This also updates the name of the fixed directory: The original name of the fixed directory was "src", so if an expanded archive created a "src" directory, then users inspecting the directory structure could see paths like "src/src" (which wasn't wrong but could be confusing). Therefore this also updates the name of the fixed directory to "spack-src". --- lib/spack/spack/stage.py | 2 +- lib/spack/spack/test/stage.py | 4 +-- .../repos/builtin/packages/aspera-cli/package.py | 2 +- .../repos/builtin/packages/bcl2fastq2/package.py | 34 +++++++++++++++------- .../repos/builtin/packages/catalyst/package.py | 9 +++--- .../repos/builtin/packages/charmpp/package.py | 2 +- var/spack/repos/builtin/packages/clhep/package.py | 3 +- var/spack/repos/builtin/packages/cuda/package.py | 2 +- var/spack/repos/builtin/packages/lmod/package.py | 2 +- var/spack/repos/builtin/packages/nseg/package.py | 2 +- .../repos/builtin/packages/singularity/package.py | 14 +++++---- 11 files changed, 46 insertions(+), 30 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 9ecc148ad8..34d19860fa 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -28,7 +28,7 @@ import spack.util.pattern as pattern from spack.util.path import canonicalize_path from spack.util.crypto import prefix_bits, bit_length -_source_path_subdir = 'src' +_source_path_subdir = 'spack-src' _stage_prefix = 'spack-stage-' diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index 2862abc1d3..ba0249f6eb 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -48,7 +48,7 @@ _include_extra = 3 # to have the following structure: # # TMPDIR/ temp stage dir -# src/ well-known stage source directory +# spack-src/ well-known stage source directory # _readme_fn Optional test_readme (contains _readme_contents) # _hidden_fn Optional hidden file (contains _hidden_contents) # _archive_fn archive_url = file:///path/to/_archive_fn @@ -56,7 +56,7 @@ _include_extra = 3 # while exploding tarball directories are expected to be structured as follows: # # TMPDIR/ temp stage dir -# src/ well-known stage source directory +# spack-src/ well-known stage source directory # archive_name/ archive dir # _readme_fn test_readme (contains _readme_contents) # _extra_fn test_extra file (contains _extra_contents) diff --git a/var/spack/repos/builtin/packages/aspera-cli/package.py b/var/spack/repos/builtin/packages/aspera-cli/package.py index 51b0623aba..f20b9f49e0 100644 --- a/var/spack/repos/builtin/packages/aspera-cli/package.py +++ b/var/spack/repos/builtin/packages/aspera-cli/package.py @@ -21,7 +21,7 @@ class AsperaCli(Package): run_env.prepend_path('PATH', self.prefix.cli.bin) def install(self, spec, prefix): - runfile = glob(join_path(self.stage.path, 'aspera-cli*.sh'))[0] + runfile = glob(join_path(self.stage.source_path, 'aspera-cli*.sh'))[0] # Update destination path filter_file('INSTALL_DIR=~/.aspera', 'INSTALL_DIR=%s' % prefix, diff --git a/var/spack/repos/builtin/packages/bcl2fastq2/package.py b/var/spack/repos/builtin/packages/bcl2fastq2/package.py index 22fab13126..8250b23112 100644 --- a/var/spack/repos/builtin/packages/bcl2fastq2/package.py +++ b/var/spack/repos/builtin/packages/bcl2fastq2/package.py @@ -5,7 +5,6 @@ from spack import * import os -import shutil import glob import llnl.util.tty as tty @@ -70,19 +69,34 @@ class Bcl2fastq2(Package): def unpack_it(self, f): def wrap(): f() # call the original expand_archive() + + # The tarfile should now reside in the well-known source + # directory (i.e., self.stage.source_path). with working_dir(self.stage.path): - if os.path.isdir('bcl2fastq'): - tty.msg("The tarball has already been unpacked") - else: + source_subdir = os.path.relpath(self.stage.source_path, + self.stage.path) + files = glob.glob(os.path.join(source_subdir, + 'bcl2fastq2*.tar.gz')) + if len(files) == 1: + # Rename the tarball so it resides in self.stage.path + # alongside the original zip file before unpacking it. + tarball = files[0] + basename = os.path.basename(tarball) + os.rename(tarball, basename) tty.msg("Unpacking bcl2fastq2 tarball") - tarball = glob.glob(join_path('spack-expanded-archive', - 'bcl2fastq2*.tar.gz'))[0] - copy(tarball, '.') - shutil.rmtree('spack-expanded-archive') tar = which('tar') - tarball = os.path.basename(tarball) - tar('-xf', tarball) + tar('-xf', basename) + + # Rename the unpacked directory to the well-known + # source path self.stage.source_path. + os.rename('bcl2fastq', source_subdir) tty.msg("Finished unpacking bcl2fastq2 tarball") + + elif self.stage.expanded: + # The unpacked files already reside in the "well known" + # source directory (i.e., self.stage.source_path). + tty.msg("The tarball has already been unpacked.") + return wrap def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/catalyst/package.py b/var/spack/repos/builtin/packages/catalyst/package.py index e64c97a749..03f4609b60 100644 --- a/var/spack/repos/builtin/packages/catalyst/package.py +++ b/var/spack/repos/builtin/packages/catalyst/package.py @@ -119,7 +119,7 @@ class Catalyst(CMakePackage): super(Catalyst, self).do_stage(mirror_only) # extract the catalyst part - paraview_dir = os.path.join(self.stage.path, + paraview_dir = os.path.join(self.stage.source_path, 'ParaView-v' + str(self.version)) catalyst_script = os.path.join(paraview_dir, 'Catalyst', 'catalyze.py') editions_dir = os.path.join(paraview_dir, 'Catalyst', 'Editions') @@ -135,10 +135,10 @@ class Catalyst(CMakePackage): if not os.path.isdir(catalyst_source_dir): os.mkdir(catalyst_source_dir) subprocess.check_call(command) - tty.msg("Generated catalyst source in %s" % self.stage.path) + tty.msg("Generated catalyst source in %s" % self.stage.source_path) else: tty.msg("Already generated %s in %s" % (self.name, - self.stage.path)) + self.stage.source_path)) def setup_environment(self, spack_env, run_env): # paraview 5.5 and later @@ -174,7 +174,8 @@ class Catalyst(CMakePackage): :return: directory containing CMakeLists.txt """ - return os.path.join(self.stage.path, 'Catalyst-v' + str(self.version)) + return os.path.join(self.stage.source_path, + 'Catalyst-v' + str(self.version)) @property def build_directory(self): diff --git a/var/spack/repos/builtin/packages/charmpp/package.py b/var/spack/repos/builtin/packages/charmpp/package.py index a81a7c7349..baeed83b7b 100644 --- a/var/spack/repos/builtin/packages/charmpp/package.py +++ b/var/spack/repos/builtin/packages/charmpp/package.py @@ -241,5 +241,5 @@ class Charmpp(Package): @run_after('install') @on_package_attributes(run_tests=True) def check_build(self): - make('-C', join_path(self.stage.path, 'charm/tests'), + make('-C', join_path(self.stage.source_path, 'charm/tests'), 'test', parallel=False) diff --git a/var/spack/repos/builtin/packages/clhep/package.py b/var/spack/repos/builtin/packages/clhep/package.py index ebfe24ec6f..9e73e35bef 100644 --- a/var/spack/repos/builtin/packages/clhep/package.py +++ b/var/spack/repos/builtin/packages/clhep/package.py @@ -51,8 +51,7 @@ class Clhep(CMakePackage): def patch(self): filter_file('SET CMP0042 OLD', 'SET CMP0042 NEW', - '%s/%s/CLHEP/CMakeLists.txt' - % (self.stage.path, self.spec.version)) + '%s/CLHEP/CMakeLists.txt' % self.stage.source_path) def cmake_args(self): cmake_args = ['-DCLHEP_BUILD_CXXSTD=-std=c++{0}'.format( diff --git a/var/spack/repos/builtin/packages/cuda/package.py b/var/spack/repos/builtin/packages/cuda/package.py index 2ad654bbc0..5409a38d52 100644 --- a/var/spack/repos/builtin/packages/cuda/package.py +++ b/var/spack/repos/builtin/packages/cuda/package.py @@ -41,7 +41,7 @@ class Cuda(Package): run_env.set('CUDA_HOME', self.prefix) def install(self, spec, prefix): - runfile = glob(join_path(self.stage.path, 'cuda*_linux*'))[0] + runfile = glob(join_path(self.stage.source_path, 'cuda*_linux*'))[0] chmod = which('chmod') chmod('+x', runfile) runfile = which(runfile) diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index d80de5cfd4..ad4ccf9b46 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -51,7 +51,7 @@ class Lmod(AutotoolsPackage): def setup_environment(self, spack_env, run_env): stage_lua_path = join_path( - self.stage.path, 'Lmod-{version}', 'src', '?.lua') + self.stage.source_path, 'src', '?.lua') spack_env.append_path('LUA_PATH', stage_lua_path.format( version=self.version), separator=';') diff --git a/var/spack/repos/builtin/packages/nseg/package.py b/var/spack/repos/builtin/packages/nseg/package.py index ed5f903516..ef552f2b36 100644 --- a/var/spack/repos/builtin/packages/nseg/package.py +++ b/var/spack/repos/builtin/packages/nseg/package.py @@ -71,7 +71,7 @@ class Nseg(MakefilePackage): for key in self.resources: for res in self.resources[key]: res_name = res.name - res_path = join_path(res.fetcher.stage.path, res.name) + res_path = join_path(res.fetcher.stage.source_path, res.name) copy(res_path, join_path(self.build_directory, res_name)) def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/singularity/package.py b/var/spack/repos/builtin/packages/singularity/package.py index 4287773a77..6b939eb161 100644 --- a/var/spack/repos/builtin/packages/singularity/package.py +++ b/var/spack/repos/builtin/packages/singularity/package.py @@ -38,7 +38,7 @@ class Singularity(MakefilePackage): # tree into the proper subdir in our overridden do_stage below. @property def gopath(self): - return join_path(self.stage.path) + return self.stage.path @property def sylabs_gopath_dir(self): @@ -52,13 +52,15 @@ class Singularity(MakefilePackage): # its home within GOPATH. def do_stage(self, mirror_only=False): super(Singularity, self).do_stage(mirror_only) - source_path = self.stage.source_path if not os.path.exists(self.singularity_gopath_dir): + # Move the expanded source to its destination tty.debug("Moving {0} to {1}".format( - source_path, self.singularity_gopath_dir)) - mkdirp(self.sylabs_gopath_dir) - shutil.move(source_path, - self.singularity_gopath_dir) + self.stage.source_path, self.singularity_gopath_dir)) + shutil.move(self.stage.source_path, self.singularity_gopath_dir) + + # The build process still needs access to the source path, + # so create a symlink. + force_symlink(self.singularity_gopath_dir, self.stage.source_path) # MakefilePackage's stages use this via working_dir() @property -- cgit v1.2.3-60-g2f50