summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPatrick Gartung <gartung@fnal.gov>2021-10-13 10:38:29 -0500
committerGitHub <noreply@github.com>2021-10-13 17:38:29 +0200
commit047c95aa8d2ff665d4788cb02b965ab3fa9bfc90 (patch)
tree068f94cf3ee68aa5ffa1a8968afa1ba84ba8a3a1
parente6b76578d29e18855b58be1eade73c43d3a295e0 (diff)
downloadspack-047c95aa8d2ff665d4788cb02b965ab3fa9bfc90.tar.gz
spack-047c95aa8d2ff665d4788cb02b965ab3fa9bfc90.tar.bz2
spack-047c95aa8d2ff665d4788cb02b965ab3fa9bfc90.tar.xz
spack-047c95aa8d2ff665d4788cb02b965ab3fa9bfc90.zip
buildcache: do one less tar file extraction
The buildcache is now extracted in a temporary folder within the current store, moved to its final place and relocated. "spack clean -s" has been extended to also clean the temporary extraction directory. Add hardlinks with absolute paths for libraries in the corge, garply and quux packages to detect incorrect handling of hardlinks in tests.
-rw-r--r--lib/spack/spack/binary_distribution.py54
-rw-r--r--lib/spack/spack/cmd/clean.py6
-rw-r--r--var/spack/repos/builtin.mock/packages/corge/package.py12
-rw-r--r--var/spack/repos/builtin.mock/packages/garply/package.py4
-rw-r--r--var/spack/repos/builtin.mock/packages/quux/package.py4
5 files changed, 41 insertions, 39 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index c976dc80fb..165ba219b3 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -4,11 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import codecs
-import glob
import hashlib
import json
import os
-import re
import shutil
import sys
import tarfile
@@ -1428,42 +1426,30 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
buildinfo = spec_dict.get('buildinfo', {})
old_relative_prefix = buildinfo.get('relative_prefix', new_relative_prefix)
rel = buildinfo.get('relative_rpaths')
- # if the original relative prefix and new relative prefix differ the
- # directory layout has changed and the buildcache cannot be installed
- # if it was created with relative rpaths
info = 'old relative prefix %s\nnew relative prefix %s\nrelative rpaths %s'
tty.debug(info %
(old_relative_prefix, new_relative_prefix, rel))
-# if (old_relative_prefix != new_relative_prefix and (rel)):
-# shutil.rmtree(tmpdir)
-# msg = "Package tarball was created from an install "
-# msg += "prefix with a different directory layout. "
-# msg += "It cannot be relocated because it "
-# msg += "uses relative rpaths."
-# raise NewLayoutException(msg)
-
- # extract the tarball in a temp directory
- with closing(tarfile.open(tarfile_path, 'r')) as tar:
- tar.extractall(path=tmpdir)
- # get the parent directory of the file .spack/binary_distribution
- # this should the directory unpacked from the tarball whose
- # name is unknown because the prefix naming is unknown
- bindist_file = glob.glob('%s/*/.spack/binary_distribution' % tmpdir)[0]
- workdir = re.sub('/.spack/binary_distribution$', '', bindist_file)
- tty.debug('workdir %s' % workdir)
- # install_tree copies hardlinks
- # create a temporary tarfile from prefix and exract it to workdir
- # tarfile preserves hardlinks
- temp_tarfile_name = tarball_name(spec, '.tar')
- temp_tarfile_path = os.path.join(tmpdir, temp_tarfile_name)
- with closing(tarfile.open(temp_tarfile_path, 'w')) as tar:
- tar.add(name='%s' % workdir,
- arcname='.')
- with closing(tarfile.open(temp_tarfile_path, 'r')) as tar:
- tar.extractall(spec.prefix)
- os.remove(temp_tarfile_path)
- # cleanup
+ # Extract the tarball into the store root, presumably on the same filesystem.
+ # The directory created is the base directory name of the old prefix.
+ # Moving the old prefix name to the new prefix location should preserve
+ # hard links and symbolic links.
+ extract_tmp = os.path.join(spack.store.layout.root, '.tmp')
+ mkdirp(extract_tmp)
+ extracted_dir = os.path.join(extract_tmp,
+ old_relative_prefix.split(os.path.sep)[-1])
+
+ with closing(tarfile.open(tarfile_path, 'r')) as tar:
+ try:
+ tar.extractall(path=extract_tmp)
+ except Exception as e:
+ shutil.rmtree(extracted_dir)
+ raise e
+ try:
+ shutil.move(extracted_dir, spec.prefix)
+ except Exception as e:
+ shutil.rmtree(extracted_dir)
+ raise e
os.remove(tarfile_path)
os.remove(specfile_path)
diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py
index c4012f5032..de4abd2fc2 100644
--- a/lib/spack/spack/cmd/clean.py
+++ b/lib/spack/spack/cmd/clean.py
@@ -76,7 +76,11 @@ def clean(parser, args):
if args.stage:
tty.msg('Removing all temporary build stages')
spack.stage.purge()
-
+ # Temp directory where buildcaches are extracted
+ extract_tmp = os.path.join(spack.store.layout.root, '.tmp')
+ if os.path.exists(extract_tmp):
+ tty.debug('Removing {0}'.format(extract_tmp))
+ shutil.rmtree(extract_tmp)
if args.downloads:
tty.msg('Removing cached downloads')
spack.caches.fetch_cache.destroy()
diff --git a/var/spack/repos/builtin.mock/packages/corge/package.py b/var/spack/repos/builtin.mock/packages/corge/package.py
index 2fd2b23038..592a50bce7 100644
--- a/var/spack/repos/builtin.mock/packages/corge/package.py
+++ b/var/spack/repos/builtin.mock/packages/corge/package.py
@@ -142,10 +142,12 @@ main(int argc, char* argv[])
'-Wl,-rpath,%s' % spec['quux'].prefix.lib64,
'-Wl,-rpath,%s' % spec['garply'].prefix.lib64,
'libcorge.dylib',
- '%s/libquux.dylib' % spec['quux'].prefix.lib64,
- '%s/libgarply.dylib' % spec['garply'].prefix.lib64)
+ '%s/libquux.dylib.3.0' % spec['quux'].prefix.lib64,
+ '%s/libgarply.dylib.3.0' % spec['garply'].prefix.lib64)
mkdirp(prefix.lib64)
copy('libcorge.dylib', '%s/libcorge.dylib' % prefix.lib64)
+ os.link('%s/libcorge.dylib' % prefix.lib64,
+ '%s/libcorge.dylib.3.0' % prefix.lib64)
else:
gpp('-fPIC', '-O2', '-g', '-DNDEBUG', '-shared',
'-Wl,-soname,libcorge.so', '-o', 'libcorge.so', 'corge.cc.o',
@@ -159,10 +161,12 @@ main(int argc, char* argv[])
'-Wl,-rpath,%s' % spec['quux'].prefix.lib64,
'-Wl,-rpath,%s' % spec['garply'].prefix.lib64,
'libcorge.so',
- '%s/libquux.so' % spec['quux'].prefix.lib64,
- '%s/libgarply.so' % spec['garply'].prefix.lib64)
+ '%s/libquux.so.3.0' % spec['quux'].prefix.lib64,
+ '%s/libgarply.so.3.0' % spec['garply'].prefix.lib64)
mkdirp(prefix.lib64)
copy('libcorge.so', '%s/libcorge.so' % prefix.lib64)
+ os.link('%s/libcorge.so' % prefix.lib64,
+ '%s/libcorge.so.3.0' % prefix.lib64)
copy('corgegator', '%s/corgegator' % prefix.lib64)
copy('%s/corge/corge.h' % self.stage.source_path,
'%s/corge/corge.h' % prefix.include)
diff --git a/var/spack/repos/builtin.mock/packages/garply/package.py b/var/spack/repos/builtin.mock/packages/garply/package.py
index 6f028eec31..6b98e1c14a 100644
--- a/var/spack/repos/builtin.mock/packages/garply/package.py
+++ b/var/spack/repos/builtin.mock/packages/garply/package.py
@@ -109,6 +109,8 @@ const int garply_version_minor = %s;
'libgarply.dylib')
mkdirp(prefix.lib64)
copy('libgarply.dylib', '%s/libgarply.dylib' % prefix.lib64)
+ os.link('%s/libgarply.dylib' % prefix.lib64,
+ '%s/libgarply.dylib.3.0' % prefix.lib64)
else:
gpp('-fPIC', '-O2', '-g', '-DNDEBUG', '-shared',
'-Wl,-soname,libgarply.so',
@@ -119,6 +121,8 @@ const int garply_version_minor = %s;
'libgarply.so')
mkdirp(prefix.lib64)
copy('libgarply.so', '%s/libgarply.so' % prefix.lib64)
+ os.link('%s/libgarply.so' % prefix.lib64,
+ '%s/libgarply.so.3.0' % prefix.lib64)
copy('garplinator', '%s/garplinator' % prefix.lib64)
copy('%s/garply/garply.h' % self.stage.source_path,
'%s/garply/garply.h' % prefix.include)
diff --git a/var/spack/repos/builtin.mock/packages/quux/package.py b/var/spack/repos/builtin.mock/packages/quux/package.py
index 676b1c1f0b..0430841c8e 100644
--- a/var/spack/repos/builtin.mock/packages/quux/package.py
+++ b/var/spack/repos/builtin.mock/packages/quux/package.py
@@ -128,6 +128,8 @@ const int quux_version_minor = %s;
'%s/libgarply.dylib' % spec['garply'].prefix.lib64)
mkdirp(prefix.lib64)
copy('libquux.dylib', '%s/libquux.dylib' % prefix.lib64)
+ os.link('%s/libquux.dylib' % prefix.lib64,
+ '%s/libquux.dylib.3.0' % prefix.lib64)
else:
gpp('-fPIC', '-O2', '-g', '-DNDEBUG', '-shared',
'-Wl,-soname,libquux.so', '-o', 'libquux.so', 'quux.cc.o',
@@ -142,6 +144,8 @@ const int quux_version_minor = %s;
'%s/libgarply.so' % spec['garply'].prefix.lib64)
mkdirp(prefix.lib64)
copy('libquux.so', '%s/libquux.so' % prefix.lib64)
+ os.link('%s/libquux.so' % prefix.lib64,
+ '%s/libquux.so.3.0' % prefix.lib64)
copy('quuxifier', '%s/quuxifier' % prefix.lib64)
copy('%s/quux/quux.h' % self.stage.source_path,
'%s/quux/quux.h' % prefix.include)