summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPatrick Gartung <gartung@fnal.gov>2020-02-14 16:10:28 -0600
committerGitHub <noreply@github.com>2020-02-14 16:10:28 -0600
commitbaafa7ec84050662660d6bff2a53ff9ff1744f8f (patch)
tree3973c367e6f42fa32dbbc8658655837e01fea2c9
parent3c28e72d2f474a3a739725e906c7cea4b8be8f4d (diff)
downloadspack-baafa7ec84050662660d6bff2a53ff9ff1744f8f.tar.gz
spack-baafa7ec84050662660d6bff2a53ff9ff1744f8f.tar.bz2
spack-baafa7ec84050662660d6bff2a53ff9ff1744f8f.tar.xz
spack-baafa7ec84050662660d6bff2a53ff9ff1744f8f.zip
Buildcache creation/extraction use temp tarfile to preserve hardlinks during copy to/from prefix. (#15003)
* Buildcache creation change the way prefix is copied to workdir. * install_tree copies hardlinked files * tarfile creates hardlinked files on extraction. * create a temporary tarfile from prefix and extract it to workdir * Use temp tarfile to move workdir to prefix to preserve hardlinks instead of copying
-rw-r--r--lib/spack/spack/binary_distribution.py38
-rw-r--r--lib/spack/spack/test/packaging.py18
2 files changed, 30 insertions, 26 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index 515a6166d2..49f8416d6f 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -18,7 +18,7 @@ import json
from six.moves.urllib.error import URLError
import llnl.util.tty as tty
-from llnl.util.filesystem import mkdirp, install_tree
+from llnl.util.filesystem import mkdirp
import spack.cmd
import spack.config as config
@@ -308,7 +308,7 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
tmpdir = tempfile.mkdtemp()
cache_prefix = build_cache_prefix(tmpdir)
- tarfile_name = tarball_name(spec, '.tar.gz')
+ tarfile_name = tarball_name(spec, '.tar.bz2')
tarfile_dir = os.path.join(cache_prefix, tarball_directory_name(spec))
tarfile_path = os.path.join(tarfile_dir, tarfile_name)
spackfile_path = os.path.join(
@@ -342,8 +342,18 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
raise NoOverwriteException(url_util.format(remote_specfile_path))
# make a copy of the install directory to work with
- workdir = os.path.join(tempfile.mkdtemp(), os.path.basename(spec.prefix))
- install_tree(spec.prefix, workdir, symlinks=True)
+ workdir = os.path.join(tmpdir, os.path.basename(spec.prefix))
+ # install_tree copies hardlinks
+ # create a temporary tarfile from prefix and exract it to workdir
+ # tarfile preserves hardlinks
+ temp_tarfile_name = tarball_name(spec, '.tar')
+ temp_tarfile_path = os.path.join(tarfile_dir, temp_tarfile_name)
+ with closing(tarfile.open(temp_tarfile_path, 'w')) as tar:
+ tar.add(name='%s' % spec.prefix,
+ arcname='.')
+ with closing(tarfile.open(temp_tarfile_path, 'r')) as tar:
+ tar.extractall(workdir)
+ os.remove(temp_tarfile_path)
# create info for later relocation and create tar
write_buildinfo_file(spec.prefix, workdir, rel=rel)
@@ -368,7 +378,7 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
tty.die(e)
# create compressed tarball of the install prefix
- with closing(tarfile.open(tarfile_path, 'w:gz')) as tar:
+ with closing(tarfile.open(tarfile_path, 'w:bz2')) as tar:
tar.add(name='%s' % workdir,
arcname='%s' % os.path.basename(spec.prefix))
# remove copy of install directory
@@ -407,8 +417,8 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
sign_tarball(key, force, specfile_path)
# put tarball, spec and signature files in .spack archive
with closing(tarfile.open(spackfile_path, 'w')) as tar:
- tar.add(name='%s' % tarfile_path, arcname='%s' % tarfile_name)
- tar.add(name='%s' % specfile_path, arcname='%s' % specfile_name)
+ tar.add(name=tarfile_path, arcname='%s' % tarfile_name)
+ tar.add(name=specfile_path, arcname='%s' % specfile_name)
if not unsigned:
tar.add(name='%s.asc' % specfile_path,
arcname='%s.asc' % specfile_name)
@@ -579,7 +589,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
stagepath = os.path.dirname(filename)
spackfile_name = tarball_name(spec, '.spack')
spackfile_path = os.path.join(stagepath, spackfile_name)
- tarfile_name = tarball_name(spec, '.tar.gz')
+ tarfile_name = tarball_name(spec, '.tar.bz2')
tarfile_path = os.path.join(tmpdir, tarfile_name)
specfile_name = tarball_name(spec, '.spec.yaml')
specfile_path = os.path.join(tmpdir, specfile_name)
@@ -638,7 +648,17 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
# so the pathname should be the same now that the directory layout
# is confirmed
workdir = os.path.join(tmpdir, os.path.basename(spec.prefix))
- install_tree(workdir, spec.prefix, symlinks=True)
+ # install_tree copies hardlinks
+ # create a temporary tarfile from prefix and exract it to workdir
+ # tarfile preserves hardlinks
+ temp_tarfile_name = tarball_name(spec, '.tar')
+ temp_tarfile_path = os.path.join(tmpdir, temp_tarfile_name)
+ with closing(tarfile.open(temp_tarfile_path, 'w')) as tar:
+ tar.add(name='%s' % workdir,
+ arcname='.')
+ with closing(tarfile.open(temp_tarfile_path, 'r')) as tar:
+ tar.extractall(spec.prefix)
+ os.remove(temp_tarfile_path)
# cleanup
os.remove(tarfile_path)
diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py
index edad8e29fa..8f11b8ec99 100644
--- a/lib/spack/spack/test/packaging.py
+++ b/lib/spack/spack/test/packaging.py
@@ -107,11 +107,6 @@ echo $PATH"""
buildcache.buildcache(parser, args)
files = os.listdir(spec.prefix)
- assert 'link_to_dummy.txt' in files
- assert 'dummy.txt' in files
- assert os.path.realpath(
- os.path.join(spec.prefix, 'link_to_dummy.txt')
- ) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt'))
# create build cache with relative path and signing
args = parser.parse_args(
@@ -129,13 +124,6 @@ echo $PATH"""
args = parser.parse_args(['install', '-f', str(pkghash)])
buildcache.buildcache(parser, args)
- files = os.listdir(spec.prefix)
- assert 'link_to_dummy.txt' in files
- assert 'dummy.txt' in files
- assert os.path.realpath(
- os.path.join(spec.prefix, 'link_to_dummy.txt')
- ) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt'))
-
else:
# create build cache without signing
args = parser.parse_args(
@@ -152,10 +140,6 @@ echo $PATH"""
files = os.listdir(spec.prefix)
assert 'link_to_dummy.txt' in files
assert 'dummy.txt' in files
- assert os.path.realpath(
- os.path.join(spec.prefix, 'link_to_dummy.txt')
- ) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt'))
-
# test overwrite install without verification
args = parser.parse_args(['install', '-f', '-u', str(pkghash)])
buildcache.buildcache(parser, args)
@@ -242,7 +226,7 @@ def test_relocate_links(tmpdir):
old_src = os.path.join(old_dir, filename)
os.symlink(old_src, filename)
filenames = [filename]
- new_dir = '/opt/rh/devtoolset/'
+ new_dir = '/opt/rh/devtoolset'
relocate_links(filenames, old_dir, new_dir)
assert os.path.realpath(filename) == os.path.join(new_dir, filename)