From 25893f154ff185e9df54c8a6e11c0fcbc02c2f20 Mon Sep 17 00:00:00 2001
From: Patrick Gartung <gartung@fnal.gov>
Date: Sat, 21 Mar 2020 13:26:37 -0500
Subject: Buildcache: attempt to build and install buildcaches in test
environment (#15577)
* Put bindist.py on it's own branch
* Working on SL7 with python 3.6
* Flake8
* Put back clearing of compiler cache
---
lib/spack/spack/test/bindist.py | 480 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 480 insertions(+)
create mode 100644 lib/spack/spack/test/bindist.py
diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py
new file mode 100644
index 0000000000..8ca7609942
--- /dev/null
+++ b/lib/spack/spack/test/bindist.py
@@ -0,0 +1,480 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""
+This test checks creating and install buildcaches
+"""
+import os
+import py
+import pytest
+import argparse
+import platform
+import spack.repo
+import spack.store
+import spack.binary_distribution as bindist
+import spack.cmd.buildcache as buildcache
+import spack.cmd.compiler as compiler
+import spack.cmd.repo as repocmd
+import spack.cmd.install as install
+import spack.cmd.uninstall as uninstall
+import spack.cmd.mirror as mirror
+from spack.spec import Spec
+from spack.directory_layout import YamlDirectoryLayout
+
+
+def_install_path_scheme = '${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}' # noqa: E501
+ndef_install_path_scheme = '${PACKAGE}/${VERSION}/${ARCHITECTURE}-${COMPILERNAME}-${COMPILERVER}-${HASH}' # noqa: E501
+
+mirror_path_def = None
+mirror_path_rel = None
+
+
+@pytest.fixture(scope='function')
+def cache_directory(tmpdir):
+ old_cache_path = spack.caches.fetch_cache
+ tmpdir.ensure('fetch_cache', dir=True)
+ fsc = spack.fetch_strategy.FsCache(str(tmpdir.join('fetch_cache')))
+ spack.config.caches = fsc
+ yield spack.config.caches
+ tmpdir.join('fetch_cache').remove()
+ spack.config.caches = old_cache_path
+
+
+@pytest.fixture(scope='session')
+def session_mirror_def(tmpdir_factory):
+ dir = tmpdir_factory.mktemp('mirror')
+ global mirror_path_rel
+ mirror_path_rel = dir
+ dir.ensure('build_cache', dir=True)
+ yield dir
+ dir.join('build_cache').remove()
+
+
+@pytest.fixture(scope='function')
+def mirror_directory_def(session_mirror_def):
+ yield str(session_mirror_def)
+
+
+@pytest.fixture(scope='session')
+def session_mirror_rel(tmpdir_factory):
+ dir = tmpdir_factory.mktemp('mirror')
+ global mirror_path_rel
+ mirror_path_rel = dir
+ dir.ensure('build_cache', dir=True)
+ yield dir
+ dir.join('build_cache').remove()
+
+
+@pytest.fixture(scope='function')
+def mirror_directory_rel(session_mirror_rel):
+ yield(session_mirror_rel)
+
+
+@pytest.fixture(scope='session')
+def config_directory(tmpdir_factory):
+ tmpdir = tmpdir_factory.mktemp('test_configs')
+ # restore some sane defaults for packages and config
+ config_path = py.path.local(spack.paths.etc_path)
+ modules_yaml = config_path.join('spack', 'defaults', 'modules.yaml')
+ os_modules_yaml = config_path.join('spack', 'defaults', '%s' %
+ platform.system().lower(),
+ 'modules.yaml')
+ packages_yaml = config_path.join('spack', 'defaults', 'packages.yaml')
+ config_yaml = config_path.join('spack', 'defaults', 'config.yaml')
+ repos_yaml = config_path.join('spack', 'defaults', 'repos.yaml')
+ tmpdir.ensure('site', dir=True)
+ tmpdir.ensure('user', dir=True)
+ tmpdir.ensure('site/%s' % platform.system().lower(), dir=True)
+ modules_yaml.copy(tmpdir.join('site', 'modules.yaml'))
+ os_modules_yaml.copy(tmpdir.join('site/%s' % platform.system().lower(),
+ 'modules.yaml'))
+ packages_yaml.copy(tmpdir.join('site', 'packages.yaml'))
+ config_yaml.copy(tmpdir.join('site', 'config.yaml'))
+ repos_yaml.copy(tmpdir.join('site', 'repos.yaml'))
+ yield tmpdir
+ tmpdir.remove()
+
+
+@pytest.fixture(scope='function')
+def default_config(tmpdir_factory, config_directory, monkeypatch):
+
+ mutable_dir = tmpdir_factory.mktemp('mutable_config').join('tmp')
+ config_directory.copy(mutable_dir)
+
+ cfg = spack.config.Configuration(
+ *[spack.config.ConfigScope(name, str(mutable_dir))
+ for name in ['site/%s' % platform.system().lower(),
+ 'site', 'user']])
+
+ monkeypatch.setattr(spack.config, 'config', cfg)
+
+ # This is essential, otherwise the cache will create weird side effects
+ # that will compromise subsequent tests if compilers.yaml is modified
+ monkeypatch.setattr(spack.compilers, '_cache_config_file', [])
+ njobs = spack.config.get('config:build_jobs')
+ if not njobs:
+ spack.config.set('config:build_jobs', 4, scope='user')
+ extensions = spack.config.get('config:template_dirs')
+ if not extensions:
+ spack.config.set('config:template_dirs',
+ [os.path.join(spack.paths.share_path, 'templates')],
+ scope='user')
+
+ mutable_dir.ensure('build_stage', dir=True)
+ build_stage = spack.config.get('config:build_stage')
+ if not build_stage:
+ spack.config.set('config:build_stage',
+ [str(mutable_dir.join('build_stage'))], scope='user')
+ timeout = spack.config.get('config:connect_timeout')
+ if not timeout:
+ spack.config.set('config:connect_timeout', 10, scope='user')
+ print(spack.config.config)
+ yield spack.config.config
+ mutable_dir.remove()
+
+
+@pytest.fixture(scope='function')
+def install_dir_default_layout(tmpdir):
+ """Hooks a fake install directory with a default layout"""
+ real_store = spack.store.store
+ real_layout = spack.store.layout
+ spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
+ spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')),
+ path_scheme=def_install_path_scheme) # noqa: E501
+ yield spack.store
+ tmpdir.join('opt').remove()
+ spack.store.store = real_store
+ spack.store.layout = real_layout
+
+
+@pytest.fixture(scope='function')
+def install_dir_non_default_layout(tmpdir):
+ """Hooks a fake install directory with a non-default layout"""
+ real_store = spack.store.store
+ real_layout = spack.store.layout
+ spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
+ spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')),
+ path_scheme=ndef_install_path_scheme) # noqa: E501
+ yield spack.store
+ tmpdir.join('opt').remove()
+ spack.store.store = real_store
+ spack.store.layout = real_layout
+
+
+@pytest.fixture(scope='session')
+def config_setup():
+ cparser = argparse.ArgumentParser()
+ compiler.setup_parser(cparser)
+ cargs = cparser.parse_args(['add', '--scope', 'site'])
+ compiler.compiler(cparser, cargs)
+ cargs = cparser.parse_args(['list', '--scope', 'site'])
+ compiler.compiler(cparser, cargs)
+
+ rparser = argparse.ArgumentParser()
+ repocmd.setup_parser(rparser)
+ rargs = rparser.parse_args(['add', spack.paths.packages_path])
+ repocmd.repo(rparser, rargs)
+ # Set some spec name used globally
+ zspec = Spec('zlib')
+ zspec.concretize()
+ espec = Spec('environment-modules')
+ espec.concretize()
+ pspec = Spec('patchelf')
+ pspec.concretize()
+
+ yield espec, zspec, pspec
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+ 'install_dir_default_layout')
+def test_default_rpaths_create_install_default_layout(tmpdir,
+ mirror_directory_def,
+ config_setup,
+ mock_stage):
+ """
+ Test the creation and installation of buildcaches with default rpaths
+ into the default directory layout scheme.
+ """
+ tspec, zspec, pspec = config_setup
+ # Install patchelf needed for relocate in linux test environment
+ # Install some packages with dependent packages
+ iparser = argparse.ArgumentParser()
+ install.setup_parser(iparser)
+ if platform.system().lower() != 'darwin':
+ iargs = iparser.parse_args(['--no-cache', pspec.name])
+ install.install(iparser, iargs)
+ iargs = iparser.parse_args(['--no-cache', tspec.name])
+ install.install(iparser, iargs)
+
+ global mirror_path_def
+ mirror_path_def = mirror_directory_def
+ mparser = argparse.ArgumentParser()
+ mirror.setup_parser(mparser)
+ margs = mparser.parse_args(
+ ['add', '--scope', 'site', 'test-mirror-def', 'file://%s' % mirror_path_def])
+ mirror.mirror(mparser, margs)
+ margs = mparser.parse_args(['list'])
+ mirror.mirror(mparser, margs)
+
+ # setup argument parser
+ parser = argparse.ArgumentParser()
+ buildcache.setup_parser(parser)
+
+ # Set default buildcache args
+ create_args = ['create', '-a', '-u', '-d', str(mirror_path_def),
+ tspec.name]
+ install_args = ['install', '-a', '-u', tspec.name]
+
+ # Create a buildache of patchelf
+ if platform.system().lower() != 'darwin':
+ args = parser.parse_args(['create', '-a', '-u', '-d',
+ str(mirror_path_def),
+ pspec.name])
+ buildcache.buildcache(parser, args)
+
+ # Create a buildache
+ args = parser.parse_args(create_args)
+ buildcache.buildcache(parser, args)
+ # Test force overwrite create buildcache
+ create_args.insert(create_args.index('-a'), '-f')
+ args = parser.parse_args(create_args)
+ buildcache.buildcache(parser, args)
+ # list the buildcaches in the mirror
+ args = parser.parse_args(['list', '-f', '-l', '-v'])
+ buildcache.buildcache(parser, args)
+
+ # Uninstall the package and deps
+ uparser = argparse.ArgumentParser()
+ uninstall.setup_parser(uparser)
+ uargs = uparser.parse_args(['-y', '--dependents', zspec.name])
+ uninstall.uninstall(uparser, uargs)
+
+ # test install
+ args = parser.parse_args(install_args)
+ buildcache.buildcache(parser, args)
+
+ # This gives warning that spec is already installed
+ buildcache.buildcache(parser, args)
+
+ # test overwrite install
+ install_args.insert(install_args.index('-a'), '-f')
+ args = parser.parse_args(install_args)
+ buildcache.buildcache(parser, args)
+
+ args = parser.parse_args(['keys', '-f'])
+ buildcache.buildcache(parser, args)
+
+ args = parser.parse_args(['list'])
+ buildcache.buildcache(parser, args)
+
+ args = parser.parse_args(['list', '-f'])
+ buildcache.buildcache(parser, args)
+
+ args = parser.parse_args(['list', '-l', '-v'])
+ buildcache.buildcache(parser, args)
+ bindist._cached_specs = set()
+ spack.stage.purge()
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.nomockstage
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+ 'install_dir_non_default_layout')
+def test_default_rpaths_install_nondefault_layout(tmpdir, config_setup,
+ mock_stage):
+ """
+ Test the creation and installation of buildcaches with default rpaths
+ into the non-default directory layout scheme.
+ """
+ tspec, zspec, pspec = config_setup
+ global mirror_path_def
+ mparser = argparse.ArgumentParser()
+ mirror.setup_parser(mparser)
+ margs = mparser.parse_args(
+ ['add', '--scope', 'site', 'test-mirror-def', 'file://%s' % mirror_path_def])
+ mirror.mirror(mparser, margs)
+
+ # setup argument parser
+ parser = argparse.ArgumentParser()
+ buildcache.setup_parser(parser)
+
+ # Set default buildcache args
+ install_args = ['install', '-a', '-u', tspec.name]
+
+ # Install patchelf needed for relocate in linux test environment
+ if platform.system().lower() != 'darwin':
+ args = parser.parse_args(['install', '-a', '-u', pspec.name])
+ buildcache.buildcache(parser, args)
+
+ # Install some packages with dependent packages
+ # test install in non-default install path scheme
+ args = parser.parse_args(install_args)
+ buildcache.buildcache(parser, args)
+ # test force install in non-default install path scheme
+ install_args.insert(install_args.index('-a'), '-f')
+ args = parser.parse_args(install_args)
+ buildcache.buildcache(parser, args)
+
+ bindist._cached_specs = set()
+ spack.stage.purge()
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.nomockstage
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+ 'install_dir_default_layout')
+def test_relative_rpaths_create_default_layout(tmpdir, mirror_directory_rel,
+ config_setup, mock_stage):
+ """
+ Test the creation and installation of buildcaches with relative
+ rpaths into the default directory layout scheme.
+ """
+ tspec, zspec, pspec = config_setup
+ global mirror_path_rel
+ global mirror_path_def
+ mirror_path_rel = mirror_directory_rel
+ mparser = argparse.ArgumentParser()
+ mirror.setup_parser(mparser)
+ margs = mparser.parse_args(
+ ['add', '--scope', 'site', 'test-mirror-def', 'file://%s' % mirror_path_def])
+ mirror.mirror(mparser, margs)
+ # setup argument parser
+ parser = argparse.ArgumentParser()
+ buildcache.setup_parser(parser)
+
+ # install patchelf buildcache on linux
+ if platform.system().lower() != 'darwin':
+ args = parser.parse_args(['install', '-a', '-u', pspec.name])
+ buildcache.buildcache(parser, args)
+
+ # set default buildcache args
+ create_args = ['create', '-a', '-u', '-r', '-d',
+ str(mirror_path_rel),
+ tspec.name]
+ install_args = ['install', '-a', '-u',
+ tspec.name]
+
+ # Create a buildache of patchelf
+ if platform.system().lower() != 'darwin':
+ args = parser.parse_args(['create', '-a', '-u', '-d',
+ str(mirror_path_rel), pspec.name])
+ buildcache.buildcache(parser, args)
+
+ # install buildcache created with default rpaths
+ args = parser.parse_args(install_args)
+ buildcache.buildcache(parser, args)
+
+ # create build cache with relatived rpaths
+ args = parser.parse_args(create_args)
+ buildcache.buildcache(parser, args)
+
+ # Uninstall the package and deps
+ uparser = argparse.ArgumentParser()
+ uninstall.setup_parser(uparser)
+ uargs = uparser.parse_args(['-y', '--dependents', zspec.name])
+ uninstall.uninstall(uparser, uargs)
+
+ bindist._cached_specs = set()
+ spack.stage.purge()
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.nomockstage
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+ 'install_dir_default_layout')
+def test_relative_rpaths_install_default_layout(tmpdir, config_setup,
+ mock_stage):
+ """
+ Test the creation and installation of buildcaches with relative
+ rpaths into the default directory layout scheme.
+ """
+ tspec, zspec, pspec = config_setup
+ global mirror_path_rel
+ mparser = argparse.ArgumentParser()
+ mirror.setup_parser(mparser)
+ margs = mparser.parse_args(
+ ['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel])
+ mirror.mirror(mparser, margs)
+
+ # setup argument parser
+ parser = argparse.ArgumentParser()
+ buildcache.setup_parser(parser)
+
+ # install patchelf buildcache on linux
+ if platform.system().lower() != 'darwin':
+ args = parser.parse_args(['install', '-a', '-u', pspec.name])
+ buildcache.buildcache(parser, args)
+
+ # set default buildcache args
+ install_args = ['install', '-a', '-u',
+ tspec.name]
+
+ # install buildcache created with relativized rpaths
+ args = parser.parse_args(install_args)
+ buildcache.buildcache(parser, args)
+
+ # This gives warning that spec is already installed
+ buildcache.buildcache(parser, args)
+
+ # Uninstall the package and deps
+ uparser = argparse.ArgumentParser()
+ uninstall.setup_parser(uparser)
+ uargs = uparser.parse_args(['-y', '--dependents', zspec.name])
+ uninstall.uninstall(uparser, uargs)
+
+ # install build cache
+ buildcache.buildcache(parser, args)
+
+ # test overwrite install
+ install_args.insert(install_args.index('-a'), '-f')
+ args = parser.parse_args(install_args)
+ buildcache.buildcache(parser, args)
+
+ bindist._cached_specs = set()
+ spack.stage.purge()
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.nomockstage
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+ 'install_dir_non_default_layout')
+def test_relative_rpaths_install_nondefault(tmpdir, config_setup,
+ mock_stage):
+ """
+ Test the installation of buildcaches with relativized rpaths
+ into the non-default directory layout scheme.
+ """
+ tspec, zspec, pspec = config_setup
+ global mirror_path_rel
+ mparser = argparse.ArgumentParser()
+ mirror.setup_parser(mparser)
+ margs = mparser.parse_args(
+ ['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel])
+ mirror.mirror(mparser, margs)
+
+ # setup argument parser
+ parser = argparse.ArgumentParser()
+ buildcache.setup_parser(parser)
+
+ # Set default buildcache args
+ install_args = ['install', '-a', '-u', tspec.name]
+
+ # Install patchelf needed for relocate in linux test environment
+ if platform.system().lower() != 'darwin':
+ args = parser.parse_args(['install', '-a', '-u', pspec.name])
+ buildcache.buildcache(parser, args)
+
+ # test install in non-default install path scheme and relative path
+ args = parser.parse_args(install_args)
+ buildcache.buildcache(parser, args)
+
+ bindist._cached_specs = set()
+ spack.stage.purge()
--
cgit v1.2.3-70-g09d2