Buildcache: attempt to build and install buildcaches in test environment (#15577)

* Put bindist.py on it's own branch

* Working on SL7 with python 3.6

* Flake8

* Put back clearing of compiler cache

1 files changed, 480 insertions, 0 deletions

diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py
new file mode 100644
index 0000000000..8ca7609942
--- /dev/null
+++ b/lib/spack/spack/test/bindist.py
@@ -0,0 +1,480 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""
+This test checks creating and install buildcaches
+"""
+import os
+import py
+import pytest
+import argparse
+import platform
+import spack.repo
+import spack.store
+import spack.binary_distribution as bindist
+import spack.cmd.buildcache as buildcache
+import spack.cmd.compiler as compiler
+import spack.cmd.repo as repocmd
+import spack.cmd.install as install
+import spack.cmd.uninstall as uninstall
+import spack.cmd.mirror as mirror
+from spack.spec import Spec
+from spack.directory_layout import YamlDirectoryLayout
+
+
+def_install_path_scheme = '${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}'  # noqa: E501
+ndef_install_path_scheme = '${PACKAGE}/${VERSION}/${ARCHITECTURE}-${COMPILERNAME}-${COMPILERVER}-${HASH}'  # noqa: E501
+
+mirror_path_def = None
+mirror_path_rel = None
+
+
+@pytest.fixture(scope='function')
+def cache_directory(tmpdir):
+    old_cache_path = spack.caches.fetch_cache
+    tmpdir.ensure('fetch_cache', dir=True)
+    fsc = spack.fetch_strategy.FsCache(str(tmpdir.join('fetch_cache')))
+    spack.config.caches = fsc
+    yield spack.config.caches
+    tmpdir.join('fetch_cache').remove()
+    spack.config.caches = old_cache_path
+
+
+@pytest.fixture(scope='session')
+def session_mirror_def(tmpdir_factory):
+    dir = tmpdir_factory.mktemp('mirror')
+    global mirror_path_rel
+    mirror_path_rel = dir
+    dir.ensure('build_cache', dir=True)
+    yield dir
+    dir.join('build_cache').remove()
+
+
+@pytest.fixture(scope='function')
+def mirror_directory_def(session_mirror_def):
+    yield str(session_mirror_def)
+
+
+@pytest.fixture(scope='session')
+def session_mirror_rel(tmpdir_factory):
+    dir = tmpdir_factory.mktemp('mirror')
+    global mirror_path_rel
+    mirror_path_rel = dir
+    dir.ensure('build_cache', dir=True)
+    yield dir
+    dir.join('build_cache').remove()
+
+
+@pytest.fixture(scope='function')
+def mirror_directory_rel(session_mirror_rel):
+    yield(session_mirror_rel)
+
+
+@pytest.fixture(scope='session')
+def config_directory(tmpdir_factory):
+    tmpdir = tmpdir_factory.mktemp('test_configs')
+    # restore some sane defaults for packages and config
+    config_path = py.path.local(spack.paths.etc_path)
+    modules_yaml = config_path.join('spack', 'defaults', 'modules.yaml')
+    os_modules_yaml = config_path.join('spack', 'defaults', '%s' %
+                                       platform.system().lower(),
+                                       'modules.yaml')
+    packages_yaml = config_path.join('spack', 'defaults', 'packages.yaml')
+    config_yaml = config_path.join('spack', 'defaults', 'config.yaml')
+    repos_yaml = config_path.join('spack', 'defaults', 'repos.yaml')
+    tmpdir.ensure('site', dir=True)
+    tmpdir.ensure('user', dir=True)
+    tmpdir.ensure('site/%s' % platform.system().lower(), dir=True)
+    modules_yaml.copy(tmpdir.join('site', 'modules.yaml'))
+    os_modules_yaml.copy(tmpdir.join('site/%s' % platform.system().lower(),
+                                     'modules.yaml'))
+    packages_yaml.copy(tmpdir.join('site', 'packages.yaml'))
+    config_yaml.copy(tmpdir.join('site', 'config.yaml'))
+    repos_yaml.copy(tmpdir.join('site', 'repos.yaml'))
+    yield tmpdir
+    tmpdir.remove()
+
+
+@pytest.fixture(scope='function')
+def default_config(tmpdir_factory, config_directory, monkeypatch):
+
+    mutable_dir = tmpdir_factory.mktemp('mutable_config').join('tmp')
+    config_directory.copy(mutable_dir)
+
+    cfg = spack.config.Configuration(
+        *[spack.config.ConfigScope(name, str(mutable_dir))
+          for name in ['site/%s' % platform.system().lower(),
+                       'site', 'user']])
+
+    monkeypatch.setattr(spack.config, 'config', cfg)
+
+    # This is essential, otherwise the cache will create weird side effects
+    # that will compromise subsequent tests if compilers.yaml is modified
+    monkeypatch.setattr(spack.compilers, '_cache_config_file', [])
+    njobs = spack.config.get('config:build_jobs')
+    if not njobs:
+        spack.config.set('config:build_jobs', 4, scope='user')
+    extensions = spack.config.get('config:template_dirs')
+    if not extensions:
+        spack.config.set('config:template_dirs',
+                         [os.path.join(spack.paths.share_path, 'templates')],
+                         scope='user')
+
+    mutable_dir.ensure('build_stage', dir=True)
+    build_stage = spack.config.get('config:build_stage')
+    if not build_stage:
+        spack.config.set('config:build_stage',
+                         [str(mutable_dir.join('build_stage'))], scope='user')
+    timeout = spack.config.get('config:connect_timeout')
+    if not timeout:
+        spack.config.set('config:connect_timeout', 10, scope='user')
+    print(spack.config.config)
+    yield spack.config.config
+    mutable_dir.remove()
+
+
+@pytest.fixture(scope='function')
+def install_dir_default_layout(tmpdir):
+    """Hooks a fake install directory with a default layout"""
+    real_store = spack.store.store
+    real_layout = spack.store.layout
+    spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
+    spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')),
+                                             path_scheme=def_install_path_scheme)  # noqa: E501
+    yield spack.store
+    tmpdir.join('opt').remove()
+    spack.store.store = real_store
+    spack.store.layout = real_layout
+
+
+@pytest.fixture(scope='function')
+def install_dir_non_default_layout(tmpdir):
+    """Hooks a fake install directory with a non-default layout"""
+    real_store = spack.store.store
+    real_layout = spack.store.layout
+    spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
+    spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')),
+                                             path_scheme=ndef_install_path_scheme)  # noqa: E501
+    yield spack.store
+    tmpdir.join('opt').remove()
+    spack.store.store = real_store
+    spack.store.layout = real_layout
+
+
+@pytest.fixture(scope='session')
+def config_setup():
+    cparser = argparse.ArgumentParser()
+    compiler.setup_parser(cparser)
+    cargs = cparser.parse_args(['add', '--scope', 'site'])
+    compiler.compiler(cparser, cargs)
+    cargs = cparser.parse_args(['list', '--scope', 'site'])
+    compiler.compiler(cparser, cargs)
+
+    rparser = argparse.ArgumentParser()
+    repocmd.setup_parser(rparser)
+    rargs = rparser.parse_args(['add', spack.paths.packages_path])
+    repocmd.repo(rparser, rargs)
+    # Set some spec name used globally
+    zspec = Spec('zlib')
+    zspec.concretize()
+    espec = Spec('environment-modules')
+    espec.concretize()
+    pspec = Spec('patchelf')
+    pspec.concretize()
+
+    yield espec, zspec, pspec
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+                         'install_dir_default_layout')
+def test_default_rpaths_create_install_default_layout(tmpdir,
+                                                      mirror_directory_def,
+                                                      config_setup,
+                                                      mock_stage):
+    """
+    Test the creation and installation of buildcaches with default rpaths
+    into the default directory layout scheme.
+    """
+    tspec, zspec, pspec = config_setup
+    # Install patchelf needed for relocate in linux test environment
+    # Install some packages with dependent packages
+    iparser = argparse.ArgumentParser()
+    install.setup_parser(iparser)
+    if platform.system().lower() != 'darwin':
+        iargs = iparser.parse_args(['--no-cache', pspec.name])
+        install.install(iparser, iargs)
+    iargs = iparser.parse_args(['--no-cache', tspec.name])
+    install.install(iparser, iargs)
+
+    global mirror_path_def
+    mirror_path_def = mirror_directory_def
+    mparser = argparse.ArgumentParser()
+    mirror.setup_parser(mparser)
+    margs = mparser.parse_args(
+        ['add', '--scope', 'site', 'test-mirror-def', 'file://%s' % mirror_path_def])
+    mirror.mirror(mparser, margs)
+    margs = mparser.parse_args(['list'])
+    mirror.mirror(mparser, margs)
+
+    # setup argument parser
+    parser = argparse.ArgumentParser()
+    buildcache.setup_parser(parser)
+
+    # Set default buildcache args
+    create_args = ['create', '-a', '-u', '-d', str(mirror_path_def),
+                   tspec.name]
+    install_args = ['install', '-a', '-u', tspec.name]
+
+    # Create a buildache of patchelf
+    if platform.system().lower() != 'darwin':
+        args = parser.parse_args(['create', '-a', '-u', '-d',
+                                  str(mirror_path_def),
+                                  pspec.name])
+        buildcache.buildcache(parser, args)
+
+    # Create a buildache
+    args = parser.parse_args(create_args)
+    buildcache.buildcache(parser, args)
+    # Test force overwrite create buildcache
+    create_args.insert(create_args.index('-a'), '-f')
+    args = parser.parse_args(create_args)
+    buildcache.buildcache(parser, args)
+    # list the buildcaches in the mirror
+    args = parser.parse_args(['list', '-f', '-l', '-v'])
+    buildcache.buildcache(parser, args)
+
+    # Uninstall the package and deps
+    uparser = argparse.ArgumentParser()
+    uninstall.setup_parser(uparser)
+    uargs = uparser.parse_args(['-y', '--dependents', zspec.name])
+    uninstall.uninstall(uparser, uargs)
+
+    # test install
+    args = parser.parse_args(install_args)
+    buildcache.buildcache(parser, args)
+
+    # This gives warning that spec is already installed
+    buildcache.buildcache(parser, args)
+
+    # test overwrite install
+    install_args.insert(install_args.index('-a'), '-f')
+    args = parser.parse_args(install_args)
+    buildcache.buildcache(parser, args)
+
+    args = parser.parse_args(['keys', '-f'])
+    buildcache.buildcache(parser, args)
+
+    args = parser.parse_args(['list'])
+    buildcache.buildcache(parser, args)
+
+    args = parser.parse_args(['list', '-f'])
+    buildcache.buildcache(parser, args)
+
+    args = parser.parse_args(['list', '-l', '-v'])
+    buildcache.buildcache(parser, args)
+    bindist._cached_specs = set()
+    spack.stage.purge()
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.nomockstage
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+                         'install_dir_non_default_layout')
+def test_default_rpaths_install_nondefault_layout(tmpdir, config_setup,
+                                                  mock_stage):
+    """
+    Test the creation and installation of buildcaches with default rpaths
+    into the non-default directory layout scheme.
+    """
+    tspec, zspec, pspec = config_setup
+    global mirror_path_def
+    mparser = argparse.ArgumentParser()
+    mirror.setup_parser(mparser)
+    margs = mparser.parse_args(
+        ['add', '--scope', 'site', 'test-mirror-def', 'file://%s' % mirror_path_def])
+    mirror.mirror(mparser, margs)
+
+    # setup argument parser
+    parser = argparse.ArgumentParser()
+    buildcache.setup_parser(parser)
+
+    # Set default buildcache args
+    install_args = ['install', '-a', '-u', tspec.name]
+
+    # Install patchelf needed for relocate in linux test environment
+    if platform.system().lower() != 'darwin':
+        args = parser.parse_args(['install', '-a', '-u', pspec.name])
+        buildcache.buildcache(parser, args)
+
+    # Install some packages with dependent packages
+    # test install in non-default install path scheme
+    args = parser.parse_args(install_args)
+    buildcache.buildcache(parser, args)
+    # test force install in non-default install path scheme
+    install_args.insert(install_args.index('-a'), '-f')
+    args = parser.parse_args(install_args)
+    buildcache.buildcache(parser, args)
+
+    bindist._cached_specs = set()
+    spack.stage.purge()
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.nomockstage
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+                         'install_dir_default_layout')
+def test_relative_rpaths_create_default_layout(tmpdir, mirror_directory_rel,
+                                               config_setup, mock_stage):
+    """
+    Test the creation and installation of buildcaches with relative
+    rpaths into the default directory layout scheme.
+    """
+    tspec, zspec, pspec = config_setup
+    global mirror_path_rel
+    global mirror_path_def
+    mirror_path_rel = mirror_directory_rel
+    mparser = argparse.ArgumentParser()
+    mirror.setup_parser(mparser)
+    margs = mparser.parse_args(
+        ['add', '--scope', 'site', 'test-mirror-def', 'file://%s' % mirror_path_def])
+    mirror.mirror(mparser, margs)
+    # setup argument parser
+    parser = argparse.ArgumentParser()
+    buildcache.setup_parser(parser)
+
+    # install patchelf buildcache on linux
+    if platform.system().lower() != 'darwin':
+        args = parser.parse_args(['install', '-a', '-u', pspec.name])
+        buildcache.buildcache(parser, args)
+
+    # set default buildcache args
+    create_args = ['create', '-a', '-u', '-r', '-d',
+                   str(mirror_path_rel),
+                   tspec.name]
+    install_args = ['install', '-a', '-u',
+                    tspec.name]
+
+    # Create a buildache of patchelf
+    if platform.system().lower() != 'darwin':
+        args = parser.parse_args(['create', '-a', '-u', '-d',
+                                  str(mirror_path_rel), pspec.name])
+        buildcache.buildcache(parser, args)
+
+    # install buildcache created with default rpaths
+    args = parser.parse_args(install_args)
+    buildcache.buildcache(parser, args)
+
+    # create build cache with relatived rpaths
+    args = parser.parse_args(create_args)
+    buildcache.buildcache(parser, args)
+
+    # Uninstall the package and deps
+    uparser = argparse.ArgumentParser()
+    uninstall.setup_parser(uparser)
+    uargs = uparser.parse_args(['-y', '--dependents', zspec.name])
+    uninstall.uninstall(uparser, uargs)
+
+    bindist._cached_specs = set()
+    spack.stage.purge()
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.nomockstage
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+                         'install_dir_default_layout')
+def test_relative_rpaths_install_default_layout(tmpdir, config_setup,
+                                                mock_stage):
+    """
+    Test the creation and installation of buildcaches with relative
+    rpaths into the default directory layout scheme.
+    """
+    tspec, zspec, pspec = config_setup
+    global mirror_path_rel
+    mparser = argparse.ArgumentParser()
+    mirror.setup_parser(mparser)
+    margs = mparser.parse_args(
+        ['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel])
+    mirror.mirror(mparser, margs)
+
+    # setup argument parser
+    parser = argparse.ArgumentParser()
+    buildcache.setup_parser(parser)
+
+    # install patchelf buildcache on linux
+    if platform.system().lower() != 'darwin':
+        args = parser.parse_args(['install', '-a', '-u', pspec.name])
+        buildcache.buildcache(parser, args)
+
+    # set default buildcache args
+    install_args = ['install', '-a', '-u',
+                    tspec.name]
+
+    # install buildcache created with relativized rpaths
+    args = parser.parse_args(install_args)
+    buildcache.buildcache(parser, args)
+
+    # This gives warning that spec is already installed
+    buildcache.buildcache(parser, args)
+
+    # Uninstall the package and deps
+    uparser = argparse.ArgumentParser()
+    uninstall.setup_parser(uparser)
+    uargs = uparser.parse_args(['-y', '--dependents', zspec.name])
+    uninstall.uninstall(uparser, uargs)
+
+    # install build cache
+    buildcache.buildcache(parser, args)
+
+    # test overwrite install
+    install_args.insert(install_args.index('-a'), '-f')
+    args = parser.parse_args(install_args)
+    buildcache.buildcache(parser, args)
+
+    bindist._cached_specs = set()
+    spack.stage.purge()
+
+
+@pytest.mark.disable_clean_stage_check
+@pytest.mark.maybeslow
+@pytest.mark.nomockstage
+@pytest.mark.usefixtures('default_config', 'cache_directory',
+                         'install_dir_non_default_layout')
+def test_relative_rpaths_install_nondefault(tmpdir, config_setup,
+                                            mock_stage):
+    """
+    Test the installation of buildcaches with relativized rpaths
+    into the non-default directory layout scheme.
+    """
+    tspec, zspec, pspec = config_setup
+    global mirror_path_rel
+    mparser = argparse.ArgumentParser()
+    mirror.setup_parser(mparser)
+    margs = mparser.parse_args(
+        ['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel])
+    mirror.mirror(mparser, margs)
+
+    # setup argument parser
+    parser = argparse.ArgumentParser()
+    buildcache.setup_parser(parser)
+
+    # Set default buildcache args
+    install_args = ['install', '-a', '-u', tspec.name]
+
+    # Install patchelf needed for relocate in linux test environment
+    if platform.system().lower() != 'darwin':
+        args = parser.parse_args(['install', '-a', '-u', pspec.name])
+        buildcache.buildcache(parser, args)
+
+    # test install in non-default install path scheme and relative path
+    args = parser.parse_args(install_args)
+    buildcache.buildcache(parser, args)
+
+    bindist._cached_specs = set()
+    spack.stage.purge()