Buildcache: bindist test without invoking spack compiler wrappers. (#15687)

* Buildcache:
   * Try mocking an install of quux, corge and garply using prebuilt binaries
   * Put patchelf install after ccache restore
   * Add script to install patchelf from source so it can be used on Ubuntu:Trusty which does not have a patchelf pat package. The script will skip building on macOS
   * Remove mirror at end of bindist test
   * Add patchelf to Ubuntu build env
   * Revert mock patchelf package to allow other tests to run.
   * Remove depends_on('patchelf', type='build') relying instead on
   * Test fixture to ensure patchelf is available.

* Call g++ command to build libraries directly during test build

* Flake8

* Install patchelf in before_install stage using apt unless on Trusty where a build is done.

* Add some symbolic links between packages

* Flake8

* Flake8:

* Update mock packages to write their own source files

* Create the stage because spec search does not create it any longer

* updates after change of list command arguments

* cleanup after merge

* flake8

1 files changed, 132 insertions, 0 deletions

diff --git a/var/spack/repos/builtin.mock/packages/quux/package.py b/var/spack/repos/builtin.mock/packages/quux/package.py
new file mode 100644
index 0000000000..6db243f154
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/quux/package.py
@@ -0,0 +1,132 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+import os
+
+
+class Quux(Package):
+    """Toy package for testing dependencies"""
+
+    homepage = "https://www.example.com"
+    url      = "https://github.com/gartung/quux/archive/v3.0.0.tar.gz"
+
+    version('3.0.0',
+            sha256='b91bc96fb746495786bddac2c527039177499f2f76d3fa9dcf0b393859e68484')
+
+    depends_on('garply')
+
+    def install(self, spec, prefix):
+        quux_cc = '''#include "quux.h"
+#include "garply/garply.h"
+#include "quux_version.h"
+#include <iostream>
+#include <stdexcept>
+
+const int Quux::version_major = quux_version_major;
+const int Quux::version_minor = quux_version_minor;
+
+Quux::Quux() {}
+
+int
+Quux::get_version() const
+{
+    return 10 * version_major + version_minor;
+}
+
+int
+Quux::quuxify() const
+{
+    int quux_version = get_version();
+    std::cout << "Quux::quuxify version " << quux_version
+              << " invoked" <<std::endl;
+    std::cout << "Quux config directory is %s" <<std::endl;
+    Garply garply;
+    int garply_version = garply.garplinate();
+
+    if (garply_version != quux_version) {
+        throw std::runtime_error(
+            "Quux found an incompatible version of Garply.");
+    }
+
+    return quux_version;
+}
+'''
+        quux_h = '''#ifndef QUUX_H_
+
+class Quux
+{
+private:
+    static const int version_major;
+    static const int version_minor;
+
+public:
+    Quux();
+    int get_version() const;
+    int quuxify() const;
+};
+
+#endif // QUUX_H_
+'''
+        quuxifier_cc = '''
+#include "quux.h"
+#include <iostream>
+
+int
+main()
+{
+    Quux quux;
+    quux.quuxify();
+
+    return 0;
+}
+'''
+        quux_version_h = '''const int quux_version_major = %s;
+const int quux_version_minor = %s;
+'''
+        mkdirp(prefix.lib64)
+        mkdirp('%s/quux' % prefix.include)
+        with open('%s/quux_version.h' % self.stage.source_path, 'w')  as f:
+            f.write(quux_version_h % (self.version[0], self.version[1:]))
+        with open('%s/quux/quux.cc' % self.stage.source_path, 'w') as f:
+            f.write(quux_cc % (prefix.config))
+        with open('%s/quux/quux.h' % self.stage.source_path, 'w') as f:
+            f.write(quux_h)
+        with open('%s/quux/quuxifier.cc' % self.stage.source_path, 'w') as f:
+            f.write(quuxifier_cc)
+        gpp = which('/usr/bin/g++')
+        gpp('-Dquux_EXPORTS',
+            '-I%s' % self.stage.source_path,
+            '-I%s' % spec['garply'].prefix.include,
+            '-O2', '-g', '-DNDEBUG', '-fPIC',
+            '-o', 'quux.cc.o',
+            '-c', 'quux/quux.cc')
+        gpp('-Dquux_EXPORTS',
+            '-I%s' % self.stage.source_path,
+            '-I%s' % spec['garply'].prefix.include,
+            '-O2', '-g', '-DNDEBUG', '-fPIC',
+            '-o', 'quuxifier.cc.o',
+            '-c', 'quux/quuxifier.cc')
+        gpp('-fPIC', '-O2', '-g', '-DNDEBUG', '-shared',
+            '-Wl,-soname,libquux.so', '-o', 'libquux.so', 'quux.cc.o',
+            '-Wl,-rpath,%s:%s::::' % (prefix.lib64,
+                                      spec['garply'].prefix.lib64),
+            '%s/libgarply.so' % spec['garply'].prefix.lib64)
+        gpp('-O2', '-g', '-DNDEBUG', '-rdynamic',
+            'quuxifier.cc.o', '-o', 'quuxifier',
+            '-Wl,-rpath,%s:%s::::' % (prefix.lib64,
+                                      spec['garply'].prefix.lib64),
+            'libquux.so',
+            '%s/libgarply.so' % spec['garply'].prefix.lib64)
+        copy('libquux.so', '%s/libquux.so' % prefix.lib64)
+        copy('quuxifier', '%s/quuxifier' % prefix.lib64)
+        copy('%s/quux/quux.h' % self.stage.source_path,
+             '%s/quux/quux.h' % prefix.include)
+        mkdirp(prefix.bin)
+        copy('quux_version.h', '%s/quux_version.h' % prefix.bin)
+        os.symlink('%s/quuxifier' % prefix.lib64, '%s/quuxifier' % prefix.bin)
+        os.symlink('%s/garplinator' % spec['garply'].prefix.lib64,
+                   '%s/garplinator' % prefix.bin)