containerize: pin the Spack version used in a container (#21910)

This PR permits to specify the `url` and `ref` of the Spack instance used in a container recipe simply by expanding the YAML schema as outlined in #20442:
```yaml
container:
  images:
    os: amazonlinux:2
    spack:
      ref: develop
      resolve_sha: true
```
The `resolve_sha` option, if true, verifies the `ref` by cloning the Spack repository in a temporary directory and transforming any tag or branch name to a commit sha. When this new ability is leveraged an additional "bootstrap" stage is added, which builds an image with Spack setup and ready to install software. The Spack repository to be used can be customized with the `url` keyword under `spack`.

Modifications:
- [x] Permit to pin the version of Spack, either by branch or tag or sha
- [x] Added a few new OSes (centos:8, amazonlinux:2, ubuntu:20.04, alpine:3, cuda:11.2.1)
- [x] Permit to print the bootstrap image as a standalone
- [x] Add documentation on the new part of the schema
- [x] Add unit tests for different use cases

1 files changed, 62 insertions, 13 deletions

diff --git a/lib/spack/spack/container/images.py b/lib/spack/spack/container/images.py
index 9d2e15f195..03591e68ee 100644
--- a/lib/spack/spack/container/images.py
+++ b/lib/spack/spack/container/images.py
@@ -2,9 +2,15 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-"""Manages the details on the images used in the build and the run stage."""
+"""Manages the details on the images used in the various stages."""
 import json
 import os.path
+import sys
+
+import llnl.util.filesystem as fs
+import llnl.util.tty as tty
+
+import spack.util.executable as executable
 
 #: Global variable used to cache in memory the content of images.json
 _data = None
@@ -39,18 +45,12 @@ def build_info(image, spack_version):
     # Don't handle error here, as a wrong image should have been
     # caught by the JSON schema
     image_data = data()["images"][image]
-    build_image = image_data['build']
-
-    # Try to check if we have a tag for this Spack version
-    try:
-        # Translate version from git to docker if necessary
-        build_tag = image_data['build_tags'].get(spack_version, spack_version)
-    except KeyError:
-        msg = ('the image "{0}" has no tag for Spack version "{1}" '
-               '[valid versions are {2}]')
-        msg = msg.format(build_image, spack_version,
-                         ', '.join(image_data['build_tags'].keys()))
-        raise ValueError(msg)
+    build_image = image_data.get('build', None)
+    if not build_image:
+        return None, None
+
+    # Translate version from git to docker if necessary
+    build_tag = image_data['build_tags'].get(spack_version, spack_version)
 
     return build_image, build_tag
 
@@ -70,6 +70,11 @@ def os_package_manager_for(image):
     return name
 
 
+def all_bootstrap_os():
+    """Return a list of all the OS that can be used to bootstrap Spack"""
+    return list(data()['images'])
+
+
 def commands_for(package_manager):
     """Returns the commands used to update system repositories, install
     system packages and clean afterwards.
@@ -82,3 +87,47 @@ def commands_for(package_manager):
     """
     info = data()["os_package_managers"][package_manager]
     return info['update'], info['install'], info['clean']
+
+
+def bootstrap_template_for(image):
+    return data()["images"][image]["bootstrap"]["template"]
+
+
+def _verify_ref(url, ref, enforce_sha):
+    # Do a checkout in a temporary directory
+    msg = 'Cloning "{0}" to verify ref "{1}"'.format(url, ref)
+    tty.info(msg, stream=sys.stderr)
+    git = executable.which('git', required=True)
+    with fs.temporary_dir():
+        git('clone', '-q', url, '.')
+        sha = git('rev-parse', '-q', ref + '^{commit}',
+                  output=str, error=os.devnull, fail_on_error=False)
+        if git.returncode:
+            msg = '"{0}" is not a valid reference for "{1}"'
+            raise RuntimeError(msg.format(sha, url))
+
+        if enforce_sha:
+            ref = sha.strip()
+
+        return ref
+
+
+def checkout_command(url, ref, enforce_sha, verify):
+    """Return the checkout command to be used in the bootstrap phase.
+
+    Args:
+        url (str): url of the Spack repository
+        ref (str): either a branch name, a tag or a commit sha
+        enforce_sha (bool): if true turns every
+        verify (bool):
+    """
+    url = url or 'https://github.com/spack/spack.git'
+    ref = ref or 'develop'
+    enforce_sha, verify = bool(enforce_sha), bool(verify)
+    # If we want to enforce a sha or verify the ref we need
+    # to checkout the repository locally
+    if enforce_sha or verify:
+        ref = _verify_ref(url, ref, enforce_sha)
+
+    command = 'git clone {0} . && git checkout {1} '.format(url, ref)
+    return command