From 9aa3b4619b867e195ba74b0da6ed1db0039c331d Mon Sep 17 00:00:00 2001 From: Jonathon Anderson <17242663+blue42u@users.noreply.github.com> Date: Wed, 9 Aug 2023 02:46:59 -0500 Subject: containerize: ensure bootstrap images contain all system dependencies (#36818) This also makes `spack bootstrap status` exit 1 if some dependency is missing --- .github/workflows/unit_tests.yaml | 1 + lib/spack/docs/bootstrapping.rst | 9 +++++++-- lib/spack/spack/cmd/bootstrap.py | 2 ++ share/spack/templates/container/amazonlinux_2.dockerfile | 2 ++ share/spack/templates/container/bootstrap-base.dockerfile | 4 +++- share/spack/templates/container/centos_7.dockerfile | 2 ++ share/spack/templates/container/centos_stream.dockerfile | 3 +++ share/spack/templates/container/leap-15.dockerfile | 4 ++++ share/spack/templates/container/ubuntu_2004.dockerfile | 3 +++ 9 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index d4bd640d07..10432c657e 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -165,6 +165,7 @@ jobs: - name: Install Python packages run: | pip install --upgrade pip setuptools pytest coverage[toml] pytest-cov clingo pytest-xdist + pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black" - name: Setup git configuration run: | # Need this for the git tests to succeed. diff --git a/lib/spack/docs/bootstrapping.rst b/lib/spack/docs/bootstrapping.rst index dec548e9c6..8eca492413 100644 --- a/lib/spack/docs/bootstrapping.rst +++ b/lib/spack/docs/bootstrapping.rst @@ -32,9 +32,14 @@ can't be found. You can readily check if any prerequisite for using Spack is mis Spack will take care of bootstrapping any missing dependency marked as [B]. Dependencies marked as [-] are instead required to be found on the system. + % echo $? + 1 + In the case of the output shown above Spack detected that both ``clingo`` and ``gnupg`` are missing and it's giving detailed information on why they are needed and whether -they can be bootstrapped. Running a command that concretize a spec, like: +they can be bootstrapped. The return code of this command summarizes the results, if any +dependencies are missing the return code is ``1``, otherwise ``0``. Running a command that +concretizes a spec, like: .. code-block:: console @@ -44,7 +49,7 @@ they can be bootstrapped. Running a command that concretize a spec, like: ==> Installing "clingo-bootstrap@spack%apple-clang@12.0.0~docs~ipo+python build_type=Release arch=darwin-catalina-x86_64" from a buildcache [ ... ] -triggers the bootstrapping of clingo from pre-built binaries as expected. +automatically triggers the bootstrapping of clingo from pre-built binaries as expected. Users can also bootstrap all the dependencies needed by Spack in a single command, which might be useful to setup containers or other similar environments: diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index be2e64dbce..672d152565 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os.path import shutil +import sys import tempfile import llnl.util.filesystem @@ -326,6 +327,7 @@ def _status(args): if missing: print(llnl.util.tty.color.colorize(legend)) print() + sys.exit(1) def _add(args): diff --git a/share/spack/templates/container/amazonlinux_2.dockerfile b/share/spack/templates/container/amazonlinux_2.dockerfile index c5b69e317d..e11dd0c482 100644 --- a/share/spack/templates/container/amazonlinux_2.dockerfile +++ b/share/spack/templates/container/amazonlinux_2.dockerfile @@ -10,6 +10,7 @@ RUN yum update -y \ gcc-gfortran \ git \ gnupg2 \ + hg \ hostname \ iproute \ make \ @@ -18,6 +19,7 @@ RUN yum update -y \ python3-pip \ python3-setuptools \ unzip \ + zstd \ && pip3 install boto3 \ && rm -rf /var/cache/yum \ && yum clean all diff --git a/share/spack/templates/container/bootstrap-base.dockerfile b/share/spack/templates/container/bootstrap-base.dockerfile index ac069bbe37..38de6c8849 100644 --- a/share/spack/templates/container/bootstrap-base.dockerfile +++ b/share/spack/templates/container/bootstrap-base.dockerfile @@ -39,7 +39,9 @@ WORKDIR /root SHELL ["docker-shell"] # Creates the package cache -RUN spack bootstrap now && spack spec hdf5+mpi +RUN spack bootstrap now \ + && spack bootstrap status --optional \ + && spack spec hdf5+mpi ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"] CMD ["interactive-shell"] diff --git a/share/spack/templates/container/centos_7.dockerfile b/share/spack/templates/container/centos_7.dockerfile index d4401c5413..2d99ecad91 100644 --- a/share/spack/templates/container/centos_7.dockerfile +++ b/share/spack/templates/container/centos_7.dockerfile @@ -13,6 +13,7 @@ RUN yum update -y \ git \ gnupg2 \ hostname \ + hg \ iproute \ make \ patch \ @@ -20,6 +21,7 @@ RUN yum update -y \ python3-pip \ python3-setuptools \ unzip \ + zstd \ && pip3 install boto3 \ && rm -rf /var/cache/yum \ && yum clean all diff --git a/share/spack/templates/container/centos_stream.dockerfile b/share/spack/templates/container/centos_stream.dockerfile index 427b242b8d..c11a658fc7 100644 --- a/share/spack/templates/container/centos_stream.dockerfile +++ b/share/spack/templates/container/centos_stream.dockerfile @@ -15,13 +15,16 @@ RUN dnf update -y \ gcc-gfortran \ git \ gnupg2 \ + hg \ hostname \ iproute \ make \ + svn \ patch \ python3.11 \ python3.11-setuptools \ unzip \ + zstd \ && python3.11 -m ensurepip \ && pip3.11 install boto3 \ && rm -rf /var/cache/dnf \ diff --git a/share/spack/templates/container/leap-15.dockerfile b/share/spack/templates/container/leap-15.dockerfile index 2427c37754..386f0b5b2f 100644 --- a/share/spack/templates/container/leap-15.dockerfile +++ b/share/spack/templates/container/leap-15.dockerfile @@ -9,12 +9,16 @@ RUN zypper ref && \ gcc-c++\ gcc-fortran\ make\ + mercurial\ git\ gzip\ patch\ python3-base \ python3-boto3\ + subversion\ tar\ + unzip\ xz\ + zstd\ && zypper clean {% endblock %} diff --git a/share/spack/templates/container/ubuntu_2004.dockerfile b/share/spack/templates/container/ubuntu_2004.dockerfile index 26b6562952..f1c3ca456d 100644 --- a/share/spack/templates/container/ubuntu_2004.dockerfile +++ b/share/spack/templates/container/ubuntu_2004.dockerfile @@ -22,10 +22,13 @@ RUN apt-get -yqq update \ iproute2 \ locales \ make \ + mercurial \ + subversion \ python3 \ python3-pip \ python3-setuptools \ unzip \ + zstd \ && locale-gen en_US.UTF-8 \ && pip3 install boto3 \ && rm -rf /var/lib/apt/lists/* -- cgit v1.2.3-70-g09d2