From 9635ff3d20c17a92a89cf82db8d3f877dd04e1c7 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Fri, 31 Jan 2020 02:19:55 +0100 Subject: `spack containerize` generates containers from envs (#14202) This PR adds a new command to Spack: ```console $ spack containerize -h usage: spack containerize [-h] [--config CONFIG] creates recipes to build images for different container runtimes optional arguments: -h, --help show this help message and exit --config CONFIG configuration for the container recipe that will be generated ``` which takes an environment with an additional `container` section: ```yaml spack: specs: - gromacs build_type=Release - mpich - fftw precision=float packages: all: target: [broadwell] container: # Select the format of the recipe e.g. docker, # singularity or anything else that is currently supported format: docker # Select from a valid list of images base: image: "ubuntu:18.04" spack: prerelease # Additional system packages that are needed at runtime os_packages: - libgomp1 ``` and turns it into a `Dockerfile` or a Singularity definition file, for instance: ```Dockerfile # Build stage with Spack pre-installed and ready to be used FROM spack/ubuntu-bionic:prerelease as builder # What we want to install and how we want to install it # is specified in a manifest file (spack.yaml) RUN mkdir /opt/spack-environment \ && (echo "spack:" \ && echo " specs:" \ && echo " - gromacs build_type=Release" \ && echo " - mpich" \ && echo " - fftw precision=float" \ && echo " packages:" \ && echo " all:" \ && echo " target:" \ && echo " - broadwell" \ && echo " config:" \ && echo " install_tree: /opt/software" \ && echo " concretization: together" \ && echo " view: /opt/view") > /opt/spack-environment/spack.yaml # Install the software, remove unecessary deps and strip executables RUN cd /opt/spack-environment && spack install && spack autoremove -y RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \ xargs file -i | \ grep 'charset=binary' | \ grep 'x-executable\|x-archive\|x-sharedlib' | \ awk -F: '{print $1}' | xargs strip -s # Modifications to the environment that are necessary to run RUN cd /opt/spack-environment && \ spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh # Bare OS image to run the installed executables FROM ubuntu:18.04 COPY --from=builder /opt/spack-environment /opt/spack-environment COPY --from=builder /opt/software /opt/software COPY --from=builder /opt/view /opt/view COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh RUN apt-get -yqq update && apt-get -yqq upgrade \ && apt-get -yqq install libgomp1 \ && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] ``` --- share/spack/spack-completion.bash | 6 +- share/spack/templates/container/Dockerfile | 51 ++++++++++++++ share/spack/templates/container/singularity.def | 90 +++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 share/spack/templates/container/Dockerfile create mode 100644 share/spack/templates/container/singularity.def (limited to 'share') diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index b408d0b234..623e9fba73 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -313,7 +313,7 @@ _spack() { then SPACK_COMPREPLY="-h --help -H --all-help --color -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars" else - SPACK_COMPREPLY="activate add arch blame bootstrap build build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config configure create deactivate debug dependencies dependents deprecate dev-build diy docs edit env extensions fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mirror module patch pkg providers pydoc python reindex remove rm repo resource restage setup spec stage test uninstall unload upload-s3 url verify versions view" + SPACK_COMPREPLY="activate add arch blame bootstrap build build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config configure containerize create deactivate debug dependencies dependents deprecate dev-build diy docs edit env extensions fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mirror module patch pkg providers pydoc python reindex remove rm repo resource restage setup spec stage test uninstall unload upload-s3 url verify versions view" fi } @@ -628,6 +628,10 @@ _spack_configure() { fi } +_spack_containerize() { + SPACK_COMPREPLY="-h --help" +} + _spack_create() { if $list_options then diff --git a/share/spack/templates/container/Dockerfile b/share/spack/templates/container/Dockerfile new file mode 100644 index 0000000000..740f46e9ee --- /dev/null +++ b/share/spack/templates/container/Dockerfile @@ -0,0 +1,51 @@ +# Build stage with Spack pre-installed and ready to be used +FROM {{ build.image }}:{{ build.tag }} as builder + +# What we want to install and how we want to install it +# is specified in a manifest file (spack.yaml) +RUN mkdir {{ paths.environment }} \ +{{ manifest }} > {{ paths.environment }}/spack.yaml + +# Install the software, remove unecessary deps +RUN cd {{ paths.environment }} && spack install && spack gc -y +{% if strip %} + +# Strip all the binaries +RUN find -L {{ paths.view }}/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s +{% endif %} + +# Modifications to the environment that are necessary to run +RUN cd {{ paths.environment }} && \ + spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh + +{% if extra_instructions.build %} +{{ extra_instructions.build }} +{% endif %} + +# Bare OS image to run the installed executables +FROM {{ run.image }} + +COPY --from=builder {{ paths.environment }} {{ paths.environment }} +COPY --from=builder {{ paths.store }} {{ paths.store }} +COPY --from=builder {{ paths.view }} {{ paths.view }} +COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh + +{% if os_packages %} +RUN {{ os_packages.update }} \ + && {{ os_packages.install }}{% for pkg in os_packages.list %} {{ pkg }}{% endfor %} \ + && {{ os_packages.clean }} +{% endif %} + +{% if extra_instructions.final %} +{{ extra_instructions.final }} +{% endif %} + +{% for label, value in labels.items() %} +LABEL "{{ label }}"="{{ value }}" +{% endfor %} + +ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] diff --git a/share/spack/templates/container/singularity.def b/share/spack/templates/container/singularity.def new file mode 100644 index 0000000000..616e677f96 --- /dev/null +++ b/share/spack/templates/container/singularity.def @@ -0,0 +1,90 @@ +Bootstrap: docker +From: {{ build.image }}:{{ build.tag }} +Stage: build + +%post + # Create the manifest file for the installation in /opt/spack-environment + mkdir {{ paths.environment }} && cd {{ paths.environment }} + cat << EOF > spack.yaml +{{ manifest }} +EOF + + # Install all the required software + . /opt/spack/share/spack/setup-env.sh + spack install + spack gc -y + spack env activate --sh -d . >> {{ paths.environment }}/environment_modifications.sh +{% if strip %} + + # Strip the binaries to reduce the size of the image + find -L {{ paths.view }}/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s +{% endif %} +{% if extra_instructions.build %} +{{ extra_instructions.build }} +{% endif %} + + +{% if apps %} +{% for application, help_text in apps.items() %} + +%apprun {{ application }} + exec /opt/view/bin/{{ application }} "$@" + +%apphelp {{ application }} + {{help_text }} +{% endfor %} +{% endif %} + +Bootstrap: docker +From: {{ run.image }} +Stage: final + +%files from build + {{ paths.environment }} /opt + {{ paths.store }} /opt + {{ paths.view }} /opt + {{ paths.environment }}/environment_modifications.sh {{ paths.environment }}/environment_modifications.sh + +%post +{% if os_packages.list %} + # Update, install and cleanup of system packages + {{ os_packages.update }} + {{ os_packages.install }} {{ os_packages.list | join | replace('\n', ' ') }} + {{ os_packages.clean }} +{% endif %} + # Modify the environment without relying on sourcing shell specific files at startup + cat {{ paths.environment }}/environment_modifications.sh >> $SINGULARITY_ENVIRONMENT +{% if extra_instructions.final %} +{{ extra_instructions.final }} +{% endif %} + +{% if runscript %} +%runscript +{{ runscript }} +{% endif %} + +{% if startscript %} +%startscript +{{ startscript }} +{% endif %} + +{% if test %} +%test +{{ test }} +{% endif %} + +{% if help %} +%help +{{ help }} +{% endif %} + +{% if labels %} +%labels +{% for label, value in labels.items() %} + {{ label }} {{ value }} +{% endfor %} +{% endif %} \ No newline at end of file -- cgit v1.2.3-70-g09d2