From 9635ff3d20c17a92a89cf82db8d3f877dd04e1c7 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Fri, 31 Jan 2020 02:19:55 +0100 Subject: `spack containerize` generates containers from envs (#14202) This PR adds a new command to Spack: ```console $ spack containerize -h usage: spack containerize [-h] [--config CONFIG] creates recipes to build images for different container runtimes optional arguments: -h, --help show this help message and exit --config CONFIG configuration for the container recipe that will be generated ``` which takes an environment with an additional `container` section: ```yaml spack: specs: - gromacs build_type=Release - mpich - fftw precision=float packages: all: target: [broadwell] container: # Select the format of the recipe e.g. docker, # singularity or anything else that is currently supported format: docker # Select from a valid list of images base: image: "ubuntu:18.04" spack: prerelease # Additional system packages that are needed at runtime os_packages: - libgomp1 ``` and turns it into a `Dockerfile` or a Singularity definition file, for instance: ```Dockerfile # Build stage with Spack pre-installed and ready to be used FROM spack/ubuntu-bionic:prerelease as builder # What we want to install and how we want to install it # is specified in a manifest file (spack.yaml) RUN mkdir /opt/spack-environment \ && (echo "spack:" \ && echo " specs:" \ && echo " - gromacs build_type=Release" \ && echo " - mpich" \ && echo " - fftw precision=float" \ && echo " packages:" \ && echo " all:" \ && echo " target:" \ && echo " - broadwell" \ && echo " config:" \ && echo " install_tree: /opt/software" \ && echo " concretization: together" \ && echo " view: /opt/view") > /opt/spack-environment/spack.yaml # Install the software, remove unecessary deps and strip executables RUN cd /opt/spack-environment && spack install && spack autoremove -y RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \ xargs file -i | \ grep 'charset=binary' | \ grep 'x-executable\|x-archive\|x-sharedlib' | \ awk -F: '{print $1}' | xargs strip -s # Modifications to the environment that are necessary to run RUN cd /opt/spack-environment && \ spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh # Bare OS image to run the installed executables FROM ubuntu:18.04 COPY --from=builder /opt/spack-environment /opt/spack-environment COPY --from=builder /opt/software /opt/software COPY --from=builder /opt/view /opt/view COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh RUN apt-get -yqq update && apt-get -yqq upgrade \ && apt-get -yqq install libgomp1 \ && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] ``` --- lib/spack/docs/containers.rst | 307 +++++++++++++++++++++++ lib/spack/docs/dockerhub_spack.png | Bin 0 -> 90235 bytes lib/spack/docs/environments.rst | 2 + lib/spack/docs/index.rst | 1 + lib/spack/spack/cmd/containerize.py | 25 ++ lib/spack/spack/container/__init__.py | 81 ++++++ lib/spack/spack/container/images.json | 50 ++++ lib/spack/spack/container/images.py | 72 ++++++ lib/spack/spack/container/writers/__init__.py | 154 ++++++++++++ lib/spack/spack/container/writers/docker.py | 30 +++ lib/spack/spack/container/writers/singularity.py | 33 +++ lib/spack/spack/schema/container.py | 82 ++++++ lib/spack/spack/schema/merged.py | 2 + lib/spack/spack/test/cmd/gc.py | 4 +- lib/spack/spack/test/cmd/test.py | 2 +- lib/spack/spack/test/container/cli.py | 16 ++ lib/spack/spack/test/container/conftest.py | 43 ++++ lib/spack/spack/test/container/docker.py | 74 ++++++ lib/spack/spack/test/container/images.py | 58 +++++ lib/spack/spack/test/container/schema.py | 16 ++ lib/spack/spack/test/container/singularity.py | 42 ++++ share/spack/spack-completion.bash | 6 +- share/spack/templates/container/Dockerfile | 51 ++++ share/spack/templates/container/singularity.def | 90 +++++++ 24 files changed, 1238 insertions(+), 3 deletions(-) create mode 100644 lib/spack/docs/containers.rst create mode 100644 lib/spack/docs/dockerhub_spack.png create mode 100644 lib/spack/spack/cmd/containerize.py create mode 100644 lib/spack/spack/container/__init__.py create mode 100644 lib/spack/spack/container/images.json create mode 100644 lib/spack/spack/container/images.py create mode 100644 lib/spack/spack/container/writers/__init__.py create mode 100644 lib/spack/spack/container/writers/docker.py create mode 100644 lib/spack/spack/container/writers/singularity.py create mode 100644 lib/spack/spack/schema/container.py create mode 100644 lib/spack/spack/test/container/cli.py create mode 100644 lib/spack/spack/test/container/conftest.py create mode 100644 lib/spack/spack/test/container/docker.py create mode 100644 lib/spack/spack/test/container/images.py create mode 100644 lib/spack/spack/test/container/schema.py create mode 100644 lib/spack/spack/test/container/singularity.py create mode 100644 share/spack/templates/container/Dockerfile create mode 100644 share/spack/templates/container/singularity.def diff --git a/lib/spack/docs/containers.rst b/lib/spack/docs/containers.rst new file mode 100644 index 0000000000..bbb21a2e00 --- /dev/null +++ b/lib/spack/docs/containers.rst @@ -0,0 +1,307 @@ +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _containers: + +================ +Container Images +================ + +Spack can be an ideal tool to setup images for containers since all the +features discussed in :ref:`environments` can greatly help to manage +the installation of software during the image build process. Nonetheless, +building a production image from scratch still requires a lot of +boilerplate to: + +- Get Spack working within the image, possibly running as root +- Minimize the physical size of the software installed +- Properly update the system software in the base image + +To facilitate users with these tedious tasks, Spack provides a command +to automatically generate recipes for container images based on +Environments: + +.. code-block:: console + + $ ls + spack.yaml + + $ spack containerize + # Build stage with Spack pre-installed and ready to be used + FROM spack/centos7:latest as builder + + # What we want to install and how we want to install it + # is specified in a manifest file (spack.yaml) + RUN mkdir /opt/spack-environment \ + && (echo "spack:" \ + && echo " specs:" \ + && echo " - gromacs+mpi" \ + && echo " - mpich" \ + && echo " concretization: together" \ + && echo " config:" \ + && echo " install_tree: /opt/software" \ + && echo " view: /opt/view") > /opt/spack-environment/spack.yaml + + # Install the software, remove unecessary deps + RUN cd /opt/spack-environment && spack install && spack gc -y + + # Strip all the binaries + RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s + + # Modifications to the environment that are necessary to run + RUN cd /opt/spack-environment && \ + spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh + + + # Bare OS image to run the installed executables + FROM centos:7 + + COPY --from=builder /opt/spack-environment /opt/spack-environment + COPY --from=builder /opt/software /opt/software + COPY --from=builder /opt/view /opt/view + COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh + + RUN yum update -y && yum install -y epel-release && yum update -y \ + && yum install -y libgomp \ + && rm -rf /var/cache/yum && yum clean all + + RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc + + + LABEL "app"="gromacs" + LABEL "mpi"="mpich" + + ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] + + +The bits that make this automation possible are discussed in details +below. All the images generated in this way will be based on +multi-stage builds with: + +- A fat ``build`` stage containing common build tools and Spack itself +- A minimal ``final`` stage containing only the software requested by the user + +----------------- +Spack Base Images +----------------- + +Docker images with Spack preinstalled and ready to be used are +built on `Docker Hub `_ +at every push to ``develop`` or to a release branch. The OS that +are currently supported are summarized in the table below: + +.. _containers-supported-os: + +.. list-table:: Supported operating systems + :header-rows: 1 + + * - Operating System + - Base Image + - Spack Image + * - Ubuntu 16.04 + - ``ubuntu:16.04`` + - ``spack/ubuntu-xenial`` + * - Ubuntu 18.04 + - ``ubuntu:16.04`` + - ``spack/ubuntu-bionic`` + * - CentOS 6 + - ``centos:6`` + - ``spack/centos6`` + * - CentOS 7 + - ``centos:7`` + - ``spack/centos7`` + +All the images are tagged with the corresponding release of Spack: + +.. image:: dockerhub_spack.png + +with the exception of the ``latest`` tag that points to the HEAD +of the ``develop`` branch. These images are available for anyone +to use and take care of all the repetitive tasks that are necessary +to setup Spack within a container. All the container recipes generated +automatically by Spack use them as base images for their ``build`` stage. + + +------------------------- +Environment Configuration +------------------------- + +Any Spack Environment can be used for the automatic generation of container +recipes. Sensible defaults are provided for things like the base image or the +version of Spack used in the image. If a finer tuning is needed it can be +obtained by adding the relevant metadata under the ``container`` attribute +of environments: + +.. code-block:: yaml + + spack: + specs: + - gromacs+mpi + - mpich + + container: + # Select the format of the recipe e.g. docker, + # singularity or anything else that is currently supported + format: docker + + # Select from a valid list of images + base: + image: "centos:7" + spack: develop + + # Whether or not to strip binaries + strip: true + + # Additional system packages that are needed at runtime + os_packages: + - libgomp + + # Extra instructions + extra_instructions: + final: | + RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc + + # Labels for the image + labels: + app: "gromacs" + mpi: "mpich" + +The tables below describe the configuration options that are currently supported: + +.. list-table:: General configuration options for the ``container`` section of ``spack.yaml`` + :header-rows: 1 + + * - Option Name + - Description + - Allowed Values + - Required + * - ``format`` + - The format of the recipe + - ``docker`` or ``singularity`` + - Yes + * - ``base:image`` + - Base image for ``final`` stage + - See :ref:`containers-supported-os` + - Yes + * - ``base:spack`` + - Version of Spack + - Valid tags for ``base:image`` + - Yes + * - ``strip`` + - Whether to strip binaries + - ``true`` (default) or ``false`` + - No + * - ``os_packages`` + - System packages to be installed + - Valid packages for the ``final`` OS + - No + * - ``extra_instructions:build`` + - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``build`` stage + - Anything understood by the current ``format`` + - No + * - ``extra_instructions:final`` + - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``final`` stage + - Anything understood by the current ``format`` + - No + * - ``labels`` + - Labels to tag the image + - Pairs of key-value strings + - No + +.. list-table:: Configuration options specific to Singularity + :header-rows: 1 + + * - Option Name + - Description + - Allowed Values + - Required + * - ``singularity:runscript`` + - Content of ``%runscript`` + - Any valid script + - No + * - ``singularity:startscript`` + - Content of ``%startscript`` + - Any valid script + - No + * - ``singularity:test`` + - Content of ``%test`` + - Any valid script + - No + * - ``singularity:help`` + - Description of the image + - Description string + - No + +Once the Environment is properly configured a recipe for a container +image can be printed to standard output by issuing the following +command from the directory where the ``spack.yaml`` resides: + +.. code-block:: console + + $ spack containerize + +The example ``spack.yaml`` above would produce for instance the +following ``Dockerfile``: + +.. code-block:: docker + + # Build stage with Spack pre-installed and ready to be used + FROM spack/centos7:latest as builder + + # What we want to install and how we want to install it + # is specified in a manifest file (spack.yaml) + RUN mkdir /opt/spack-environment \ + && (echo "spack:" \ + && echo " specs:" \ + && echo " - gromacs+mpi" \ + && echo " - mpich" \ + && echo " concretization: together" \ + && echo " config:" \ + && echo " install_tree: /opt/software" \ + && echo " view: /opt/view") > /opt/spack-environment/spack.yaml + + # Install the software, remove unecessary deps + RUN cd /opt/spack-environment && spack install && spack gc -y + + # Strip all the binaries + RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s + + # Modifications to the environment that are necessary to run + RUN cd /opt/spack-environment && \ + spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh + + + # Bare OS image to run the installed executables + FROM centos:7 + + COPY --from=builder /opt/spack-environment /opt/spack-environment + COPY --from=builder /opt/software /opt/software + COPY --from=builder /opt/view /opt/view + COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh + + RUN yum update -y && yum install -y epel-release && yum update -y \ + && yum install -y libgomp \ + && rm -rf /var/cache/yum && yum clean all + + RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc + + + LABEL "app"="gromacs" + LABEL "mpi"="mpich" + + ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] + +.. note:: + Spack can also produce Singularity definition files to build the image. The + minimum version of Singularity required to build a SIF (Singularity Image Format) + from them is ``3.5.3``. \ No newline at end of file diff --git a/lib/spack/docs/dockerhub_spack.png b/lib/spack/docs/dockerhub_spack.png new file mode 100644 index 0000000000..44ff0ed7ed Binary files /dev/null and b/lib/spack/docs/dockerhub_spack.png differ diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 336d574bd7..5ec1ec9032 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -49,6 +49,8 @@ Spack uses a "manifest and lock" model similar to `Bundler gemfiles managers. The user input file is named ``spack.yaml`` and the lock file is named ``spack.lock`` +.. _environments-using: + ------------------ Using Environments ------------------ diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 489c15645a..8170f152a4 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -66,6 +66,7 @@ or refer to the full manual below. config_yaml build_settings environments + containers mirrors module_file_support repositories diff --git a/lib/spack/spack/cmd/containerize.py b/lib/spack/spack/cmd/containerize.py new file mode 100644 index 0000000000..cc2c001560 --- /dev/null +++ b/lib/spack/spack/cmd/containerize.py @@ -0,0 +1,25 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os +import os.path +import spack.container + +description = ("creates recipes to build images for different" + " container runtimes") +section = "container" +level = "long" + + +def containerize(parser, args): + config_dir = args.env_dir or os.getcwd() + config_file = os.path.abspath(os.path.join(config_dir, 'spack.yaml')) + if not os.path.exists(config_file): + msg = 'file not found: {0}' + raise ValueError(msg.format(config_file)) + + config = spack.container.validate(config_file) + + recipe = spack.container.recipe(config) + print(recipe) diff --git a/lib/spack/spack/container/__init__.py b/lib/spack/spack/container/__init__.py new file mode 100644 index 0000000000..fc3750355a --- /dev/null +++ b/lib/spack/spack/container/__init__.py @@ -0,0 +1,81 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Package that provides functions and classes to +generate container recipes from a Spack environment +""" +import warnings + +import spack.environment +import spack.schema.env as env +import spack.util.spack_yaml as syaml +from .writers import recipe + +__all__ = ['validate', 'recipe'] + + +def validate(configuration_file): + """Validate a Spack environment YAML file that is being used to generate a + recipe for a container. + + Since a few attributes of the configuration must have specific values for + the container recipe, this function returns a sanitized copy of the + configuration in the input file. If any modification is needed, a warning + will be issued. + + Args: + configuration_file (str): path to the Spack environment YAML file + + Returns: + A sanitized copy of the configuration stored in the input file + """ + import jsonschema + with open(configuration_file) as f: + config = syaml.load(f) + + # Ensure we have a "container" attribute with sensible defaults set + env_dict = spack.environment.config_dict(config) + env_dict.setdefault('container', { + 'format': 'docker', + 'base': {'image': 'ubuntu:18.04', 'spack': 'develop'} + }) + env_dict['container'].setdefault('format', 'docker') + env_dict['container'].setdefault( + 'base', {'image': 'ubuntu:18.04', 'spack': 'develop'} + ) + + # Remove attributes that are not needed / allowed in the + # container recipe + for subsection in ('cdash', 'gitlab_ci', 'modules'): + if subsection in env_dict: + msg = ('the subsection "{0}" in "{1}" is not used when generating' + ' container recipes and will be discarded') + warnings.warn(msg.format(subsection, configuration_file)) + env_dict.pop(subsection) + + # Set the default value of the concretization strategy to "together" and + # warn if the user explicitly set another value + env_dict.setdefault('concretization', 'together') + if env_dict['concretization'] != 'together': + msg = ('the "concretization" attribute of the environment is set ' + 'to "{0}" [the advised value is instead "together"]') + warnings.warn(msg.format(env_dict['concretization'])) + + # Check if the install tree was explicitly set to a custom value and warn + # that it will be overridden + environment_config = env_dict.get('config', {}) + if environment_config.get('install_tree', None): + msg = ('the "config:install_tree" attribute has been set explicitly ' + 'and will be overridden in the container image') + warnings.warn(msg) + + # Likewise for the view + environment_view = env_dict.get('view', None) + if environment_view: + msg = ('the "view" attribute has been set explicitly ' + 'and will be overridden in the container image') + warnings.warn(msg) + + jsonschema.validate(config, schema=env.schema) + return config diff --git a/lib/spack/spack/container/images.json b/lib/spack/spack/container/images.json new file mode 100644 index 0000000000..ecd911815d --- /dev/null +++ b/lib/spack/spack/container/images.json @@ -0,0 +1,50 @@ +{ + "ubuntu:18.04": { + "update": "apt-get -yqq update && apt-get -yqq upgrade", + "install": "apt-get -yqq install", + "clean": "rm -rf /var/lib/apt/lists/*", + "environment": [], + "build": "spack/ubuntu-bionic", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + }, + "ubuntu:16.04": { + "update": "apt-get -yqq update && apt-get -yqq upgrade", + "install": "apt-get -yqq install", + "clean": "rm -rf /var/lib/apt/lists/*", + "environment": [], + "build": "spack/ubuntu-xenial", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + }, + "centos:7": { + "update": "yum update -y && yum install -y epel-release && yum update -y", + "install": "yum install -y", + "clean": "rm -rf /var/cache/yum && yum clean all", + "environment": [], + "build": "spack/centos7", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + }, + "centos:6": { + "update": "yum update -y && yum install -y epel-release && yum update -y", + "install": "yum install -y", + "clean": "rm -rf /var/cache/yum && yum clean all", + "environment": [], + "build": "spack/centos6", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + } +} \ No newline at end of file diff --git a/lib/spack/spack/container/images.py b/lib/spack/spack/container/images.py new file mode 100644 index 0000000000..421fc24425 --- /dev/null +++ b/lib/spack/spack/container/images.py @@ -0,0 +1,72 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Manages the details on the images used in the build and the run stage.""" +import json +import os.path + +#: Global variable used to cache in memory the content of images.json +_data = None + + +def data(): + """Returns a dictionary with the static data on the images. + + The dictionary is read from a JSON file lazily the first time + this function is called. + """ + global _data + if not _data: + json_dir = os.path.abspath(os.path.dirname(__file__)) + json_file = os.path.join(json_dir, 'images.json') + with open(json_file) as f: + _data = json.load(f) + return _data + + +def build_info(image, spack_version): + """Returns the name of the build image and its tag. + + Args: + image (str): image to be used at run-time. Should be of the form + : e.g. "ubuntu:18.04" + spack_version (str): version of Spack that we want to use to build + + Returns: + A tuple with (image_name, image_tag) for the build image + """ + # Don't handle error here, as a wrong image should have been + # caught by the JSON schema + image_data = data()[image] + build_image = image_data['build'] + + # Try to check if we have a tag for this Spack version + try: + build_tag = image_data['build_tags'][spack_version] + except KeyError: + msg = ('the image "{0}" has no tag for Spack version "{1}" ' + '[valid versions are {2}]') + msg = msg.format(build_image, spack_version, + ', '.join(image_data['build_tags'].keys())) + raise ValueError(msg) + + return build_image, build_tag + + +def package_info(image): + """Returns the commands used to update system repositories, install + system packages and clean afterwards. + + Args: + image (str): image to be used at run-time. Should be of the form + : e.g. "ubuntu:18.04" + + Returns: + A tuple of (update, install, clean) commands. + """ + image_data = data()[image] + update = image_data['update'] + install = image_data['install'] + clean = image_data['clean'] + return update, install, clean diff --git a/lib/spack/spack/container/writers/__init__.py b/lib/spack/spack/container/writers/__init__.py new file mode 100644 index 0000000000..a1d2fa3102 --- /dev/null +++ b/lib/spack/spack/container/writers/__init__.py @@ -0,0 +1,154 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Writers for different kind of recipes and related +convenience functions. +""" +import collections +import copy + +import spack.environment +import spack.schema.env +import spack.tengine as tengine +import spack.util.spack_yaml as syaml + +from spack.container.images import build_info, package_info + +#: Caches all the writers that are currently supported +_writer_factory = {} + + +def writer(name): + """Decorator to register a factory for a recipe writer. + + Each factory should take a configuration dictionary and return a + properly configured writer that, when called, prints the + corresponding recipe. + """ + def _decorator(factory): + _writer_factory[name] = factory + return factory + return _decorator + + +def create(configuration): + """Returns a writer that conforms to the configuration passed as input. + + Args: + configuration: how to generate the current recipe + """ + name = spack.environment.config_dict(configuration)['container']['format'] + return _writer_factory[name](configuration) + + +def recipe(configuration): + """Returns a recipe that conforms to the configuration passed as input. + + Args: + configuration: how to generate the current recipe + """ + return create(configuration)() + + +class PathContext(tengine.Context): + """Generic context used to instantiate templates of recipes that + install software in a common location and make it available + directly via PATH. + """ + def __init__(self, config): + self.config = spack.environment.config_dict(config) + self.container_config = self.config['container'] + + @tengine.context_property + def run(self): + """Information related to the run image.""" + image = self.container_config['base']['image'] + Run = collections.namedtuple('Run', ['image']) + return Run(image=image) + + @tengine.context_property + def build(self): + """Information related to the build image.""" + + # Map the final image to the correct build image + run_image = self.container_config['base']['image'] + spack_version = self.container_config['base']['spack'] + image, tag = build_info(run_image, spack_version) + + Build = collections.namedtuple('Build', ['image', 'tag']) + return Build(image=image, tag=tag) + + @tengine.context_property + def strip(self): + """Whether or not to strip binaries in the image""" + return self.container_config.get('strip', True) + + @tengine.context_property + def paths(self): + """Important paths in the image""" + Paths = collections.namedtuple('Paths', [ + 'environment', 'store', 'view' + ]) + return Paths( + environment='/opt/spack-environment', + store='/opt/software', + view='/opt/view' + ) + + @tengine.context_property + def manifest(self): + """The spack.yaml file that should be used in the image""" + import jsonschema + # Copy in the part of spack.yaml prescribed in the configuration file + manifest = copy.deepcopy(self.config) + manifest.pop('container') + + # Ensure that a few paths are where they need to be + manifest.setdefault('config', syaml.syaml_dict()) + manifest['config']['install_tree'] = self.paths.store + manifest['view'] = self.paths.view + manifest = {'spack': manifest} + + # Validate the manifest file + jsonschema.validate(manifest, schema=spack.schema.env.schema) + + return syaml.dump(manifest, default_flow_style=False).strip() + + @tengine.context_property + def os_packages(self): + """Additional system packages that are needed at run-time.""" + package_list = self.container_config.get('os_packages', None) + if not package_list: + return package_list + + image = self.container_config['base']['image'] + update, install, clean = package_info(image) + Packages = collections.namedtuple( + 'Packages', ['update', 'install', 'list', 'clean'] + ) + return Packages(update=update, install=install, + list=package_list, clean=clean) + + @tengine.context_property + def extra_instructions(self): + Extras = collections.namedtuple('Extra', ['build', 'final']) + extras = self.container_config.get('extra_instructions', {}) + build, final = extras.get('build', None), extras.get('final', None) + return Extras(build=build, final=final) + + @tengine.context_property + def labels(self): + return self.container_config.get('labels', {}) + + def __call__(self): + """Returns the recipe as a string""" + env = tengine.make_environment() + t = env.get_template(self.template_name) + return t.render(**self.to_dict()) + + +# Import after function definition all the modules in this package, +# so that registration of writers will happen automatically +import spack.container.writers.singularity # noqa +import spack.container.writers.docker # noqa diff --git a/lib/spack/spack/container/writers/docker.py b/lib/spack/spack/container/writers/docker.py new file mode 100644 index 0000000000..557d22c803 --- /dev/null +++ b/lib/spack/spack/container/writers/docker.py @@ -0,0 +1,30 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import spack.tengine as tengine + +from . import writer, PathContext + + +@writer('docker') +class DockerContext(PathContext): + """Context used to instantiate a Dockerfile""" + #: Name of the template used for Dockerfiles + template_name = 'container/Dockerfile' + + @tengine.context_property + def manifest(self): + manifest_str = super(DockerContext, self).manifest + # Docker doesn't support HEREDOC so we need to resort to + # a horrible echo trick to have the manifest in the Dockerfile + echoed_lines = [] + for idx, line in enumerate(manifest_str.split('\n')): + if idx == 0: + echoed_lines.append('&& (echo "' + line + '" \\') + continue + echoed_lines.append('&& echo "' + line + '" \\') + + echoed_lines[-1] = echoed_lines[-1].replace(' \\', ')') + + return '\n'.join(echoed_lines) diff --git a/lib/spack/spack/container/writers/singularity.py b/lib/spack/spack/container/writers/singularity.py new file mode 100644 index 0000000000..32f29eb83d --- /dev/null +++ b/lib/spack/spack/container/writers/singularity.py @@ -0,0 +1,33 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import spack.tengine as tengine +from . import writer, PathContext + + +@writer('singularity') +class SingularityContext(PathContext): + """Context used to instantiate a Singularity definition file""" + #: Name of the template used for Singularity definition files + template_name = 'container/singularity.def' + + @property + def singularity_config(self): + return self.container_config.get('singularity', {}) + + @tengine.context_property + def runscript(self): + return self.singularity_config.get('runscript', '') + + @tengine.context_property + def startscript(self): + return self.singularity_config.get('startscript', '') + + @tengine.context_property + def test(self): + return self.singularity_config.get('test', '') + + @tengine.context_property + def help(self): + return self.singularity_config.get('help', '') diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py new file mode 100644 index 0000000000..cb1ed8d63a --- /dev/null +++ b/lib/spack/spack/schema/container.py @@ -0,0 +1,82 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Schema for the 'container' subsection of Spack environments.""" + +#: Schema for the container attribute included in Spack environments +container_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + # The recipe formats that are currently supported by the command + 'format': { + 'type': 'string', + 'enum': ['docker', 'singularity'] + }, + # Describes the base image to start from and the version + # of Spack to be used + 'base': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'image': { + 'type': 'string', + 'enum': ['ubuntu:18.04', + 'ubuntu:16.04', + 'centos:7', + 'centos:6'] + }, + 'spack': { + 'type': 'string', + 'enum': ['develop', '0.14', '0.14.0'] + } + }, + 'required': ['image', 'spack'] + }, + # Whether or not to strip installed binaries + 'strip': { + 'type': 'boolean', + 'default': True + }, + # Additional system packages that are needed at runtime + 'os_packages': { + 'type': 'array', + 'items': { + 'type': 'string' + } + }, + # Add labels to the image + 'labels': { + 'type': 'object', + }, + # Add a custom extra section at the bottom of a stage + 'extra_instructions': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'build': {'type': 'string'}, + 'final': {'type': 'string'} + } + }, + # Reserved for properties that are specific to each format + 'singularity': { + 'type': 'object', + 'additionalProperties': False, + 'default': {}, + 'properties': { + 'runscript': {'type': 'string'}, + 'startscript': {'type': 'string'}, + 'test': {'type': 'string'}, + 'help': {'type': 'string'} + } + }, + 'docker': { + 'type': 'object', + 'additionalProperties': False, + 'default': {}, + } + } +} + +properties = {'container': container_schema} diff --git a/lib/spack/spack/schema/merged.py b/lib/spack/spack/schema/merged.py index d56228c116..e118acf286 100644 --- a/lib/spack/spack/schema/merged.py +++ b/lib/spack/spack/schema/merged.py @@ -13,6 +13,7 @@ from llnl.util.lang import union_dicts import spack.schema.cdash import spack.schema.compilers import spack.schema.config +import spack.schema.container import spack.schema.gitlab_ci import spack.schema.mirrors import spack.schema.modules @@ -26,6 +27,7 @@ properties = union_dicts( spack.schema.cdash.properties, spack.schema.compilers.properties, spack.schema.config.properties, + spack.schema.container.properties, spack.schema.gitlab_ci.properties, spack.schema.mirrors.properties, spack.schema.modules.properties, diff --git a/lib/spack/spack/test/cmd/gc.py b/lib/spack/spack/test/cmd/gc.py index 76eb608cf2..22c85a1d78 100644 --- a/lib/spack/spack/test/cmd/gc.py +++ b/lib/spack/spack/test/cmd/gc.py @@ -30,7 +30,9 @@ def test_packages_are_removed(config, mutable_database, capsys): @pytest.mark.db -def test_gc_with_environment(config, mutable_database, capsys): +def test_gc_with_environment( + config, mutable_database, mutable_mock_env_path, capsys +): s = spack.spec.Spec('simple-inheritance') s.concretize() s.package.do_install(fake=True, explicit=True) diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py index 3595f91953..9a64209cfa 100644 --- a/lib/spack/spack/test/cmd/test.py +++ b/lib/spack/spack/test/cmd/test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/container/cli.py b/lib/spack/spack/test/container/cli.py new file mode 100644 index 0000000000..8e5403f072 --- /dev/null +++ b/lib/spack/spack/test/container/cli.py @@ -0,0 +1,16 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import llnl.util.filesystem as fs +import spack.main + + +containerize = spack.main.SpackCommand('containerize') + + +def test_command(configuration_dir, capsys): + with capsys.disabled(): + with fs.working_dir(configuration_dir): + output = containerize() + assert 'FROM spack/ubuntu-bionic' in output diff --git a/lib/spack/spack/test/container/conftest.py b/lib/spack/spack/test/container/conftest.py new file mode 100644 index 0000000000..802b34c5f8 --- /dev/null +++ b/lib/spack/spack/test/container/conftest.py @@ -0,0 +1,43 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + +import spack.util.spack_yaml as syaml + + +@pytest.fixture() +def minimal_configuration(): + return { + 'spack': { + 'specs': [ + 'gromacs', + 'mpich', + 'fftw precision=float' + ], + 'container': { + 'format': 'docker', + 'base': { + 'image': 'ubuntu:18.04', + 'spack': 'develop' + } + } + } + } + + +@pytest.fixture() +def config_dumper(tmpdir): + """Function that dumps an environment config in a temporary folder.""" + def dumper(configuration): + content = syaml.dump(configuration, default_flow_style=False) + config_file = tmpdir / 'spack.yaml' + config_file.write(content) + return str(tmpdir) + return dumper + + +@pytest.fixture() +def configuration_dir(minimal_configuration, config_dumper): + return config_dumper(minimal_configuration) diff --git a/lib/spack/spack/test/container/docker.py b/lib/spack/spack/test/container/docker.py new file mode 100644 index 0000000000..fbdc085828 --- /dev/null +++ b/lib/spack/spack/test/container/docker.py @@ -0,0 +1,74 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.container.writers as writers + + +def test_manifest(minimal_configuration): + writer = writers.create(minimal_configuration) + manifest_str = writer.manifest + for line in manifest_str.split('\n'): + assert 'echo' in line + + +def test_build_and_run_images(minimal_configuration): + writer = writers.create(minimal_configuration) + + # Test the output of run property + run = writer.run + assert run.image == 'ubuntu:18.04' + + # Test the output of the build property + build = writer.build + assert build.image == 'spack/ubuntu-bionic' + assert build.tag == 'latest' + + +def test_packages(minimal_configuration): + # In this minimal configuration we don't have packages + writer = writers.create(minimal_configuration) + assert writer.os_packages is None + + # If we add them a list should be returned + pkgs = ['libgomp1'] + minimal_configuration['spack']['container']['os_packages'] = pkgs + writer = writers.create(minimal_configuration) + p = writer.os_packages + assert p.update + assert p.install + assert p.clean + assert p.list == pkgs + + +def test_ensure_render_works(minimal_configuration): + # Here we just want to ensure that nothing is raised + writer = writers.create(minimal_configuration) + writer() + + +def test_strip_is_set_from_config(minimal_configuration): + writer = writers.create(minimal_configuration) + assert writer.strip is True + + minimal_configuration['spack']['container']['strip'] = False + writer = writers.create(minimal_configuration) + assert writer.strip is False + + +def test_extra_instructions_is_set_from_config(minimal_configuration): + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (None, None) + + test_line = 'RUN echo Hello world!' + e = minimal_configuration['spack']['container'] + e['extra_instructions'] = {} + e['extra_instructions']['build'] = test_line + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (test_line, None) + + e['extra_instructions']['final'] = test_line + del e['extra_instructions']['build'] + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (None, test_line) diff --git a/lib/spack/spack/test/container/images.py b/lib/spack/spack/test/container/images.py new file mode 100644 index 0000000000..808676c39a --- /dev/null +++ b/lib/spack/spack/test/container/images.py @@ -0,0 +1,58 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os.path + +import pytest + +import spack.container + + +@pytest.mark.parametrize('image,spack_version,expected', [ + ('ubuntu:18.04', 'develop', ('spack/ubuntu-bionic', 'latest')), + ('ubuntu:18.04', '0.14.0', ('spack/ubuntu-bionic', '0.14.0')), +]) +def test_build_info(image, spack_version, expected): + output = spack.container.images.build_info(image, spack_version) + assert output == expected + + +@pytest.mark.parametrize('image,spack_version', [ + ('ubuntu:18.04', 'doesnotexist') +]) +def test_build_info_error(image, spack_version): + with pytest.raises(ValueError, match=r"has no tag for"): + spack.container.images.build_info(image, spack_version) + + +@pytest.mark.parametrize('image', [ + 'ubuntu:18.04' +]) +def test_package_info(image): + update, install, clean = spack.container.images.package_info(image) + assert update + assert install + assert clean + + +@pytest.mark.parametrize('extra_config,expected_msg', [ + ({'modules': {'enable': ['tcl']}}, 'the subsection "modules" in'), + ({'concretization': 'separately'}, 'the "concretization" attribute'), + ({'config': {'install_tree': '/some/dir'}}, + 'the "config:install_tree" attribute has been set'), + ({'view': '/some/dir'}, 'the "view" attribute has been set') +]) +def test_validate( + extra_config, expected_msg, minimal_configuration, config_dumper +): + minimal_configuration['spack'].update(extra_config) + spack_yaml_dir = config_dumper(minimal_configuration) + spack_yaml = os.path.join(spack_yaml_dir, 'spack.yaml') + + with pytest.warns(UserWarning) as w: + spack.container.validate(spack_yaml) + + # Tests are designed to raise only one warning + assert len(w) == 1 + assert expected_msg in str(w.pop().message) diff --git a/lib/spack/spack/test/container/schema.py b/lib/spack/spack/test/container/schema.py new file mode 100644 index 0000000000..3f33a3f9f7 --- /dev/null +++ b/lib/spack/spack/test/container/schema.py @@ -0,0 +1,16 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.container +import spack.schema.container + + +def test_images_in_schema(): + properties = spack.schema.container.container_schema['properties'] + allowed_images = set( + properties['base']['properties']['image']['enum'] + ) + images_in_json = set(x for x in spack.container.images.data()) + assert images_in_json == allowed_images diff --git a/lib/spack/spack/test/container/singularity.py b/lib/spack/spack/test/container/singularity.py new file mode 100644 index 0000000000..445a119f6c --- /dev/null +++ b/lib/spack/spack/test/container/singularity.py @@ -0,0 +1,42 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + +import spack.container.writers as writers + + +@pytest.fixture +def singularity_configuration(minimal_configuration): + minimal_configuration['spack']['container']['format'] = 'singularity' + return minimal_configuration + + +def test_ensure_render_works(singularity_configuration): + container_config = singularity_configuration['spack']['container'] + assert container_config['format'] == 'singularity' + # Here we just want to ensure that nothing is raised + writer = writers.create(singularity_configuration) + writer() + + +@pytest.mark.parametrize('properties,expected', [ + ({'runscript': '/opt/view/bin/h5ls'}, + {'runscript': '/opt/view/bin/h5ls', + 'startscript': '', + 'test': '', + 'help': ''}) +]) +def test_singularity_specific_properties( + properties, expected, singularity_configuration +): + # Set the property in the configuration + container_config = singularity_configuration['spack']['container'] + for name, value in properties.items(): + container_config.setdefault('singularity', {})[name] = value + + # Assert the properties return the expected values + writer = writers.create(singularity_configuration) + for name, value in expected.items(): + assert getattr(writer, name) == value diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index b408d0b234..623e9fba73 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -313,7 +313,7 @@ _spack() { then SPACK_COMPREPLY="-h --help -H --all-help --color -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars" else - SPACK_COMPREPLY="activate add arch blame bootstrap build build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config configure create deactivate debug dependencies dependents deprecate dev-build diy docs edit env extensions fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mirror module patch pkg providers pydoc python reindex remove rm repo resource restage setup spec stage test uninstall unload upload-s3 url verify versions view" + SPACK_COMPREPLY="activate add arch blame bootstrap build build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config configure containerize create deactivate debug dependencies dependents deprecate dev-build diy docs edit env extensions fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mirror module patch pkg providers pydoc python reindex remove rm repo resource restage setup spec stage test uninstall unload upload-s3 url verify versions view" fi } @@ -628,6 +628,10 @@ _spack_configure() { fi } +_spack_containerize() { + SPACK_COMPREPLY="-h --help" +} + _spack_create() { if $list_options then diff --git a/share/spack/templates/container/Dockerfile b/share/spack/templates/container/Dockerfile new file mode 100644 index 0000000000..740f46e9ee --- /dev/null +++ b/share/spack/templates/container/Dockerfile @@ -0,0 +1,51 @@ +# Build stage with Spack pre-installed and ready to be used +FROM {{ build.image }}:{{ build.tag }} as builder + +# What we want to install and how we want to install it +# is specified in a manifest file (spack.yaml) +RUN mkdir {{ paths.environment }} \ +{{ manifest }} > {{ paths.environment }}/spack.yaml + +# Install the software, remove unecessary deps +RUN cd {{ paths.environment }} && spack install && spack gc -y +{% if strip %} + +# Strip all the binaries +RUN find -L {{ paths.view }}/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s +{% endif %} + +# Modifications to the environment that are necessary to run +RUN cd {{ paths.environment }} && \ + spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh + +{% if extra_instructions.build %} +{{ extra_instructions.build }} +{% endif %} + +# Bare OS image to run the installed executables +FROM {{ run.image }} + +COPY --from=builder {{ paths.environment }} {{ paths.environment }} +COPY --from=builder {{ paths.store }} {{ paths.store }} +COPY --from=builder {{ paths.view }} {{ paths.view }} +COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh + +{% if os_packages %} +RUN {{ os_packages.update }} \ + && {{ os_packages.install }}{% for pkg in os_packages.list %} {{ pkg }}{% endfor %} \ + && {{ os_packages.clean }} +{% endif %} + +{% if extra_instructions.final %} +{{ extra_instructions.final }} +{% endif %} + +{% for label, value in labels.items() %} +LABEL "{{ label }}"="{{ value }}" +{% endfor %} + +ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] diff --git a/share/spack/templates/container/singularity.def b/share/spack/templates/container/singularity.def new file mode 100644 index 0000000000..616e677f96 --- /dev/null +++ b/share/spack/templates/container/singularity.def @@ -0,0 +1,90 @@ +Bootstrap: docker +From: {{ build.image }}:{{ build.tag }} +Stage: build + +%post + # Create the manifest file for the installation in /opt/spack-environment + mkdir {{ paths.environment }} && cd {{ paths.environment }} + cat << EOF > spack.yaml +{{ manifest }} +EOF + + # Install all the required software + . /opt/spack/share/spack/setup-env.sh + spack install + spack gc -y + spack env activate --sh -d . >> {{ paths.environment }}/environment_modifications.sh +{% if strip %} + + # Strip the binaries to reduce the size of the image + find -L {{ paths.view }}/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s +{% endif %} +{% if extra_instructions.build %} +{{ extra_instructions.build }} +{% endif %} + + +{% if apps %} +{% for application, help_text in apps.items() %} + +%apprun {{ application }} + exec /opt/view/bin/{{ application }} "$@" + +%apphelp {{ application }} + {{help_text }} +{% endfor %} +{% endif %} + +Bootstrap: docker +From: {{ run.image }} +Stage: final + +%files from build + {{ paths.environment }} /opt + {{ paths.store }} /opt + {{ paths.view }} /opt + {{ paths.environment }}/environment_modifications.sh {{ paths.environment }}/environment_modifications.sh + +%post +{% if os_packages.list %} + # Update, install and cleanup of system packages + {{ os_packages.update }} + {{ os_packages.install }} {{ os_packages.list | join | replace('\n', ' ') }} + {{ os_packages.clean }} +{% endif %} + # Modify the environment without relying on sourcing shell specific files at startup + cat {{ paths.environment }}/environment_modifications.sh >> $SINGULARITY_ENVIRONMENT +{% if extra_instructions.final %} +{{ extra_instructions.final }} +{% endif %} + +{% if runscript %} +%runscript +{{ runscript }} +{% endif %} + +{% if startscript %} +%startscript +{{ startscript }} +{% endif %} + +{% if test %} +%test +{{ test }} +{% endif %} + +{% if help %} +%help +{{ help }} +{% endif %} + +{% if labels %} +%labels +{% for label, value in labels.items() %} + {{ label }} {{ value }} +{% endfor %} +{% endif %} \ No newline at end of file -- cgit v1.2.3-60-g2f50