diff options
Diffstat (limited to 'lib')
21 files changed, 1092 insertions, 2 deletions
diff --git a/lib/spack/docs/containers.rst b/lib/spack/docs/containers.rst new file mode 100644 index 0000000000..bbb21a2e00 --- /dev/null +++ b/lib/spack/docs/containers.rst @@ -0,0 +1,307 @@ +.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _containers: + +================ +Container Images +================ + +Spack can be an ideal tool to setup images for containers since all the +features discussed in :ref:`environments` can greatly help to manage +the installation of software during the image build process. Nonetheless, +building a production image from scratch still requires a lot of +boilerplate to: + +- Get Spack working within the image, possibly running as root +- Minimize the physical size of the software installed +- Properly update the system software in the base image + +To facilitate users with these tedious tasks, Spack provides a command +to automatically generate recipes for container images based on +Environments: + +.. code-block:: console + + $ ls + spack.yaml + + $ spack containerize + # Build stage with Spack pre-installed and ready to be used + FROM spack/centos7:latest as builder + + # What we want to install and how we want to install it + # is specified in a manifest file (spack.yaml) + RUN mkdir /opt/spack-environment \ + && (echo "spack:" \ + && echo " specs:" \ + && echo " - gromacs+mpi" \ + && echo " - mpich" \ + && echo " concretization: together" \ + && echo " config:" \ + && echo " install_tree: /opt/software" \ + && echo " view: /opt/view") > /opt/spack-environment/spack.yaml + + # Install the software, remove unecessary deps + RUN cd /opt/spack-environment && spack install && spack gc -y + + # Strip all the binaries + RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s + + # Modifications to the environment that are necessary to run + RUN cd /opt/spack-environment && \ + spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh + + + # Bare OS image to run the installed executables + FROM centos:7 + + COPY --from=builder /opt/spack-environment /opt/spack-environment + COPY --from=builder /opt/software /opt/software + COPY --from=builder /opt/view /opt/view + COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh + + RUN yum update -y && yum install -y epel-release && yum update -y \ + && yum install -y libgomp \ + && rm -rf /var/cache/yum && yum clean all + + RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc + + + LABEL "app"="gromacs" + LABEL "mpi"="mpich" + + ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] + + +The bits that make this automation possible are discussed in details +below. All the images generated in this way will be based on +multi-stage builds with: + +- A fat ``build`` stage containing common build tools and Spack itself +- A minimal ``final`` stage containing only the software requested by the user + +----------------- +Spack Base Images +----------------- + +Docker images with Spack preinstalled and ready to be used are +built on `Docker Hub <https://hub.docker.com/u/spack>`_ +at every push to ``develop`` or to a release branch. The OS that +are currently supported are summarized in the table below: + +.. _containers-supported-os: + +.. list-table:: Supported operating systems + :header-rows: 1 + + * - Operating System + - Base Image + - Spack Image + * - Ubuntu 16.04 + - ``ubuntu:16.04`` + - ``spack/ubuntu-xenial`` + * - Ubuntu 18.04 + - ``ubuntu:16.04`` + - ``spack/ubuntu-bionic`` + * - CentOS 6 + - ``centos:6`` + - ``spack/centos6`` + * - CentOS 7 + - ``centos:7`` + - ``spack/centos7`` + +All the images are tagged with the corresponding release of Spack: + +.. image:: dockerhub_spack.png + +with the exception of the ``latest`` tag that points to the HEAD +of the ``develop`` branch. These images are available for anyone +to use and take care of all the repetitive tasks that are necessary +to setup Spack within a container. All the container recipes generated +automatically by Spack use them as base images for their ``build`` stage. + + +------------------------- +Environment Configuration +------------------------- + +Any Spack Environment can be used for the automatic generation of container +recipes. Sensible defaults are provided for things like the base image or the +version of Spack used in the image. If a finer tuning is needed it can be +obtained by adding the relevant metadata under the ``container`` attribute +of environments: + +.. code-block:: yaml + + spack: + specs: + - gromacs+mpi + - mpich + + container: + # Select the format of the recipe e.g. docker, + # singularity or anything else that is currently supported + format: docker + + # Select from a valid list of images + base: + image: "centos:7" + spack: develop + + # Whether or not to strip binaries + strip: true + + # Additional system packages that are needed at runtime + os_packages: + - libgomp + + # Extra instructions + extra_instructions: + final: | + RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc + + # Labels for the image + labels: + app: "gromacs" + mpi: "mpich" + +The tables below describe the configuration options that are currently supported: + +.. list-table:: General configuration options for the ``container`` section of ``spack.yaml`` + :header-rows: 1 + + * - Option Name + - Description + - Allowed Values + - Required + * - ``format`` + - The format of the recipe + - ``docker`` or ``singularity`` + - Yes + * - ``base:image`` + - Base image for ``final`` stage + - See :ref:`containers-supported-os` + - Yes + * - ``base:spack`` + - Version of Spack + - Valid tags for ``base:image`` + - Yes + * - ``strip`` + - Whether to strip binaries + - ``true`` (default) or ``false`` + - No + * - ``os_packages`` + - System packages to be installed + - Valid packages for the ``final`` OS + - No + * - ``extra_instructions:build`` + - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``build`` stage + - Anything understood by the current ``format`` + - No + * - ``extra_instructions:final`` + - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``final`` stage + - Anything understood by the current ``format`` + - No + * - ``labels`` + - Labels to tag the image + - Pairs of key-value strings + - No + +.. list-table:: Configuration options specific to Singularity + :header-rows: 1 + + * - Option Name + - Description + - Allowed Values + - Required + * - ``singularity:runscript`` + - Content of ``%runscript`` + - Any valid script + - No + * - ``singularity:startscript`` + - Content of ``%startscript`` + - Any valid script + - No + * - ``singularity:test`` + - Content of ``%test`` + - Any valid script + - No + * - ``singularity:help`` + - Description of the image + - Description string + - No + +Once the Environment is properly configured a recipe for a container +image can be printed to standard output by issuing the following +command from the directory where the ``spack.yaml`` resides: + +.. code-block:: console + + $ spack containerize + +The example ``spack.yaml`` above would produce for instance the +following ``Dockerfile``: + +.. code-block:: docker + + # Build stage with Spack pre-installed and ready to be used + FROM spack/centos7:latest as builder + + # What we want to install and how we want to install it + # is specified in a manifest file (spack.yaml) + RUN mkdir /opt/spack-environment \ + && (echo "spack:" \ + && echo " specs:" \ + && echo " - gromacs+mpi" \ + && echo " - mpich" \ + && echo " concretization: together" \ + && echo " config:" \ + && echo " install_tree: /opt/software" \ + && echo " view: /opt/view") > /opt/spack-environment/spack.yaml + + # Install the software, remove unecessary deps + RUN cd /opt/spack-environment && spack install && spack gc -y + + # Strip all the binaries + RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s + + # Modifications to the environment that are necessary to run + RUN cd /opt/spack-environment && \ + spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh + + + # Bare OS image to run the installed executables + FROM centos:7 + + COPY --from=builder /opt/spack-environment /opt/spack-environment + COPY --from=builder /opt/software /opt/software + COPY --from=builder /opt/view /opt/view + COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh + + RUN yum update -y && yum install -y epel-release && yum update -y \ + && yum install -y libgomp \ + && rm -rf /var/cache/yum && yum clean all + + RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc + + + LABEL "app"="gromacs" + LABEL "mpi"="mpich" + + ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"] + +.. note:: + Spack can also produce Singularity definition files to build the image. The + minimum version of Singularity required to build a SIF (Singularity Image Format) + from them is ``3.5.3``.
\ No newline at end of file diff --git a/lib/spack/docs/dockerhub_spack.png b/lib/spack/docs/dockerhub_spack.png Binary files differnew file mode 100644 index 0000000000..44ff0ed7ed --- /dev/null +++ b/lib/spack/docs/dockerhub_spack.png diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 336d574bd7..5ec1ec9032 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -49,6 +49,8 @@ Spack uses a "manifest and lock" model similar to `Bundler gemfiles managers. The user input file is named ``spack.yaml`` and the lock file is named ``spack.lock`` +.. _environments-using: + ------------------ Using Environments ------------------ diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 489c15645a..8170f152a4 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -66,6 +66,7 @@ or refer to the full manual below. config_yaml build_settings environments + containers mirrors module_file_support repositories diff --git a/lib/spack/spack/cmd/containerize.py b/lib/spack/spack/cmd/containerize.py new file mode 100644 index 0000000000..cc2c001560 --- /dev/null +++ b/lib/spack/spack/cmd/containerize.py @@ -0,0 +1,25 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os +import os.path +import spack.container + +description = ("creates recipes to build images for different" + " container runtimes") +section = "container" +level = "long" + + +def containerize(parser, args): + config_dir = args.env_dir or os.getcwd() + config_file = os.path.abspath(os.path.join(config_dir, 'spack.yaml')) + if not os.path.exists(config_file): + msg = 'file not found: {0}' + raise ValueError(msg.format(config_file)) + + config = spack.container.validate(config_file) + + recipe = spack.container.recipe(config) + print(recipe) diff --git a/lib/spack/spack/container/__init__.py b/lib/spack/spack/container/__init__.py new file mode 100644 index 0000000000..fc3750355a --- /dev/null +++ b/lib/spack/spack/container/__init__.py @@ -0,0 +1,81 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Package that provides functions and classes to +generate container recipes from a Spack environment +""" +import warnings + +import spack.environment +import spack.schema.env as env +import spack.util.spack_yaml as syaml +from .writers import recipe + +__all__ = ['validate', 'recipe'] + + +def validate(configuration_file): + """Validate a Spack environment YAML file that is being used to generate a + recipe for a container. + + Since a few attributes of the configuration must have specific values for + the container recipe, this function returns a sanitized copy of the + configuration in the input file. If any modification is needed, a warning + will be issued. + + Args: + configuration_file (str): path to the Spack environment YAML file + + Returns: + A sanitized copy of the configuration stored in the input file + """ + import jsonschema + with open(configuration_file) as f: + config = syaml.load(f) + + # Ensure we have a "container" attribute with sensible defaults set + env_dict = spack.environment.config_dict(config) + env_dict.setdefault('container', { + 'format': 'docker', + 'base': {'image': 'ubuntu:18.04', 'spack': 'develop'} + }) + env_dict['container'].setdefault('format', 'docker') + env_dict['container'].setdefault( + 'base', {'image': 'ubuntu:18.04', 'spack': 'develop'} + ) + + # Remove attributes that are not needed / allowed in the + # container recipe + for subsection in ('cdash', 'gitlab_ci', 'modules'): + if subsection in env_dict: + msg = ('the subsection "{0}" in "{1}" is not used when generating' + ' container recipes and will be discarded') + warnings.warn(msg.format(subsection, configuration_file)) + env_dict.pop(subsection) + + # Set the default value of the concretization strategy to "together" and + # warn if the user explicitly set another value + env_dict.setdefault('concretization', 'together') + if env_dict['concretization'] != 'together': + msg = ('the "concretization" attribute of the environment is set ' + 'to "{0}" [the advised value is instead "together"]') + warnings.warn(msg.format(env_dict['concretization'])) + + # Check if the install tree was explicitly set to a custom value and warn + # that it will be overridden + environment_config = env_dict.get('config', {}) + if environment_config.get('install_tree', None): + msg = ('the "config:install_tree" attribute has been set explicitly ' + 'and will be overridden in the container image') + warnings.warn(msg) + + # Likewise for the view + environment_view = env_dict.get('view', None) + if environment_view: + msg = ('the "view" attribute has been set explicitly ' + 'and will be overridden in the container image') + warnings.warn(msg) + + jsonschema.validate(config, schema=env.schema) + return config diff --git a/lib/spack/spack/container/images.json b/lib/spack/spack/container/images.json new file mode 100644 index 0000000000..ecd911815d --- /dev/null +++ b/lib/spack/spack/container/images.json @@ -0,0 +1,50 @@ +{ + "ubuntu:18.04": { + "update": "apt-get -yqq update && apt-get -yqq upgrade", + "install": "apt-get -yqq install", + "clean": "rm -rf /var/lib/apt/lists/*", + "environment": [], + "build": "spack/ubuntu-bionic", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + }, + "ubuntu:16.04": { + "update": "apt-get -yqq update && apt-get -yqq upgrade", + "install": "apt-get -yqq install", + "clean": "rm -rf /var/lib/apt/lists/*", + "environment": [], + "build": "spack/ubuntu-xenial", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + }, + "centos:7": { + "update": "yum update -y && yum install -y epel-release && yum update -y", + "install": "yum install -y", + "clean": "rm -rf /var/cache/yum && yum clean all", + "environment": [], + "build": "spack/centos7", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + }, + "centos:6": { + "update": "yum update -y && yum install -y epel-release && yum update -y", + "install": "yum install -y", + "clean": "rm -rf /var/cache/yum && yum clean all", + "environment": [], + "build": "spack/centos6", + "build_tags": { + "develop": "latest", + "0.14": "0.14", + "0.14.0": "0.14.0" + } + } +}
\ No newline at end of file diff --git a/lib/spack/spack/container/images.py b/lib/spack/spack/container/images.py new file mode 100644 index 0000000000..421fc24425 --- /dev/null +++ b/lib/spack/spack/container/images.py @@ -0,0 +1,72 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Manages the details on the images used in the build and the run stage.""" +import json +import os.path + +#: Global variable used to cache in memory the content of images.json +_data = None + + +def data(): + """Returns a dictionary with the static data on the images. + + The dictionary is read from a JSON file lazily the first time + this function is called. + """ + global _data + if not _data: + json_dir = os.path.abspath(os.path.dirname(__file__)) + json_file = os.path.join(json_dir, 'images.json') + with open(json_file) as f: + _data = json.load(f) + return _data + + +def build_info(image, spack_version): + """Returns the name of the build image and its tag. + + Args: + image (str): image to be used at run-time. Should be of the form + <image_name>:<image_tag> e.g. "ubuntu:18.04" + spack_version (str): version of Spack that we want to use to build + + Returns: + A tuple with (image_name, image_tag) for the build image + """ + # Don't handle error here, as a wrong image should have been + # caught by the JSON schema + image_data = data()[image] + build_image = image_data['build'] + + # Try to check if we have a tag for this Spack version + try: + build_tag = image_data['build_tags'][spack_version] + except KeyError: + msg = ('the image "{0}" has no tag for Spack version "{1}" ' + '[valid versions are {2}]') + msg = msg.format(build_image, spack_version, + ', '.join(image_data['build_tags'].keys())) + raise ValueError(msg) + + return build_image, build_tag + + +def package_info(image): + """Returns the commands used to update system repositories, install + system packages and clean afterwards. + + Args: + image (str): image to be used at run-time. Should be of the form + <image_name>:<image_tag> e.g. "ubuntu:18.04" + + Returns: + A tuple of (update, install, clean) commands. + """ + image_data = data()[image] + update = image_data['update'] + install = image_data['install'] + clean = image_data['clean'] + return update, install, clean diff --git a/lib/spack/spack/container/writers/__init__.py b/lib/spack/spack/container/writers/__init__.py new file mode 100644 index 0000000000..a1d2fa3102 --- /dev/null +++ b/lib/spack/spack/container/writers/__init__.py @@ -0,0 +1,154 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Writers for different kind of recipes and related +convenience functions. +""" +import collections +import copy + +import spack.environment +import spack.schema.env +import spack.tengine as tengine +import spack.util.spack_yaml as syaml + +from spack.container.images import build_info, package_info + +#: Caches all the writers that are currently supported +_writer_factory = {} + + +def writer(name): + """Decorator to register a factory for a recipe writer. + + Each factory should take a configuration dictionary and return a + properly configured writer that, when called, prints the + corresponding recipe. + """ + def _decorator(factory): + _writer_factory[name] = factory + return factory + return _decorator + + +def create(configuration): + """Returns a writer that conforms to the configuration passed as input. + + Args: + configuration: how to generate the current recipe + """ + name = spack.environment.config_dict(configuration)['container']['format'] + return _writer_factory[name](configuration) + + +def recipe(configuration): + """Returns a recipe that conforms to the configuration passed as input. + + Args: + configuration: how to generate the current recipe + """ + return create(configuration)() + + +class PathContext(tengine.Context): + """Generic context used to instantiate templates of recipes that + install software in a common location and make it available + directly via PATH. + """ + def __init__(self, config): + self.config = spack.environment.config_dict(config) + self.container_config = self.config['container'] + + @tengine.context_property + def run(self): + """Information related to the run image.""" + image = self.container_config['base']['image'] + Run = collections.namedtuple('Run', ['image']) + return Run(image=image) + + @tengine.context_property + def build(self): + """Information related to the build image.""" + + # Map the final image to the correct build image + run_image = self.container_config['base']['image'] + spack_version = self.container_config['base']['spack'] + image, tag = build_info(run_image, spack_version) + + Build = collections.namedtuple('Build', ['image', 'tag']) + return Build(image=image, tag=tag) + + @tengine.context_property + def strip(self): + """Whether or not to strip binaries in the image""" + return self.container_config.get('strip', True) + + @tengine.context_property + def paths(self): + """Important paths in the image""" + Paths = collections.namedtuple('Paths', [ + 'environment', 'store', 'view' + ]) + return Paths( + environment='/opt/spack-environment', + store='/opt/software', + view='/opt/view' + ) + + @tengine.context_property + def manifest(self): + """The spack.yaml file that should be used in the image""" + import jsonschema + # Copy in the part of spack.yaml prescribed in the configuration file + manifest = copy.deepcopy(self.config) + manifest.pop('container') + + # Ensure that a few paths are where they need to be + manifest.setdefault('config', syaml.syaml_dict()) + manifest['config']['install_tree'] = self.paths.store + manifest['view'] = self.paths.view + manifest = {'spack': manifest} + + # Validate the manifest file + jsonschema.validate(manifest, schema=spack.schema.env.schema) + + return syaml.dump(manifest, default_flow_style=False).strip() + + @tengine.context_property + def os_packages(self): + """Additional system packages that are needed at run-time.""" + package_list = self.container_config.get('os_packages', None) + if not package_list: + return package_list + + image = self.container_config['base']['image'] + update, install, clean = package_info(image) + Packages = collections.namedtuple( + 'Packages', ['update', 'install', 'list', 'clean'] + ) + return Packages(update=update, install=install, + list=package_list, clean=clean) + + @tengine.context_property + def extra_instructions(self): + Extras = collections.namedtuple('Extra', ['build', 'final']) + extras = self.container_config.get('extra_instructions', {}) + build, final = extras.get('build', None), extras.get('final', None) + return Extras(build=build, final=final) + + @tengine.context_property + def labels(self): + return self.container_config.get('labels', {}) + + def __call__(self): + """Returns the recipe as a string""" + env = tengine.make_environment() + t = env.get_template(self.template_name) + return t.render(**self.to_dict()) + + +# Import after function definition all the modules in this package, +# so that registration of writers will happen automatically +import spack.container.writers.singularity # noqa +import spack.container.writers.docker # noqa diff --git a/lib/spack/spack/container/writers/docker.py b/lib/spack/spack/container/writers/docker.py new file mode 100644 index 0000000000..557d22c803 --- /dev/null +++ b/lib/spack/spack/container/writers/docker.py @@ -0,0 +1,30 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import spack.tengine as tengine + +from . import writer, PathContext + + +@writer('docker') +class DockerContext(PathContext): + """Context used to instantiate a Dockerfile""" + #: Name of the template used for Dockerfiles + template_name = 'container/Dockerfile' + + @tengine.context_property + def manifest(self): + manifest_str = super(DockerContext, self).manifest + # Docker doesn't support HEREDOC so we need to resort to + # a horrible echo trick to have the manifest in the Dockerfile + echoed_lines = [] + for idx, line in enumerate(manifest_str.split('\n')): + if idx == 0: + echoed_lines.append('&& (echo "' + line + '" \\') + continue + echoed_lines.append('&& echo "' + line + '" \\') + + echoed_lines[-1] = echoed_lines[-1].replace(' \\', ')') + + return '\n'.join(echoed_lines) diff --git a/lib/spack/spack/container/writers/singularity.py b/lib/spack/spack/container/writers/singularity.py new file mode 100644 index 0000000000..32f29eb83d --- /dev/null +++ b/lib/spack/spack/container/writers/singularity.py @@ -0,0 +1,33 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import spack.tengine as tengine +from . import writer, PathContext + + +@writer('singularity') +class SingularityContext(PathContext): + """Context used to instantiate a Singularity definition file""" + #: Name of the template used for Singularity definition files + template_name = 'container/singularity.def' + + @property + def singularity_config(self): + return self.container_config.get('singularity', {}) + + @tengine.context_property + def runscript(self): + return self.singularity_config.get('runscript', '') + + @tengine.context_property + def startscript(self): + return self.singularity_config.get('startscript', '') + + @tengine.context_property + def test(self): + return self.singularity_config.get('test', '') + + @tengine.context_property + def help(self): + return self.singularity_config.get('help', '') diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py new file mode 100644 index 0000000000..cb1ed8d63a --- /dev/null +++ b/lib/spack/spack/schema/container.py @@ -0,0 +1,82 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Schema for the 'container' subsection of Spack environments.""" + +#: Schema for the container attribute included in Spack environments +container_schema = { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + # The recipe formats that are currently supported by the command + 'format': { + 'type': 'string', + 'enum': ['docker', 'singularity'] + }, + # Describes the base image to start from and the version + # of Spack to be used + 'base': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'image': { + 'type': 'string', + 'enum': ['ubuntu:18.04', + 'ubuntu:16.04', + 'centos:7', + 'centos:6'] + }, + 'spack': { + 'type': 'string', + 'enum': ['develop', '0.14', '0.14.0'] + } + }, + 'required': ['image', 'spack'] + }, + # Whether or not to strip installed binaries + 'strip': { + 'type': 'boolean', + 'default': True + }, + # Additional system packages that are needed at runtime + 'os_packages': { + 'type': 'array', + 'items': { + 'type': 'string' + } + }, + # Add labels to the image + 'labels': { + 'type': 'object', + }, + # Add a custom extra section at the bottom of a stage + 'extra_instructions': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'build': {'type': 'string'}, + 'final': {'type': 'string'} + } + }, + # Reserved for properties that are specific to each format + 'singularity': { + 'type': 'object', + 'additionalProperties': False, + 'default': {}, + 'properties': { + 'runscript': {'type': 'string'}, + 'startscript': {'type': 'string'}, + 'test': {'type': 'string'}, + 'help': {'type': 'string'} + } + }, + 'docker': { + 'type': 'object', + 'additionalProperties': False, + 'default': {}, + } + } +} + +properties = {'container': container_schema} diff --git a/lib/spack/spack/schema/merged.py b/lib/spack/spack/schema/merged.py index d56228c116..e118acf286 100644 --- a/lib/spack/spack/schema/merged.py +++ b/lib/spack/spack/schema/merged.py @@ -13,6 +13,7 @@ from llnl.util.lang import union_dicts import spack.schema.cdash import spack.schema.compilers import spack.schema.config +import spack.schema.container import spack.schema.gitlab_ci import spack.schema.mirrors import spack.schema.modules @@ -26,6 +27,7 @@ properties = union_dicts( spack.schema.cdash.properties, spack.schema.compilers.properties, spack.schema.config.properties, + spack.schema.container.properties, spack.schema.gitlab_ci.properties, spack.schema.mirrors.properties, spack.schema.modules.properties, diff --git a/lib/spack/spack/test/cmd/gc.py b/lib/spack/spack/test/cmd/gc.py index 76eb608cf2..22c85a1d78 100644 --- a/lib/spack/spack/test/cmd/gc.py +++ b/lib/spack/spack/test/cmd/gc.py @@ -30,7 +30,9 @@ def test_packages_are_removed(config, mutable_database, capsys): @pytest.mark.db -def test_gc_with_environment(config, mutable_database, capsys): +def test_gc_with_environment( + config, mutable_database, mutable_mock_env_path, capsys +): s = spack.spec.Spec('simple-inheritance') s.concretize() s.package.do_install(fake=True, explicit=True) diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py index 3595f91953..9a64209cfa 100644 --- a/lib/spack/spack/test/cmd/test.py +++ b/lib/spack/spack/test/cmd/test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/lib/spack/spack/test/container/cli.py b/lib/spack/spack/test/container/cli.py new file mode 100644 index 0000000000..8e5403f072 --- /dev/null +++ b/lib/spack/spack/test/container/cli.py @@ -0,0 +1,16 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import llnl.util.filesystem as fs +import spack.main + + +containerize = spack.main.SpackCommand('containerize') + + +def test_command(configuration_dir, capsys): + with capsys.disabled(): + with fs.working_dir(configuration_dir): + output = containerize() + assert 'FROM spack/ubuntu-bionic' in output diff --git a/lib/spack/spack/test/container/conftest.py b/lib/spack/spack/test/container/conftest.py new file mode 100644 index 0000000000..802b34c5f8 --- /dev/null +++ b/lib/spack/spack/test/container/conftest.py @@ -0,0 +1,43 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + +import spack.util.spack_yaml as syaml + + +@pytest.fixture() +def minimal_configuration(): + return { + 'spack': { + 'specs': [ + 'gromacs', + 'mpich', + 'fftw precision=float' + ], + 'container': { + 'format': 'docker', + 'base': { + 'image': 'ubuntu:18.04', + 'spack': 'develop' + } + } + } + } + + +@pytest.fixture() +def config_dumper(tmpdir): + """Function that dumps an environment config in a temporary folder.""" + def dumper(configuration): + content = syaml.dump(configuration, default_flow_style=False) + config_file = tmpdir / 'spack.yaml' + config_file.write(content) + return str(tmpdir) + return dumper + + +@pytest.fixture() +def configuration_dir(minimal_configuration, config_dumper): + return config_dumper(minimal_configuration) diff --git a/lib/spack/spack/test/container/docker.py b/lib/spack/spack/test/container/docker.py new file mode 100644 index 0000000000..fbdc085828 --- /dev/null +++ b/lib/spack/spack/test/container/docker.py @@ -0,0 +1,74 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.container.writers as writers + + +def test_manifest(minimal_configuration): + writer = writers.create(minimal_configuration) + manifest_str = writer.manifest + for line in manifest_str.split('\n'): + assert 'echo' in line + + +def test_build_and_run_images(minimal_configuration): + writer = writers.create(minimal_configuration) + + # Test the output of run property + run = writer.run + assert run.image == 'ubuntu:18.04' + + # Test the output of the build property + build = writer.build + assert build.image == 'spack/ubuntu-bionic' + assert build.tag == 'latest' + + +def test_packages(minimal_configuration): + # In this minimal configuration we don't have packages + writer = writers.create(minimal_configuration) + assert writer.os_packages is None + + # If we add them a list should be returned + pkgs = ['libgomp1'] + minimal_configuration['spack']['container']['os_packages'] = pkgs + writer = writers.create(minimal_configuration) + p = writer.os_packages + assert p.update + assert p.install + assert p.clean + assert p.list == pkgs + + +def test_ensure_render_works(minimal_configuration): + # Here we just want to ensure that nothing is raised + writer = writers.create(minimal_configuration) + writer() + + +def test_strip_is_set_from_config(minimal_configuration): + writer = writers.create(minimal_configuration) + assert writer.strip is True + + minimal_configuration['spack']['container']['strip'] = False + writer = writers.create(minimal_configuration) + assert writer.strip is False + + +def test_extra_instructions_is_set_from_config(minimal_configuration): + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (None, None) + + test_line = 'RUN echo Hello world!' + e = minimal_configuration['spack']['container'] + e['extra_instructions'] = {} + e['extra_instructions']['build'] = test_line + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (test_line, None) + + e['extra_instructions']['final'] = test_line + del e['extra_instructions']['build'] + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (None, test_line) diff --git a/lib/spack/spack/test/container/images.py b/lib/spack/spack/test/container/images.py new file mode 100644 index 0000000000..808676c39a --- /dev/null +++ b/lib/spack/spack/test/container/images.py @@ -0,0 +1,58 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os.path + +import pytest + +import spack.container + + +@pytest.mark.parametrize('image,spack_version,expected', [ + ('ubuntu:18.04', 'develop', ('spack/ubuntu-bionic', 'latest')), + ('ubuntu:18.04', '0.14.0', ('spack/ubuntu-bionic', '0.14.0')), +]) +def test_build_info(image, spack_version, expected): + output = spack.container.images.build_info(image, spack_version) + assert output == expected + + +@pytest.mark.parametrize('image,spack_version', [ + ('ubuntu:18.04', 'doesnotexist') +]) +def test_build_info_error(image, spack_version): + with pytest.raises(ValueError, match=r"has no tag for"): + spack.container.images.build_info(image, spack_version) + + +@pytest.mark.parametrize('image', [ + 'ubuntu:18.04' +]) +def test_package_info(image): + update, install, clean = spack.container.images.package_info(image) + assert update + assert install + assert clean + + +@pytest.mark.parametrize('extra_config,expected_msg', [ + ({'modules': {'enable': ['tcl']}}, 'the subsection "modules" in'), + ({'concretization': 'separately'}, 'the "concretization" attribute'), + ({'config': {'install_tree': '/some/dir'}}, + 'the "config:install_tree" attribute has been set'), + ({'view': '/some/dir'}, 'the "view" attribute has been set') +]) +def test_validate( + extra_config, expected_msg, minimal_configuration, config_dumper +): + minimal_configuration['spack'].update(extra_config) + spack_yaml_dir = config_dumper(minimal_configuration) + spack_yaml = os.path.join(spack_yaml_dir, 'spack.yaml') + + with pytest.warns(UserWarning) as w: + spack.container.validate(spack_yaml) + + # Tests are designed to raise only one warning + assert len(w) == 1 + assert expected_msg in str(w.pop().message) diff --git a/lib/spack/spack/test/container/schema.py b/lib/spack/spack/test/container/schema.py new file mode 100644 index 0000000000..3f33a3f9f7 --- /dev/null +++ b/lib/spack/spack/test/container/schema.py @@ -0,0 +1,16 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.container +import spack.schema.container + + +def test_images_in_schema(): + properties = spack.schema.container.container_schema['properties'] + allowed_images = set( + properties['base']['properties']['image']['enum'] + ) + images_in_json = set(x for x in spack.container.images.data()) + assert images_in_json == allowed_images diff --git a/lib/spack/spack/test/container/singularity.py b/lib/spack/spack/test/container/singularity.py new file mode 100644 index 0000000000..445a119f6c --- /dev/null +++ b/lib/spack/spack/test/container/singularity.py @@ -0,0 +1,42 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import pytest + +import spack.container.writers as writers + + +@pytest.fixture +def singularity_configuration(minimal_configuration): + minimal_configuration['spack']['container']['format'] = 'singularity' + return minimal_configuration + + +def test_ensure_render_works(singularity_configuration): + container_config = singularity_configuration['spack']['container'] + assert container_config['format'] == 'singularity' + # Here we just want to ensure that nothing is raised + writer = writers.create(singularity_configuration) + writer() + + +@pytest.mark.parametrize('properties,expected', [ + ({'runscript': '/opt/view/bin/h5ls'}, + {'runscript': '/opt/view/bin/h5ls', + 'startscript': '', + 'test': '', + 'help': ''}) +]) +def test_singularity_specific_properties( + properties, expected, singularity_configuration +): + # Set the property in the configuration + container_config = singularity_configuration['spack']['container'] + for name, value in properties.items(): + container_config.setdefault('singularity', {})[name] = value + + # Assert the properties return the expected values + writer = writers.create(singularity_configuration) + for name, value in expected.items(): + assert getattr(writer, name) == value |
