summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/containers.rst307
-rw-r--r--lib/spack/docs/dockerhub_spack.pngbin0 -> 90235 bytes
-rw-r--r--lib/spack/docs/environments.rst2
-rw-r--r--lib/spack/docs/index.rst1
-rw-r--r--lib/spack/spack/cmd/containerize.py25
-rw-r--r--lib/spack/spack/container/__init__.py81
-rw-r--r--lib/spack/spack/container/images.json50
-rw-r--r--lib/spack/spack/container/images.py72
-rw-r--r--lib/spack/spack/container/writers/__init__.py154
-rw-r--r--lib/spack/spack/container/writers/docker.py30
-rw-r--r--lib/spack/spack/container/writers/singularity.py33
-rw-r--r--lib/spack/spack/schema/container.py82
-rw-r--r--lib/spack/spack/schema/merged.py2
-rw-r--r--lib/spack/spack/test/cmd/gc.py4
-rw-r--r--lib/spack/spack/test/cmd/test.py2
-rw-r--r--lib/spack/spack/test/container/cli.py16
-rw-r--r--lib/spack/spack/test/container/conftest.py43
-rw-r--r--lib/spack/spack/test/container/docker.py74
-rw-r--r--lib/spack/spack/test/container/images.py58
-rw-r--r--lib/spack/spack/test/container/schema.py16
-rw-r--r--lib/spack/spack/test/container/singularity.py42
21 files changed, 1092 insertions, 2 deletions
diff --git a/lib/spack/docs/containers.rst b/lib/spack/docs/containers.rst
new file mode 100644
index 0000000000..bbb21a2e00
--- /dev/null
+++ b/lib/spack/docs/containers.rst
@@ -0,0 +1,307 @@
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+ Spack Project Developers. See the top-level COPYRIGHT file for details.
+
+ SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+.. _containers:
+
+================
+Container Images
+================
+
+Spack can be an ideal tool to setup images for containers since all the
+features discussed in :ref:`environments` can greatly help to manage
+the installation of software during the image build process. Nonetheless,
+building a production image from scratch still requires a lot of
+boilerplate to:
+
+- Get Spack working within the image, possibly running as root
+- Minimize the physical size of the software installed
+- Properly update the system software in the base image
+
+To facilitate users with these tedious tasks, Spack provides a command
+to automatically generate recipes for container images based on
+Environments:
+
+.. code-block:: console
+
+ $ ls
+ spack.yaml
+
+ $ spack containerize
+ # Build stage with Spack pre-installed and ready to be used
+ FROM spack/centos7:latest as builder
+
+ # What we want to install and how we want to install it
+ # is specified in a manifest file (spack.yaml)
+ RUN mkdir /opt/spack-environment \
+ && (echo "spack:" \
+ && echo " specs:" \
+ && echo " - gromacs+mpi" \
+ && echo " - mpich" \
+ && echo " concretization: together" \
+ && echo " config:" \
+ && echo " install_tree: /opt/software" \
+ && echo " view: /opt/view") > /opt/spack-environment/spack.yaml
+
+ # Install the software, remove unecessary deps
+ RUN cd /opt/spack-environment && spack install && spack gc -y
+
+ # Strip all the binaries
+ RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \
+ xargs file -i | \
+ grep 'charset=binary' | \
+ grep 'x-executable\|x-archive\|x-sharedlib' | \
+ awk -F: '{print $1}' | xargs strip -s
+
+ # Modifications to the environment that are necessary to run
+ RUN cd /opt/spack-environment && \
+ spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+
+
+ # Bare OS image to run the installed executables
+ FROM centos:7
+
+ COPY --from=builder /opt/spack-environment /opt/spack-environment
+ COPY --from=builder /opt/software /opt/software
+ COPY --from=builder /opt/view /opt/view
+ COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh
+
+ RUN yum update -y && yum install -y epel-release && yum update -y \
+ && yum install -y libgomp \
+ && rm -rf /var/cache/yum && yum clean all
+
+ RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc
+
+
+ LABEL "app"="gromacs"
+ LABEL "mpi"="mpich"
+
+ ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"]
+
+
+The bits that make this automation possible are discussed in details
+below. All the images generated in this way will be based on
+multi-stage builds with:
+
+- A fat ``build`` stage containing common build tools and Spack itself
+- A minimal ``final`` stage containing only the software requested by the user
+
+-----------------
+Spack Base Images
+-----------------
+
+Docker images with Spack preinstalled and ready to be used are
+built on `Docker Hub <https://hub.docker.com/u/spack>`_
+at every push to ``develop`` or to a release branch. The OS that
+are currently supported are summarized in the table below:
+
+.. _containers-supported-os:
+
+.. list-table:: Supported operating systems
+ :header-rows: 1
+
+ * - Operating System
+ - Base Image
+ - Spack Image
+ * - Ubuntu 16.04
+ - ``ubuntu:16.04``
+ - ``spack/ubuntu-xenial``
+ * - Ubuntu 18.04
+ - ``ubuntu:16.04``
+ - ``spack/ubuntu-bionic``
+ * - CentOS 6
+ - ``centos:6``
+ - ``spack/centos6``
+ * - CentOS 7
+ - ``centos:7``
+ - ``spack/centos7``
+
+All the images are tagged with the corresponding release of Spack:
+
+.. image:: dockerhub_spack.png
+
+with the exception of the ``latest`` tag that points to the HEAD
+of the ``develop`` branch. These images are available for anyone
+to use and take care of all the repetitive tasks that are necessary
+to setup Spack within a container. All the container recipes generated
+automatically by Spack use them as base images for their ``build`` stage.
+
+
+-------------------------
+Environment Configuration
+-------------------------
+
+Any Spack Environment can be used for the automatic generation of container
+recipes. Sensible defaults are provided for things like the base image or the
+version of Spack used in the image. If a finer tuning is needed it can be
+obtained by adding the relevant metadata under the ``container`` attribute
+of environments:
+
+.. code-block:: yaml
+
+ spack:
+ specs:
+ - gromacs+mpi
+ - mpich
+
+ container:
+ # Select the format of the recipe e.g. docker,
+ # singularity or anything else that is currently supported
+ format: docker
+
+ # Select from a valid list of images
+ base:
+ image: "centos:7"
+ spack: develop
+
+ # Whether or not to strip binaries
+ strip: true
+
+ # Additional system packages that are needed at runtime
+ os_packages:
+ - libgomp
+
+ # Extra instructions
+ extra_instructions:
+ final: |
+ RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc
+
+ # Labels for the image
+ labels:
+ app: "gromacs"
+ mpi: "mpich"
+
+The tables below describe the configuration options that are currently supported:
+
+.. list-table:: General configuration options for the ``container`` section of ``spack.yaml``
+ :header-rows: 1
+
+ * - Option Name
+ - Description
+ - Allowed Values
+ - Required
+ * - ``format``
+ - The format of the recipe
+ - ``docker`` or ``singularity``
+ - Yes
+ * - ``base:image``
+ - Base image for ``final`` stage
+ - See :ref:`containers-supported-os`
+ - Yes
+ * - ``base:spack``
+ - Version of Spack
+ - Valid tags for ``base:image``
+ - Yes
+ * - ``strip``
+ - Whether to strip binaries
+ - ``true`` (default) or ``false``
+ - No
+ * - ``os_packages``
+ - System packages to be installed
+ - Valid packages for the ``final`` OS
+ - No
+ * - ``extra_instructions:build``
+ - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``build`` stage
+ - Anything understood by the current ``format``
+ - No
+ * - ``extra_instructions:final``
+ - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``final`` stage
+ - Anything understood by the current ``format``
+ - No
+ * - ``labels``
+ - Labels to tag the image
+ - Pairs of key-value strings
+ - No
+
+.. list-table:: Configuration options specific to Singularity
+ :header-rows: 1
+
+ * - Option Name
+ - Description
+ - Allowed Values
+ - Required
+ * - ``singularity:runscript``
+ - Content of ``%runscript``
+ - Any valid script
+ - No
+ * - ``singularity:startscript``
+ - Content of ``%startscript``
+ - Any valid script
+ - No
+ * - ``singularity:test``
+ - Content of ``%test``
+ - Any valid script
+ - No
+ * - ``singularity:help``
+ - Description of the image
+ - Description string
+ - No
+
+Once the Environment is properly configured a recipe for a container
+image can be printed to standard output by issuing the following
+command from the directory where the ``spack.yaml`` resides:
+
+.. code-block:: console
+
+ $ spack containerize
+
+The example ``spack.yaml`` above would produce for instance the
+following ``Dockerfile``:
+
+.. code-block:: docker
+
+ # Build stage with Spack pre-installed and ready to be used
+ FROM spack/centos7:latest as builder
+
+ # What we want to install and how we want to install it
+ # is specified in a manifest file (spack.yaml)
+ RUN mkdir /opt/spack-environment \
+ && (echo "spack:" \
+ && echo " specs:" \
+ && echo " - gromacs+mpi" \
+ && echo " - mpich" \
+ && echo " concretization: together" \
+ && echo " config:" \
+ && echo " install_tree: /opt/software" \
+ && echo " view: /opt/view") > /opt/spack-environment/spack.yaml
+
+ # Install the software, remove unecessary deps
+ RUN cd /opt/spack-environment && spack install && spack gc -y
+
+ # Strip all the binaries
+ RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \
+ xargs file -i | \
+ grep 'charset=binary' | \
+ grep 'x-executable\|x-archive\|x-sharedlib' | \
+ awk -F: '{print $1}' | xargs strip -s
+
+ # Modifications to the environment that are necessary to run
+ RUN cd /opt/spack-environment && \
+ spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+
+
+ # Bare OS image to run the installed executables
+ FROM centos:7
+
+ COPY --from=builder /opt/spack-environment /opt/spack-environment
+ COPY --from=builder /opt/software /opt/software
+ COPY --from=builder /opt/view /opt/view
+ COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh
+
+ RUN yum update -y && yum install -y epel-release && yum update -y \
+ && yum install -y libgomp \
+ && rm -rf /var/cache/yum && yum clean all
+
+ RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc
+
+
+ LABEL "app"="gromacs"
+ LABEL "mpi"="mpich"
+
+ ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"]
+
+.. note::
+ Spack can also produce Singularity definition files to build the image. The
+ minimum version of Singularity required to build a SIF (Singularity Image Format)
+ from them is ``3.5.3``. \ No newline at end of file
diff --git a/lib/spack/docs/dockerhub_spack.png b/lib/spack/docs/dockerhub_spack.png
new file mode 100644
index 0000000000..44ff0ed7ed
--- /dev/null
+++ b/lib/spack/docs/dockerhub_spack.png
Binary files differ
diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst
index 336d574bd7..5ec1ec9032 100644
--- a/lib/spack/docs/environments.rst
+++ b/lib/spack/docs/environments.rst
@@ -49,6 +49,8 @@ Spack uses a "manifest and lock" model similar to `Bundler gemfiles
managers. The user input file is named ``spack.yaml`` and the lock
file is named ``spack.lock``
+.. _environments-using:
+
------------------
Using Environments
------------------
diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst
index 489c15645a..8170f152a4 100644
--- a/lib/spack/docs/index.rst
+++ b/lib/spack/docs/index.rst
@@ -66,6 +66,7 @@ or refer to the full manual below.
config_yaml
build_settings
environments
+ containers
mirrors
module_file_support
repositories
diff --git a/lib/spack/spack/cmd/containerize.py b/lib/spack/spack/cmd/containerize.py
new file mode 100644
index 0000000000..cc2c001560
--- /dev/null
+++ b/lib/spack/spack/cmd/containerize.py
@@ -0,0 +1,25 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
+import os.path
+import spack.container
+
+description = ("creates recipes to build images for different"
+ " container runtimes")
+section = "container"
+level = "long"
+
+
+def containerize(parser, args):
+ config_dir = args.env_dir or os.getcwd()
+ config_file = os.path.abspath(os.path.join(config_dir, 'spack.yaml'))
+ if not os.path.exists(config_file):
+ msg = 'file not found: {0}'
+ raise ValueError(msg.format(config_file))
+
+ config = spack.container.validate(config_file)
+
+ recipe = spack.container.recipe(config)
+ print(recipe)
diff --git a/lib/spack/spack/container/__init__.py b/lib/spack/spack/container/__init__.py
new file mode 100644
index 0000000000..fc3750355a
--- /dev/null
+++ b/lib/spack/spack/container/__init__.py
@@ -0,0 +1,81 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Package that provides functions and classes to
+generate container recipes from a Spack environment
+"""
+import warnings
+
+import spack.environment
+import spack.schema.env as env
+import spack.util.spack_yaml as syaml
+from .writers import recipe
+
+__all__ = ['validate', 'recipe']
+
+
+def validate(configuration_file):
+ """Validate a Spack environment YAML file that is being used to generate a
+ recipe for a container.
+
+ Since a few attributes of the configuration must have specific values for
+ the container recipe, this function returns a sanitized copy of the
+ configuration in the input file. If any modification is needed, a warning
+ will be issued.
+
+ Args:
+ configuration_file (str): path to the Spack environment YAML file
+
+ Returns:
+ A sanitized copy of the configuration stored in the input file
+ """
+ import jsonschema
+ with open(configuration_file) as f:
+ config = syaml.load(f)
+
+ # Ensure we have a "container" attribute with sensible defaults set
+ env_dict = spack.environment.config_dict(config)
+ env_dict.setdefault('container', {
+ 'format': 'docker',
+ 'base': {'image': 'ubuntu:18.04', 'spack': 'develop'}
+ })
+ env_dict['container'].setdefault('format', 'docker')
+ env_dict['container'].setdefault(
+ 'base', {'image': 'ubuntu:18.04', 'spack': 'develop'}
+ )
+
+ # Remove attributes that are not needed / allowed in the
+ # container recipe
+ for subsection in ('cdash', 'gitlab_ci', 'modules'):
+ if subsection in env_dict:
+ msg = ('the subsection "{0}" in "{1}" is not used when generating'
+ ' container recipes and will be discarded')
+ warnings.warn(msg.format(subsection, configuration_file))
+ env_dict.pop(subsection)
+
+ # Set the default value of the concretization strategy to "together" and
+ # warn if the user explicitly set another value
+ env_dict.setdefault('concretization', 'together')
+ if env_dict['concretization'] != 'together':
+ msg = ('the "concretization" attribute of the environment is set '
+ 'to "{0}" [the advised value is instead "together"]')
+ warnings.warn(msg.format(env_dict['concretization']))
+
+ # Check if the install tree was explicitly set to a custom value and warn
+ # that it will be overridden
+ environment_config = env_dict.get('config', {})
+ if environment_config.get('install_tree', None):
+ msg = ('the "config:install_tree" attribute has been set explicitly '
+ 'and will be overridden in the container image')
+ warnings.warn(msg)
+
+ # Likewise for the view
+ environment_view = env_dict.get('view', None)
+ if environment_view:
+ msg = ('the "view" attribute has been set explicitly '
+ 'and will be overridden in the container image')
+ warnings.warn(msg)
+
+ jsonschema.validate(config, schema=env.schema)
+ return config
diff --git a/lib/spack/spack/container/images.json b/lib/spack/spack/container/images.json
new file mode 100644
index 0000000000..ecd911815d
--- /dev/null
+++ b/lib/spack/spack/container/images.json
@@ -0,0 +1,50 @@
+{
+ "ubuntu:18.04": {
+ "update": "apt-get -yqq update && apt-get -yqq upgrade",
+ "install": "apt-get -yqq install",
+ "clean": "rm -rf /var/lib/apt/lists/*",
+ "environment": [],
+ "build": "spack/ubuntu-bionic",
+ "build_tags": {
+ "develop": "latest",
+ "0.14": "0.14",
+ "0.14.0": "0.14.0"
+ }
+ },
+ "ubuntu:16.04": {
+ "update": "apt-get -yqq update && apt-get -yqq upgrade",
+ "install": "apt-get -yqq install",
+ "clean": "rm -rf /var/lib/apt/lists/*",
+ "environment": [],
+ "build": "spack/ubuntu-xenial",
+ "build_tags": {
+ "develop": "latest",
+ "0.14": "0.14",
+ "0.14.0": "0.14.0"
+ }
+ },
+ "centos:7": {
+ "update": "yum update -y && yum install -y epel-release && yum update -y",
+ "install": "yum install -y",
+ "clean": "rm -rf /var/cache/yum && yum clean all",
+ "environment": [],
+ "build": "spack/centos7",
+ "build_tags": {
+ "develop": "latest",
+ "0.14": "0.14",
+ "0.14.0": "0.14.0"
+ }
+ },
+ "centos:6": {
+ "update": "yum update -y && yum install -y epel-release && yum update -y",
+ "install": "yum install -y",
+ "clean": "rm -rf /var/cache/yum && yum clean all",
+ "environment": [],
+ "build": "spack/centos6",
+ "build_tags": {
+ "develop": "latest",
+ "0.14": "0.14",
+ "0.14.0": "0.14.0"
+ }
+ }
+} \ No newline at end of file
diff --git a/lib/spack/spack/container/images.py b/lib/spack/spack/container/images.py
new file mode 100644
index 0000000000..421fc24425
--- /dev/null
+++ b/lib/spack/spack/container/images.py
@@ -0,0 +1,72 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Manages the details on the images used in the build and the run stage."""
+import json
+import os.path
+
+#: Global variable used to cache in memory the content of images.json
+_data = None
+
+
+def data():
+ """Returns a dictionary with the static data on the images.
+
+ The dictionary is read from a JSON file lazily the first time
+ this function is called.
+ """
+ global _data
+ if not _data:
+ json_dir = os.path.abspath(os.path.dirname(__file__))
+ json_file = os.path.join(json_dir, 'images.json')
+ with open(json_file) as f:
+ _data = json.load(f)
+ return _data
+
+
+def build_info(image, spack_version):
+ """Returns the name of the build image and its tag.
+
+ Args:
+ image (str): image to be used at run-time. Should be of the form
+ <image_name>:<image_tag> e.g. "ubuntu:18.04"
+ spack_version (str): version of Spack that we want to use to build
+
+ Returns:
+ A tuple with (image_name, image_tag) for the build image
+ """
+ # Don't handle error here, as a wrong image should have been
+ # caught by the JSON schema
+ image_data = data()[image]
+ build_image = image_data['build']
+
+ # Try to check if we have a tag for this Spack version
+ try:
+ build_tag = image_data['build_tags'][spack_version]
+ except KeyError:
+ msg = ('the image "{0}" has no tag for Spack version "{1}" '
+ '[valid versions are {2}]')
+ msg = msg.format(build_image, spack_version,
+ ', '.join(image_data['build_tags'].keys()))
+ raise ValueError(msg)
+
+ return build_image, build_tag
+
+
+def package_info(image):
+ """Returns the commands used to update system repositories, install
+ system packages and clean afterwards.
+
+ Args:
+ image (str): image to be used at run-time. Should be of the form
+ <image_name>:<image_tag> e.g. "ubuntu:18.04"
+
+ Returns:
+ A tuple of (update, install, clean) commands.
+ """
+ image_data = data()[image]
+ update = image_data['update']
+ install = image_data['install']
+ clean = image_data['clean']
+ return update, install, clean
diff --git a/lib/spack/spack/container/writers/__init__.py b/lib/spack/spack/container/writers/__init__.py
new file mode 100644
index 0000000000..a1d2fa3102
--- /dev/null
+++ b/lib/spack/spack/container/writers/__init__.py
@@ -0,0 +1,154 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Writers for different kind of recipes and related
+convenience functions.
+"""
+import collections
+import copy
+
+import spack.environment
+import spack.schema.env
+import spack.tengine as tengine
+import spack.util.spack_yaml as syaml
+
+from spack.container.images import build_info, package_info
+
+#: Caches all the writers that are currently supported
+_writer_factory = {}
+
+
+def writer(name):
+ """Decorator to register a factory for a recipe writer.
+
+ Each factory should take a configuration dictionary and return a
+ properly configured writer that, when called, prints the
+ corresponding recipe.
+ """
+ def _decorator(factory):
+ _writer_factory[name] = factory
+ return factory
+ return _decorator
+
+
+def create(configuration):
+ """Returns a writer that conforms to the configuration passed as input.
+
+ Args:
+ configuration: how to generate the current recipe
+ """
+ name = spack.environment.config_dict(configuration)['container']['format']
+ return _writer_factory[name](configuration)
+
+
+def recipe(configuration):
+ """Returns a recipe that conforms to the configuration passed as input.
+
+ Args:
+ configuration: how to generate the current recipe
+ """
+ return create(configuration)()
+
+
+class PathContext(tengine.Context):
+ """Generic context used to instantiate templates of recipes that
+ install software in a common location and make it available
+ directly via PATH.
+ """
+ def __init__(self, config):
+ self.config = spack.environment.config_dict(config)
+ self.container_config = self.config['container']
+
+ @tengine.context_property
+ def run(self):
+ """Information related to the run image."""
+ image = self.container_config['base']['image']
+ Run = collections.namedtuple('Run', ['image'])
+ return Run(image=image)
+
+ @tengine.context_property
+ def build(self):
+ """Information related to the build image."""
+
+ # Map the final image to the correct build image
+ run_image = self.container_config['base']['image']
+ spack_version = self.container_config['base']['spack']
+ image, tag = build_info(run_image, spack_version)
+
+ Build = collections.namedtuple('Build', ['image', 'tag'])
+ return Build(image=image, tag=tag)
+
+ @tengine.context_property
+ def strip(self):
+ """Whether or not to strip binaries in the image"""
+ return self.container_config.get('strip', True)
+
+ @tengine.context_property
+ def paths(self):
+ """Important paths in the image"""
+ Paths = collections.namedtuple('Paths', [
+ 'environment', 'store', 'view'
+ ])
+ return Paths(
+ environment='/opt/spack-environment',
+ store='/opt/software',
+ view='/opt/view'
+ )
+
+ @tengine.context_property
+ def manifest(self):
+ """The spack.yaml file that should be used in the image"""
+ import jsonschema
+ # Copy in the part of spack.yaml prescribed in the configuration file
+ manifest = copy.deepcopy(self.config)
+ manifest.pop('container')
+
+ # Ensure that a few paths are where they need to be
+ manifest.setdefault('config', syaml.syaml_dict())
+ manifest['config']['install_tree'] = self.paths.store
+ manifest['view'] = self.paths.view
+ manifest = {'spack': manifest}
+
+ # Validate the manifest file
+ jsonschema.validate(manifest, schema=spack.schema.env.schema)
+
+ return syaml.dump(manifest, default_flow_style=False).strip()
+
+ @tengine.context_property
+ def os_packages(self):
+ """Additional system packages that are needed at run-time."""
+ package_list = self.container_config.get('os_packages', None)
+ if not package_list:
+ return package_list
+
+ image = self.container_config['base']['image']
+ update, install, clean = package_info(image)
+ Packages = collections.namedtuple(
+ 'Packages', ['update', 'install', 'list', 'clean']
+ )
+ return Packages(update=update, install=install,
+ list=package_list, clean=clean)
+
+ @tengine.context_property
+ def extra_instructions(self):
+ Extras = collections.namedtuple('Extra', ['build', 'final'])
+ extras = self.container_config.get('extra_instructions', {})
+ build, final = extras.get('build', None), extras.get('final', None)
+ return Extras(build=build, final=final)
+
+ @tengine.context_property
+ def labels(self):
+ return self.container_config.get('labels', {})
+
+ def __call__(self):
+ """Returns the recipe as a string"""
+ env = tengine.make_environment()
+ t = env.get_template(self.template_name)
+ return t.render(**self.to_dict())
+
+
+# Import after function definition all the modules in this package,
+# so that registration of writers will happen automatically
+import spack.container.writers.singularity # noqa
+import spack.container.writers.docker # noqa
diff --git a/lib/spack/spack/container/writers/docker.py b/lib/spack/spack/container/writers/docker.py
new file mode 100644
index 0000000000..557d22c803
--- /dev/null
+++ b/lib/spack/spack/container/writers/docker.py
@@ -0,0 +1,30 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import spack.tengine as tengine
+
+from . import writer, PathContext
+
+
+@writer('docker')
+class DockerContext(PathContext):
+ """Context used to instantiate a Dockerfile"""
+ #: Name of the template used for Dockerfiles
+ template_name = 'container/Dockerfile'
+
+ @tengine.context_property
+ def manifest(self):
+ manifest_str = super(DockerContext, self).manifest
+ # Docker doesn't support HEREDOC so we need to resort to
+ # a horrible echo trick to have the manifest in the Dockerfile
+ echoed_lines = []
+ for idx, line in enumerate(manifest_str.split('\n')):
+ if idx == 0:
+ echoed_lines.append('&& (echo "' + line + '" \\')
+ continue
+ echoed_lines.append('&& echo "' + line + '" \\')
+
+ echoed_lines[-1] = echoed_lines[-1].replace(' \\', ')')
+
+ return '\n'.join(echoed_lines)
diff --git a/lib/spack/spack/container/writers/singularity.py b/lib/spack/spack/container/writers/singularity.py
new file mode 100644
index 0000000000..32f29eb83d
--- /dev/null
+++ b/lib/spack/spack/container/writers/singularity.py
@@ -0,0 +1,33 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import spack.tengine as tengine
+from . import writer, PathContext
+
+
+@writer('singularity')
+class SingularityContext(PathContext):
+ """Context used to instantiate a Singularity definition file"""
+ #: Name of the template used for Singularity definition files
+ template_name = 'container/singularity.def'
+
+ @property
+ def singularity_config(self):
+ return self.container_config.get('singularity', {})
+
+ @tengine.context_property
+ def runscript(self):
+ return self.singularity_config.get('runscript', '')
+
+ @tengine.context_property
+ def startscript(self):
+ return self.singularity_config.get('startscript', '')
+
+ @tengine.context_property
+ def test(self):
+ return self.singularity_config.get('test', '')
+
+ @tengine.context_property
+ def help(self):
+ return self.singularity_config.get('help', '')
diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py
new file mode 100644
index 0000000000..cb1ed8d63a
--- /dev/null
+++ b/lib/spack/spack/schema/container.py
@@ -0,0 +1,82 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Schema for the 'container' subsection of Spack environments."""
+
+#: Schema for the container attribute included in Spack environments
+container_schema = {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'properties': {
+ # The recipe formats that are currently supported by the command
+ 'format': {
+ 'type': 'string',
+ 'enum': ['docker', 'singularity']
+ },
+ # Describes the base image to start from and the version
+ # of Spack to be used
+ 'base': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'properties': {
+ 'image': {
+ 'type': 'string',
+ 'enum': ['ubuntu:18.04',
+ 'ubuntu:16.04',
+ 'centos:7',
+ 'centos:6']
+ },
+ 'spack': {
+ 'type': 'string',
+ 'enum': ['develop', '0.14', '0.14.0']
+ }
+ },
+ 'required': ['image', 'spack']
+ },
+ # Whether or not to strip installed binaries
+ 'strip': {
+ 'type': 'boolean',
+ 'default': True
+ },
+ # Additional system packages that are needed at runtime
+ 'os_packages': {
+ 'type': 'array',
+ 'items': {
+ 'type': 'string'
+ }
+ },
+ # Add labels to the image
+ 'labels': {
+ 'type': 'object',
+ },
+ # Add a custom extra section at the bottom of a stage
+ 'extra_instructions': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'properties': {
+ 'build': {'type': 'string'},
+ 'final': {'type': 'string'}
+ }
+ },
+ # Reserved for properties that are specific to each format
+ 'singularity': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'default': {},
+ 'properties': {
+ 'runscript': {'type': 'string'},
+ 'startscript': {'type': 'string'},
+ 'test': {'type': 'string'},
+ 'help': {'type': 'string'}
+ }
+ },
+ 'docker': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'default': {},
+ }
+ }
+}
+
+properties = {'container': container_schema}
diff --git a/lib/spack/spack/schema/merged.py b/lib/spack/spack/schema/merged.py
index d56228c116..e118acf286 100644
--- a/lib/spack/spack/schema/merged.py
+++ b/lib/spack/spack/schema/merged.py
@@ -13,6 +13,7 @@ from llnl.util.lang import union_dicts
import spack.schema.cdash
import spack.schema.compilers
import spack.schema.config
+import spack.schema.container
import spack.schema.gitlab_ci
import spack.schema.mirrors
import spack.schema.modules
@@ -26,6 +27,7 @@ properties = union_dicts(
spack.schema.cdash.properties,
spack.schema.compilers.properties,
spack.schema.config.properties,
+ spack.schema.container.properties,
spack.schema.gitlab_ci.properties,
spack.schema.mirrors.properties,
spack.schema.modules.properties,
diff --git a/lib/spack/spack/test/cmd/gc.py b/lib/spack/spack/test/cmd/gc.py
index 76eb608cf2..22c85a1d78 100644
--- a/lib/spack/spack/test/cmd/gc.py
+++ b/lib/spack/spack/test/cmd/gc.py
@@ -30,7 +30,9 @@ def test_packages_are_removed(config, mutable_database, capsys):
@pytest.mark.db
-def test_gc_with_environment(config, mutable_database, capsys):
+def test_gc_with_environment(
+ config, mutable_database, mutable_mock_env_path, capsys
+):
s = spack.spec.Spec('simple-inheritance')
s.concretize()
s.package.do_install(fake=True, explicit=True)
diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py
index 3595f91953..9a64209cfa 100644
--- a/lib/spack/spack/test/cmd/test.py
+++ b/lib/spack/spack/test/cmd/test.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
diff --git a/lib/spack/spack/test/container/cli.py b/lib/spack/spack/test/container/cli.py
new file mode 100644
index 0000000000..8e5403f072
--- /dev/null
+++ b/lib/spack/spack/test/container/cli.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import llnl.util.filesystem as fs
+import spack.main
+
+
+containerize = spack.main.SpackCommand('containerize')
+
+
+def test_command(configuration_dir, capsys):
+ with capsys.disabled():
+ with fs.working_dir(configuration_dir):
+ output = containerize()
+ assert 'FROM spack/ubuntu-bionic' in output
diff --git a/lib/spack/spack/test/container/conftest.py b/lib/spack/spack/test/container/conftest.py
new file mode 100644
index 0000000000..802b34c5f8
--- /dev/null
+++ b/lib/spack/spack/test/container/conftest.py
@@ -0,0 +1,43 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import pytest
+
+import spack.util.spack_yaml as syaml
+
+
+@pytest.fixture()
+def minimal_configuration():
+ return {
+ 'spack': {
+ 'specs': [
+ 'gromacs',
+ 'mpich',
+ 'fftw precision=float'
+ ],
+ 'container': {
+ 'format': 'docker',
+ 'base': {
+ 'image': 'ubuntu:18.04',
+ 'spack': 'develop'
+ }
+ }
+ }
+ }
+
+
+@pytest.fixture()
+def config_dumper(tmpdir):
+ """Function that dumps an environment config in a temporary folder."""
+ def dumper(configuration):
+ content = syaml.dump(configuration, default_flow_style=False)
+ config_file = tmpdir / 'spack.yaml'
+ config_file.write(content)
+ return str(tmpdir)
+ return dumper
+
+
+@pytest.fixture()
+def configuration_dir(minimal_configuration, config_dumper):
+ return config_dumper(minimal_configuration)
diff --git a/lib/spack/spack/test/container/docker.py b/lib/spack/spack/test/container/docker.py
new file mode 100644
index 0000000000..fbdc085828
--- /dev/null
+++ b/lib/spack/spack/test/container/docker.py
@@ -0,0 +1,74 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import spack.container.writers as writers
+
+
+def test_manifest(minimal_configuration):
+ writer = writers.create(minimal_configuration)
+ manifest_str = writer.manifest
+ for line in manifest_str.split('\n'):
+ assert 'echo' in line
+
+
+def test_build_and_run_images(minimal_configuration):
+ writer = writers.create(minimal_configuration)
+
+ # Test the output of run property
+ run = writer.run
+ assert run.image == 'ubuntu:18.04'
+
+ # Test the output of the build property
+ build = writer.build
+ assert build.image == 'spack/ubuntu-bionic'
+ assert build.tag == 'latest'
+
+
+def test_packages(minimal_configuration):
+ # In this minimal configuration we don't have packages
+ writer = writers.create(minimal_configuration)
+ assert writer.os_packages is None
+
+ # If we add them a list should be returned
+ pkgs = ['libgomp1']
+ minimal_configuration['spack']['container']['os_packages'] = pkgs
+ writer = writers.create(minimal_configuration)
+ p = writer.os_packages
+ assert p.update
+ assert p.install
+ assert p.clean
+ assert p.list == pkgs
+
+
+def test_ensure_render_works(minimal_configuration):
+ # Here we just want to ensure that nothing is raised
+ writer = writers.create(minimal_configuration)
+ writer()
+
+
+def test_strip_is_set_from_config(minimal_configuration):
+ writer = writers.create(minimal_configuration)
+ assert writer.strip is True
+
+ minimal_configuration['spack']['container']['strip'] = False
+ writer = writers.create(minimal_configuration)
+ assert writer.strip is False
+
+
+def test_extra_instructions_is_set_from_config(minimal_configuration):
+ writer = writers.create(minimal_configuration)
+ assert writer.extra_instructions == (None, None)
+
+ test_line = 'RUN echo Hello world!'
+ e = minimal_configuration['spack']['container']
+ e['extra_instructions'] = {}
+ e['extra_instructions']['build'] = test_line
+ writer = writers.create(minimal_configuration)
+ assert writer.extra_instructions == (test_line, None)
+
+ e['extra_instructions']['final'] = test_line
+ del e['extra_instructions']['build']
+ writer = writers.create(minimal_configuration)
+ assert writer.extra_instructions == (None, test_line)
diff --git a/lib/spack/spack/test/container/images.py b/lib/spack/spack/test/container/images.py
new file mode 100644
index 0000000000..808676c39a
--- /dev/null
+++ b/lib/spack/spack/test/container/images.py
@@ -0,0 +1,58 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os.path
+
+import pytest
+
+import spack.container
+
+
+@pytest.mark.parametrize('image,spack_version,expected', [
+ ('ubuntu:18.04', 'develop', ('spack/ubuntu-bionic', 'latest')),
+ ('ubuntu:18.04', '0.14.0', ('spack/ubuntu-bionic', '0.14.0')),
+])
+def test_build_info(image, spack_version, expected):
+ output = spack.container.images.build_info(image, spack_version)
+ assert output == expected
+
+
+@pytest.mark.parametrize('image,spack_version', [
+ ('ubuntu:18.04', 'doesnotexist')
+])
+def test_build_info_error(image, spack_version):
+ with pytest.raises(ValueError, match=r"has no tag for"):
+ spack.container.images.build_info(image, spack_version)
+
+
+@pytest.mark.parametrize('image', [
+ 'ubuntu:18.04'
+])
+def test_package_info(image):
+ update, install, clean = spack.container.images.package_info(image)
+ assert update
+ assert install
+ assert clean
+
+
+@pytest.mark.parametrize('extra_config,expected_msg', [
+ ({'modules': {'enable': ['tcl']}}, 'the subsection "modules" in'),
+ ({'concretization': 'separately'}, 'the "concretization" attribute'),
+ ({'config': {'install_tree': '/some/dir'}},
+ 'the "config:install_tree" attribute has been set'),
+ ({'view': '/some/dir'}, 'the "view" attribute has been set')
+])
+def test_validate(
+ extra_config, expected_msg, minimal_configuration, config_dumper
+):
+ minimal_configuration['spack'].update(extra_config)
+ spack_yaml_dir = config_dumper(minimal_configuration)
+ spack_yaml = os.path.join(spack_yaml_dir, 'spack.yaml')
+
+ with pytest.warns(UserWarning) as w:
+ spack.container.validate(spack_yaml)
+
+ # Tests are designed to raise only one warning
+ assert len(w) == 1
+ assert expected_msg in str(w.pop().message)
diff --git a/lib/spack/spack/test/container/schema.py b/lib/spack/spack/test/container/schema.py
new file mode 100644
index 0000000000..3f33a3f9f7
--- /dev/null
+++ b/lib/spack/spack/test/container/schema.py
@@ -0,0 +1,16 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import spack.container
+import spack.schema.container
+
+
+def test_images_in_schema():
+ properties = spack.schema.container.container_schema['properties']
+ allowed_images = set(
+ properties['base']['properties']['image']['enum']
+ )
+ images_in_json = set(x for x in spack.container.images.data())
+ assert images_in_json == allowed_images
diff --git a/lib/spack/spack/test/container/singularity.py b/lib/spack/spack/test/container/singularity.py
new file mode 100644
index 0000000000..445a119f6c
--- /dev/null
+++ b/lib/spack/spack/test/container/singularity.py
@@ -0,0 +1,42 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import pytest
+
+import spack.container.writers as writers
+
+
+@pytest.fixture
+def singularity_configuration(minimal_configuration):
+ minimal_configuration['spack']['container']['format'] = 'singularity'
+ return minimal_configuration
+
+
+def test_ensure_render_works(singularity_configuration):
+ container_config = singularity_configuration['spack']['container']
+ assert container_config['format'] == 'singularity'
+ # Here we just want to ensure that nothing is raised
+ writer = writers.create(singularity_configuration)
+ writer()
+
+
+@pytest.mark.parametrize('properties,expected', [
+ ({'runscript': '/opt/view/bin/h5ls'},
+ {'runscript': '/opt/view/bin/h5ls',
+ 'startscript': '',
+ 'test': '',
+ 'help': ''})
+])
+def test_singularity_specific_properties(
+ properties, expected, singularity_configuration
+):
+ # Set the property in the configuration
+ container_config = singularity_configuration['spack']['container']
+ for name, value in properties.items():
+ container_config.setdefault('singularity', {})[name] = value
+
+ # Assert the properties return the expected values
+ writer = writers.create(singularity_configuration)
+ for name, value in expected.items():
+ assert getattr(writer, name) == value