From aa1c814c7592c9e71f564181af3a8f2a115bd304 Mon Sep 17 00:00:00 2001 From: Omar Padron Date: Fri, 26 Oct 2018 13:15:05 -0400 Subject: docker: unite Dockerfiles; auto-deploy images to DockerHub (#9329) * Unite Dockerfiles - add build/run/push scripts * update docker documentation * update .travis.yml * switch to using a preprocessor on Dockerfiles * skip building docker images on pull requests * update files with copyright info * tweak when travis builds for docker files are done --- share/spack/docker/.env | 1 - share/spack/docker/Dockerfile | 132 ++++++++++++++++++ share/spack/docker/build-image.sh | 84 +++++++++++ share/spack/docker/build/arch.dockerfile | 59 -------- share/spack/docker/build/centos.dockerfile | 57 -------- share/spack/docker/build/common/handle-prompt.sh | 165 ---------------------- share/spack/docker/build/common/handle-ssh.sh | 41 ------ share/spack/docker/build/common/modules.yaml | 6 - share/spack/docker/build/fedora.dockerfile | 56 -------- share/spack/docker/build/opensuse.dockerfile | 65 --------- share/spack/docker/build/scilinux.dockerfile | 62 --------- share/spack/docker/build/ubuntu.dockerfile | 51 ------- share/spack/docker/config/arch.bash | 18 +++ share/spack/docker/config/centos.bash | 16 +++ share/spack/docker/config/fedora.bash | 16 +++ share/spack/docker/config/opensuse.bash | 16 +++ share/spack/docker/config/scilinux.bash | 17 +++ share/spack/docker/config/ubuntu.bash | 15 ++ share/spack/docker/docker-compose.yml | 56 -------- share/spack/docker/dpp.bash | 77 +++++++++++ share/spack/docker/entrypoint.bash | 43 ++++++ share/spack/docker/handle-prompt.sh | 169 +++++++++++++++++++++++ share/spack/docker/handle-ssh.sh | 41 ++++++ share/spack/docker/modules.yaml | 6 + share/spack/docker/push-image.sh | 1 + share/spack/docker/render-image-template.sh | 1 + share/spack/docker/run-image.sh | 1 + 27 files changed, 653 insertions(+), 619 deletions(-) delete mode 100644 share/spack/docker/.env create mode 100644 share/spack/docker/Dockerfile create mode 100755 share/spack/docker/build-image.sh delete mode 100644 share/spack/docker/build/arch.dockerfile delete mode 100644 share/spack/docker/build/centos.dockerfile delete mode 100644 share/spack/docker/build/common/handle-prompt.sh delete mode 100644 share/spack/docker/build/common/handle-ssh.sh delete mode 100644 share/spack/docker/build/common/modules.yaml delete mode 100644 share/spack/docker/build/fedora.dockerfile delete mode 100644 share/spack/docker/build/opensuse.dockerfile delete mode 100644 share/spack/docker/build/scilinux.dockerfile delete mode 100644 share/spack/docker/build/ubuntu.dockerfile create mode 100644 share/spack/docker/config/arch.bash create mode 100644 share/spack/docker/config/centos.bash create mode 100644 share/spack/docker/config/fedora.bash create mode 100644 share/spack/docker/config/opensuse.bash create mode 100644 share/spack/docker/config/scilinux.bash create mode 100644 share/spack/docker/config/ubuntu.bash delete mode 100644 share/spack/docker/docker-compose.yml create mode 100755 share/spack/docker/dpp.bash create mode 100755 share/spack/docker/entrypoint.bash create mode 100644 share/spack/docker/handle-prompt.sh create mode 100644 share/spack/docker/handle-ssh.sh create mode 100644 share/spack/docker/modules.yaml create mode 120000 share/spack/docker/push-image.sh create mode 120000 share/spack/docker/render-image-template.sh create mode 120000 share/spack/docker/run-image.sh (limited to 'share') diff --git a/share/spack/docker/.env b/share/spack/docker/.env deleted file mode 100644 index 924ceb3f73..0000000000 --- a/share/spack/docker/.env +++ /dev/null @@ -1 +0,0 @@ -COMPOSE_PROJECT_NAME=spack diff --git a/share/spack/docker/Dockerfile b/share/spack/docker/Dockerfile new file mode 100644 index 0000000000..0c3b49f366 --- /dev/null +++ b/share/spack/docker/Dockerfile @@ -0,0 +1,132 @@ +ARG BASE + +FROM $BASE +MAINTAINER Spack Maintainers + +ARG BASE +ARG DISTRO +ARG DISTRO_VERSION + +ENV DOCKERFILE_BASE=$BASE \ + DOCKERFILE_DISTRO=$DISTRO \ + DOCKERFILE_DISTRO_VERSION=$DISTRO_VERSION \ + SPACK_ROOT=/spack \ + FORCE_UNSAFE_CONFIGURE=1 \ + DEBIAN_FRONTEND=noninteractive \ + container=docker + +COPY bin $SPACK_ROOT/bin +COPY etc $SPACK_ROOT/etc +COPY lib $SPACK_ROOT/lib +COPY share $SPACK_ROOT/share +COPY var $SPACK_ROOT/var +RUN mkdir -p $SPACK_ROOT/opt/spack + +MASK PUSH +MASK [[ $DISTRO == arch ]] +RUN pacman -Sy --noconfirm \ + base-devel ca-certificates curl gcc \ + gcc-fortran git gnupg2 iproute2 \ + make openssh python python-pip \ + sudo tcl \ + && echo 'nobody ALL=(ALL) NOPASSWD: ALL' > \ + /etc/sudoers.d/nobody-sudo \ + && sudo -u nobody git clone --depth 1 \ + https://aur.archlinux.org/lua-posix.git /tmp/lua-posix \ + && sudo -u nobody git clone --depth 1 \ + https://aur.archlinux.org/lmod.git /tmp/lmod \ + && ( cd /tmp/lua-posix \ + && sudo -u nobody makepkg -si --asdeps --noconfirm ) \ + && ( cd /tmp/lmod \ + && sudo -u nobody makepkg -si --noconfirm ) \ + && rm -rf /tmp/lua-posix /tmp/lmod /etc/sudoers.d/nobody-sudo + +MASK [[ $DISTRO =~ (centos|rhel.*) ]] +RUN yum update -y + + MASK PUSH + MASK [[ $DISTRO =~ rhel.* ]] + RUN yum install -y yum-conf-repos.noarch \ + && yum update -y + MASK POP + +RUN yum install -y epel-release \ + && yum update -y \ + && yum --enablerepo epel groupinstall -y "Development Tools" \ + && yum --enablerepo epel install -y \ + curl findutils gcc-c++ gcc \ + gcc-gfortran git gnupg2 hostname \ + iproute Lmod make patch \ + openssh-server python python-pip tcl \ + && rm -rf /var/cache/yum \ + && yum clean all + +MASK [[ $DISTRO == fedora ]] +RUN dnf update -y \ + && dnf group install -y "C Development Tools and Libraries" \ + && dnf install -y \ + @development-tools \ + curl findutils gcc-c++ gcc \ + gcc-gfortran git gnupg2 hostname \ + iproute Lmod make patch \ + openssh-server python tcl \ + && dnf clean all + +MASK [[ $DISTRO == opensuse ]] +RUN zypper -n ref \ + && zypper -n up --skip-interactive --no-recommends \ + && zypper -n install -l --no-recommends --type pattern \ + devel_basis devel_C_C++ \ + && zypper -n install -l --no-recommends \ + bash bash-completion ca-certificates curl \ + findutils gcc gcc-locale gcc-c++ \ + gcc-fortran git glibc-locale gpg2 \ + hostname iproute lua-lmod make \ + patch openssh python python-pip \ + python-xml tcl \ + && zypper clean \ + && rm -rf /var/cache/zypp/* + +MASK [[ $DISTRO == ubuntu ]] +RUN apt-get -yqq update \ + && apt-get -yqq install \ + build-essential ca-certificates curl g++ \ + gcc gfortran git gnupg2 \ + iproute2 lmod lua-posix make \ + openssh-server python python-pip tcl + + MASK PUSH + MASK [[ $DISTRO_VERSION == bionic ]] + # [WORKAROUND] + # https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 + RUN ln -s posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so + MASK POP + +RUN rm -rf /var/lib/apt/lists/* + +MASK POP + +RUN rm -rf $SPACK_ROOT/.git \ + && pip install boto3 \ + && ( echo ". /usr/share/lmod/lmod/init/bash" \ + && echo ". $SPACK_ROOT/share/spack/setup-env.sh" \ + && echo ". $SPACK_ROOT/share/spack/spack-completion.bash" ) \ + >> /etc/profile.d/spack.sh \ + && ln -s $SPACK_ROOT/share/spack/docker/handle-ssh.sh \ + /etc/profile.d/handle-ssh.sh \ + && ln -s $SPACK_ROOT/share/spack/docker/handle-prompt.sh \ + /etc/profile.d/handle-prompt.sh \ + && mkdir -p /root/.spack \ + && cp $SPACK_ROOT/share/spack/docker/modules.yaml \ + /root/.spack/modules.yaml \ + && rm -rf /root/*.* + +MASK PUSH +MASK [[ $DISTRO_VERSION =~ (centos|fedora|opensuse|rhel.*) ]] +RUN rm -f /run/nologin +MASK POP + +WORKDIR /root +ENTRYPOINT ["bash", "/spack/share/spack/docker/entrypoint.bash"] +CMD ["docker-shell"] + diff --git a/share/spack/docker/build-image.sh b/share/spack/docker/build-image.sh new file mode 100755 index 0000000000..fb12bdb833 --- /dev/null +++ b/share/spack/docker/build-image.sh @@ -0,0 +1,84 @@ +#! /usr/bin/env bash +# +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +script="$( basename "$0" )" +cd "$( dirname "$0" )" + +if [ -z "$BASE_IMAGE" ] ; then + BASE_IMAGE="ubuntu" +fi + +if [ -z "$BASE_TAG" ] ; then + BASE_TAG="latest" +fi + +if [ -z "$DISTRO" ] ; then + DISTRO="${BASE_IMAGE}" +fi + +if [ -z "$DISTRO_VERSION" ] ; then + DISTRO_VERSION="${BASE_TAG}" +fi + +if [ -z "$BASE_NAME" ] ; then + BASE_NAME="${DISTRO}" +fi + +if [ "$BASE_TAG" '=' 'latest' ] ; then + BASE_TAG="" +fi + +if [ -n "$BASE_TAG" ] ; then + BASE_TAG=":${BASE_TAG}" +fi + +TAG="spack/${BASE_NAME}${BASE_TAG}" + +export BASE_IMAGE BASE_TAG DISTRO DISTRO_VERSION BASE_NAME TAG + +if [ "$script" '=' 'run-image.sh' ] ; then + com="docker run --rm -ti" + + if [ -z "$DISABLE_MOUNT" ] ; then + DISABLE_MOUNT=1 + if [ -z "$*" ] ; then + DISABLE_MOUNT=0 + fi + fi + + if [ "$DISABLE_MOUNT" '==' 0 ] ; then + com="${com} -v \"$( readlink -f ../../.. ):/spack\"" + fi + + eval "exec ${com}" "${TAG}" "$@" +elif [ "$script" '=' 'render-image-template.sh' ] ; then + ./dpp.bash Dockerfile +elif [ "$script" '=' 'push-image.sh' ] ; then + docker push "${TAG}" + for tag in ${EXTRA_TAGS} ; do + docker push "spack/${BASE_NAME}:${tag}" + done +else + tag_options="-t ${TAG}" + for tag in ${EXTRA_TAGS} ; do + tag_options="${tag_options} -t spack/${BASE_NAME}:${tag}" + done + + cache_options="" + if docker pull "${TAG}" ; then + cache_options="--cache-from ${TAG}" + fi + + exec ./render-image-template.sh | + docker build -f - \ + ${cache_options} \ + ${tag_options} \ + --build-arg BASE="${BASE_IMAGE}${BASE_TAG}" \ + --build-arg DISTRO="${DISTRO}" \ + --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ + ../../.. +fi diff --git a/share/spack/docker/build/arch.dockerfile b/share/spack/docker/build/arch.dockerfile deleted file mode 100644 index 6aa9ac0118..0000000000 --- a/share/spack/docker/build/arch.dockerfile +++ /dev/null @@ -1,59 +0,0 @@ -FROM base/archlinux -MAINTAINER Omar Padron - -ENV SPACK_ROOT=/spack \ - FORCE_UNSAFE_CONFIGURE=1 \ - DISTRO=arch - -RUN pacman -Sy --noconfirm \ - base-devel \ - ca-certificates \ - curl \ - gcc \ - gcc-fortran \ - git \ - gnupg2 \ - iproute2 \ - make \ - openssh \ - python \ - sudo \ - tcl && \ - git clone --depth 1 git://github.com/spack/spack.git /spack && \ - echo 'nobody ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nobody-sudo && \ - sudo -u nobody git clone --depth 1 \ - https://aur.archlinux.org/lua-posix.git /tmp/lua-posix && \ - sudo -u nobody git clone --depth 1 \ - https://aur.archlinux.org/lmod.git /tmp/lmod && \ - ( cd /tmp/lua-posix ; sudo -u nobody makepkg -si --asdeps --noconfirm ) && \ - ( cd /tmp/lmod ; sudo -u nobody makepkg -si --noconfirm ) && \ - rm -rf /tmp/lua-posix /tmp/lmod /spack/.git /etc/sudoers.d/nobody-sudo - -RUN ( cd /usr/share/lmod ; ln -s $( ls -d ./* | head -n 1 ) ./lmod ) - -RUN echo "source /usr/share/lmod/lmod/init/bash" \ - > /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/setup-env.sh" \ - >> /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/spack-completion.bash" \ - >> /etc/profile.d/spack.sh - -COPY common/handle-ssh.sh /etc/profile.d/handle-ssh.sh -COPY common/handle-prompt.sh /etc/profile.d/handle-prompt.sh.source - -RUN ( \ - echo "export DISTRO=$DISTRO" ; \ - echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ - echo "then" ; \ - echo "source /etc/profile.d/handle-prompt.sh.source" ; \ - echo "fi" ; \ -) > /etc/profile.d/handle-prompt.sh - -RUN mkdir -p /root/.spack -COPY common/modules.yaml /root/.spack/modules.yaml - -RUN rm -rf /root/*.* - -WORKDIR /root -ENTRYPOINT ["bash"] -CMD ["-l"] diff --git a/share/spack/docker/build/centos.dockerfile b/share/spack/docker/build/centos.dockerfile deleted file mode 100644 index 20c24b038f..0000000000 --- a/share/spack/docker/build/centos.dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -FROM centos -MAINTAINER Omar Padron - -ENV SPACK_ROOT=/spack \ - FORCE_UNSAFE_CONFIGURE=1 \ - DISTRO=centos - -RUN yum update -y && \ - yum install -y epel-release && \ - yum update -y && \ - yum groupinstall -y "Development Tools" && \ - yum install -y \ - curl \ - findutils \ - gcc-c++ \ - gcc \ - gcc-gfortran \ - git \ - gnupg2 \ - hostname \ - iproute \ - Lmod \ - make \ - patch \ - openssh-server \ - python \ - tcl && \ - git clone --depth 1 git://github.com/spack/spack.git /spack && \ - rm -rf /spack/.git /var/cache/yum && yum clean all - -RUN echo "source /usr/share/lmod/lmod/init/bash" \ - > /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/setup-env.sh" \ - >> /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/spack-completion.bash" \ - >> /etc/profile.d/spack.sh -COPY common/handle-ssh.sh /etc/profile.d/handle-ssh.sh -COPY common/handle-prompt.sh /etc/profile.d/handle-prompt.sh.source - -RUN ( \ - echo "export DISTRO=$DISTRO" ; \ - echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ - echo "then" ; \ - echo "source /etc/profile.d/handle-prompt.sh.source" ; \ - echo "fi" ; \ -) > /etc/profile.d/handle-prompt.sh - -RUN mkdir -p /root/.spack -COPY common/modules.yaml /root/.spack/modules.yaml - -RUN rm -f /run/nologin - -RUN rm -rf /root/*.* - -WORKDIR /root -ENTRYPOINT ["bash"] -CMD ["-l"] diff --git a/share/spack/docker/build/common/handle-prompt.sh b/share/spack/docker/build/common/handle-prompt.sh deleted file mode 100644 index 5a4910e3fa..0000000000 --- a/share/spack/docker/build/common/handle-prompt.sh +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -__tmp="`mktemp -d`" - -__trylock() { - local dir - dir="$__tmp/$1.lock" - mkdir "$dir" &>/dev/null - return $? -} - -__queue_init() { - local r - local w - - mkdir "$__tmp/$1.read.lock" ; r=$? - mkdir "$__tmp/$1.write.lock" ; w=$? - - if [ "$r" '=' '0' -a "$w" '=' '0' ] ; then - return 0 - else - return 1 - fi -} - -__queue_try_read() { - __trylock "$1.read" - return $? -} - -__queue_try_write() { - __trylock "$1.write" - return $? -} - -__queue_make_readable() { - rm -r "$__tmp/$1.read.lock" &>/dev/null - return $? -} - -__queue_make_writable() { - rm -r "$__tmp/$1.write.lock" &>/dev/null - return $? -} - -__read() { - cat "$__tmp/$1" 2> /dev/null - return $? -} - -__write() { - cat > "$__tmp/$1" 2> /dev/null - return $? -} - -__revparse_head() { - head="`git -C /spack rev-parse $@ HEAD 2>/dev/null`" - result="$?" - if [ "$result" '!=' '0' ] ; then - head="`git --git-dir=/spack/.git \\ - --work-tree=/spack rev-parse $@ HEAD 2>/dev/null`" - result="$?" - fi - - echo "$head" - return $result -} - -__git_head() { - head="`__revparse_head --abbrev-ref`" - if [ "$?" '=' '0' ] ; then - if [ "$head" '=' 'HEAD' ] ; then - head="`__revparse_head | cut -c1-8`..." - fi - - echo "$head" - fi -} - -__update_prompt() { - local prompt - prompt='' - linux_distro="$DISTRO" - if [ -n "$linux_distro" ] ; then - linux_distro='\[\e[1;34m\][\[\e[0;34m\]'"$linux_distro"'\[\e[1;34m\]]' - if [ -n "$prompt" ] ; then - prompt="$prompt " - fi - prompt="$prompt$linux_distro" - fi - - git_head="`__git_head`" - - if [ -n "$git_head" ] ; then - git_head='\[\e[1;32m\](\[\e[0;32m\]'"$git_head"'\[\e[1;32m\])' - if [ -n "$prompt" ] ; then - prompt="$prompt " - fi - prompt="$prompt$git_head" - fi - - if [ -n "$prompt" ] ; then - prompt="$prompt " - fi - prompt="$prompt"'\[\e[0;m\]\W: ' - echo "$prompt" | __write prompt -} - -set -m -( - __queue_init query - __queue_init prompt - - __update_prompt - __queue_make_readable prompt - - __queue_make_writable query - - while sleep 0.010 ; do - last_q_time='' - - while sleep 0.010 ; do - q_time="`date +%s%N`" - if __queue_try_read query ; then - last_q_time="$q_time" - __queue_make_writable query - fi - - if [ -n "$last_q_time" -a \ - "$(( (q_time - last_q_time)/10000000 > 100 ))" '=' '1' ] ; then - break - fi - done - - __update_prompt - __queue_make_readable prompt - done -) &>/dev/null & -set +m - -__update_prompt_main_first_call=1 -__update_prompt_main() { - if [ "$__update_prompt_main_first_call" '=' '1' ] ; then - while sleep 0.001 ; do - if __queue_try_read prompt ; then - PS1="`__read prompt`" - break - fi - done - __update_prompt_main_first_call=0 - else - if __queue_try_read prompt ; then - PS1="`__read prompt`" - fi - fi - - if __queue_try_write query ; then - __queue_make_readable query - fi -} - -PROMPT_COMMAND=__update_prompt_main diff --git a/share/spack/docker/build/common/handle-ssh.sh b/share/spack/docker/build/common/handle-ssh.sh deleted file mode 100644 index f2139897cb..0000000000 --- a/share/spack/docker/build/common/handle-ssh.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -uid="`id -u`" -if [ "$uid" '=' '0' ] ; then - for key_type in dsa ecdsa ed25519 rsa ; do - private_key_file="/etc/ssh/ssh_host_${key_type}_key" - public_key_file="$private_key_file.pub" - - if [ '!' -f "$private_key_file" ] ; then - ssh-keygen \ - -q -t "$key_type" -N "" -f "$private_key_file" - chmod 600 "$private_key_file" - chmod 644 "$public_key_file" - fi - done - - mkdir -p /var/run/sshd - - pgrep -u 0 -U 0 sshd &> /dev/null - if [ '!' "$?" '=' '0' ] ; then - nohup /usr/sbin/sshd -f /etc/ssh/sshd_config < /dev/null &> /dev/null - fi -fi - -if [ '!' -f "$HOME/.ssh/id_rsa" ] ; then - ssh-keygen \ - -t rsa -C "spack.developer@docker.host" -N "" -f "$HOME/.ssh/id_rsa" - cat "$HOME/.ssh/id_rsa.pub" >> "$HOME/.ssh/authorized_keys" - chmod 600 "$HOME/.ssh/authorized_keys" - - docker_ip="`ip address show dev eth0 | - grep inet | - cut -d' ' -f 6 | - cut -d/ -f 1`" - - ssh-keyscan -t rsa 127.0.0.1 localhost "$docker_ip" "`hostname`" \ - > "$HOME/.ssh/known_hosts" 2> /dev/null -fi diff --git a/share/spack/docker/build/common/modules.yaml b/share/spack/docker/build/common/modules.yaml deleted file mode 100644 index 02907ecc40..0000000000 --- a/share/spack/docker/build/common/modules.yaml +++ /dev/null @@ -1,6 +0,0 @@ -modules: - enable: - - lmod - lmod: - core_compilers: - - gcc diff --git a/share/spack/docker/build/fedora.dockerfile b/share/spack/docker/build/fedora.dockerfile deleted file mode 100644 index bf06411d21..0000000000 --- a/share/spack/docker/build/fedora.dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -FROM fedora:24 -MAINTAINER Omar Padron - -ENV SPACK_ROOT=/spack \ - FORCE_UNSAFE_CONFIGURE=1 \ - DISTRO=fedora - -RUN dnf update -y && \ - dnf group install -y "C Development Tools and Libraries" && \ - dnf install -y \ - @development-tools \ - curl \ - findutils \ - gcc-c++ \ - gcc \ - gcc-gfortran \ - git \ - gnupg2 \ - hostname \ - iproute \ - Lmod \ - make \ - patch \ - openssh-server \ - python \ - tcl && \ - git clone --depth 1 git://github.com/spack/spack.git /spack && \ - rm -rf /spack/.git && dnf clean all - -RUN echo "source /usr/share/lmod/lmod/init/bash" \ - > /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/setup-env.sh" \ - >> /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/spack-completion.bash" \ - >> /etc/profile.d/spack.sh -COPY common/handle-ssh.sh /etc/profile.d/handle-ssh.sh -COPY common/handle-prompt.sh /etc/profile.d/handle-prompt.sh.source - -RUN ( \ - echo "export DISTRO=$DISTRO" ; \ - echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ - echo "then" ; \ - echo "source /etc/profile.d/handle-prompt.sh.source" ; \ - echo "fi" ; \ -) > /etc/profile.d/handle-prompt.sh - -RUN mkdir -p /root/.spack -COPY common/modules.yaml /root/.spack/modules.yaml - -RUN rm -f /run/nologin - -RUN rm -rf /root/*.* - -WORKDIR /root -ENTRYPOINT ["bash"] -CMD ["-l"] diff --git a/share/spack/docker/build/opensuse.dockerfile b/share/spack/docker/build/opensuse.dockerfile deleted file mode 100644 index 46189a41c9..0000000000 --- a/share/spack/docker/build/opensuse.dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -FROM opensuse -MAINTAINER Omar Padron - -ENV SPACK_ROOT=/spack \ - FORCE_UNSAFE_CONFIGURE=1 \ - DISTRO=opensuse - -RUN zypper -n ref && \ - zypper -n up --skip-interactive --no-recommends && \ - zypper -n install -l --no-recommends --type pattern \ - devel_basis \ - devel_C_C++ && \ - zypper -n install -l --no-recommends \ - bash \ - bash-completion \ - ca-certificates \ - curl \ - findutils \ - gcc \ - gcc-locale \ - gcc-c++ \ - gcc-fortran \ - git \ - glibc-locale \ - gpg2 \ - hostname \ - iproute \ - lua-lmod \ - make \ - patch \ - openssh \ - python \ - python-xml \ - tcl && \ - git clone --depth 1 git://github.com/spack/spack.git /spack && \ - zypper clean && \ - rm -rf /spack/.git /var/cache/zypp/* - -RUN echo "source /usr/share/lmod/lmod/init/bash" \ - > /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/setup-env.sh" \ - >> /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/spack-completion.bash" \ - >> /etc/profile.d/spack.sh -COPY common/handle-ssh.sh /etc/profile.d/handle-ssh.sh -COPY common/handle-prompt.sh /etc/profile.d/handle-prompt.sh.source - -RUN ( \ - echo "export DISTRO=$DISTRO" ; \ - echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ - echo "then" ; \ - echo "source /etc/profile.d/handle-prompt.sh.source" ; \ - echo "fi" ; \ -) > /etc/profile.d/handle-prompt.sh - -RUN mkdir -p /root/.spack -COPY common/modules.yaml /root/.spack/modules.yaml - -RUN rm -f /run/nologin - -RUN rm -rf /root/*.* - -WORKDIR /root -ENTRYPOINT ["bash"] -CMD ["-l"] diff --git a/share/spack/docker/build/scilinux.dockerfile b/share/spack/docker/build/scilinux.dockerfile deleted file mode 100644 index 3827f5f60e..0000000000 --- a/share/spack/docker/build/scilinux.dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -FROM sl:7 -MAINTAINER Patrick Gartung (gartung@fnal.gov) - -ENV SPACK_ROOT=/spack \ - FORCE_UNSAFE_CONFIGURE=1 \ - DISTRO=rhel7 \ - container=docker - -RUN yum update -y && \ - yum install -y yum-conf-repos.noarch && \ - yum update -y && \ - yum -y install epel-release && \ - yum update -y && \ - yum --enablerepo epel \ - groupinstall -y "Development Tools" && \ - yum --enablerepo epel \ - install -y \ - curl \ - findutils \ - gcc-c++ \ - gcc \ - gcc-gfortran \ - git \ - gnupg2 \ - hostname \ - iproute \ - Lmod \ - make \ - patch \ - openssh-server \ - python \ - tcl -RUN git clone --depth=1 git://github.com/spack/spack.git /spack && \ - rm -rf /var/cache/yum /spack/.git && yum clean all - -RUN echo "source /usr/share/lmod/lmod/init/bash" \ - > /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/setup-env.sh" \ - >> /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/spack-completion.bash" \ - >> /etc/profile.d/spack.sh -COPY common/handle-ssh.sh /etc/profile.d/handle-ssh.sh -COPY common/handle-prompt.sh /etc/profile.d/handle-prompt.sh.source - -RUN ( \ - echo "export DISTRO=$DISTRO" ; \ - echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ - echo "then" ; \ - echo "source /etc/profile.d/handle-prompt.sh.source" ; \ - echo "fi" ; \ -) > /etc/profile.d/handle-prompt.sh - -RUN mkdir -p /root/.spack -COPY common/modules.yaml /root/.spack/modules.yaml - -RUN rm -f /run/nologin - -RUN rm -rf /root/*.* - -WORKDIR /root -ENTRYPOINT ["bash"] -CMD ["-l"] diff --git a/share/spack/docker/build/ubuntu.dockerfile b/share/spack/docker/build/ubuntu.dockerfile deleted file mode 100644 index ac608ddd12..0000000000 --- a/share/spack/docker/build/ubuntu.dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -FROM ubuntu -MAINTAINER Omar Padron - -ENV DEBIAN_FRONTEND=noninteractive \ - SPACK_ROOT=/spack \ - FORCE_UNSAFE_CONFIGURE=1 \ - DISTRO=ubuntu - -RUN apt-get -yqq update && apt-get -yqq install \ - build-essential \ - ca-certificates \ - curl \ - g++ \ - gcc \ - gfortran \ - git \ - gnupg2 \ - lmod \ - make \ - openssh-server \ - python \ - tcl && \ - git clone --depth 1 git://github.com/spack/spack.git /spack && \ - rm -rf /spack/.git && rm -rf /var/lib/apt/lists/* - -RUN echo "source /usr/share/lmod/lmod/init/bash" \ - > /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/setup-env.sh" \ - >> /etc/profile.d/spack.sh -RUN echo "source /spack/share/spack/spack-completion.bash" \ - >> /etc/profile.d/spack.sh -COPY common/handle-ssh.sh /etc/profile.d/handle-ssh.sh -COPY common/handle-prompt.sh /etc/profile.d/handle-prompt.sh.source - - -RUN ( \ - echo "export DISTRO=$DISTRO" ; \ - echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ - echo "then" ; \ - echo "source /etc/profile.d/handle-prompt.sh.source" ; \ - echo "fi" ; \ -) > /etc/profile.d/handle-prompt.sh - -RUN mkdir -p /root/.spack -COPY common/modules.yaml /root/.spack/modules.yaml - -RUN rm -rf /root/*.* - -WORKDIR /root -ENTRYPOINT ["bash"] -CMD ["-l"] diff --git a/share/spack/docker/config/arch.bash b/share/spack/docker/config/arch.bash new file mode 100644 index 0000000000..3cb2951c4b --- /dev/null +++ b/share/spack/docker/config/arch.bash @@ -0,0 +1,18 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +unset DISTRO +unset DISTRO_VERSION +unset BASE_IMAGE +unset BASE_NAME +unset BASE_TAG +unset TAG +unset EXTRA_TAGS + +export BASE_IMAGE="base/archlinux" +export BASE_NAME="archlinux" +export BASE_TAG="2018.10.01" +export DISTRO="arch" +export EXTRA_TAGS="latest" diff --git a/share/spack/docker/config/centos.bash b/share/spack/docker/config/centos.bash new file mode 100644 index 0000000000..bef7ad276b --- /dev/null +++ b/share/spack/docker/config/centos.bash @@ -0,0 +1,16 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +unset DISTRO +unset DISTRO_VERSION +unset BASE_IMAGE +unset BASE_NAME +unset BASE_TAG +unset TAG +unset EXTRA_TAGS + +export BASE_IMAGE=centos +export BASE_TAG="7" +export EXTRA_TAGS="latest" diff --git a/share/spack/docker/config/fedora.bash b/share/spack/docker/config/fedora.bash new file mode 100644 index 0000000000..fcf7425693 --- /dev/null +++ b/share/spack/docker/config/fedora.bash @@ -0,0 +1,16 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +unset DISTRO +unset DISTRO_VERSION +unset BASE_IMAGE +unset BASE_NAME +unset BASE_TAG +unset TAG +unset EXTRA_TAGS + +export BASE_IMAGE=fedora +export BASE_TAG="24" +export EXTRA_TAGS="latest" diff --git a/share/spack/docker/config/opensuse.bash b/share/spack/docker/config/opensuse.bash new file mode 100644 index 0000000000..b0e11c17ac --- /dev/null +++ b/share/spack/docker/config/opensuse.bash @@ -0,0 +1,16 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +unset DISTRO +unset DISTRO_VERSION +unset BASE_IMAGE +unset BASE_NAME +unset BASE_TAG +unset TAG +unset EXTRA_TAGS + +export BASE_IMAGE=opensuse +export BASE_TAG="tumbleweed" +export EXTRA_TAGS="latest" diff --git a/share/spack/docker/config/scilinux.bash b/share/spack/docker/config/scilinux.bash new file mode 100644 index 0000000000..71fd737f69 --- /dev/null +++ b/share/spack/docker/config/scilinux.bash @@ -0,0 +1,17 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +unset DISTRO +unset BASE_IMAGE +unset BASE_NAME +unset BASE_TAG +unset TAG +unset EXTRA_TAGS + +export BASE_IMAGE=sl +export BASE_TAG="7" +export BASE_NAME=scilinux +export DISTRO=rhel7 +export EXTRA_TAGS="latest" diff --git a/share/spack/docker/config/ubuntu.bash b/share/spack/docker/config/ubuntu.bash new file mode 100644 index 0000000000..588848fb40 --- /dev/null +++ b/share/spack/docker/config/ubuntu.bash @@ -0,0 +1,15 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +unset DISTRO +unset BASE_IMAGE +unset BASE_NAME +unset BASE_TAG +unset TAG +unset EXTRA_TAGS + +export BASE_IMAGE=ubuntu +export BASE_TAG="bionic" +export EXTRA_TAGS="latest" diff --git a/share/spack/docker/docker-compose.yml b/share/spack/docker/docker-compose.yml deleted file mode 100644 index 4f37127e3f..0000000000 --- a/share/spack/docker/docker-compose.yml +++ /dev/null @@ -1,56 +0,0 @@ -version: '3' -services: - arch: - build: - context: build - dockerfile: arch.dockerfile - volumes: - - '../../..:/spack' - environment: - PROMPT: "${PROMPT:-0}" - centos: - build: - context: build - dockerfile: centos.dockerfile - volumes: - - '../../..:/spack' - environment: - PROMPT: "${PROMPT:-0}" - fedora: - build: - context: build - dockerfile: fedora.dockerfile - volumes: - - '../../..:/spack' - environment: - PROMPT: "${PROMPT:-0}" - opensuse: - build: - context: build - dockerfile: opensuse.dockerfile - volumes: - - '../../..:/spack' - environment: - PROMPT: "${PROMPT:-0}" - scilinux: - build: - context: build - dockerfile: scilinux.dockerfile - volumes: - - '../../..:/spack' - environment: - PROMPT: "${PROMPT:-0}" - scilinux: - build: ./spack_scilinux - volumes: - - '../../..:/spack' - environment: - PROMPT: "${PROMPT:-0}" - ubuntu: - build: - context: build - dockerfile: ubuntu.dockerfile - volumes: - - '../../..:/spack' - environment: - PROMPT: "${PROMPT:-0}" diff --git a/share/spack/docker/dpp.bash b/share/spack/docker/dpp.bash new file mode 100755 index 0000000000..f41a9c5260 --- /dev/null +++ b/share/spack/docker/dpp.bash @@ -0,0 +1,77 @@ +#! /usr/bin/env bash +# +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +function prefix_tokens() { + line="$1" ; shift + nprefix="$1" + + line="${line::$nprefix} " + echo "${line::$nprefix}" +} + + +# read file contents, or stdin +cat "$1" | + +# remove blank lines +grep -v '^ *$' | + +# remove leading whitespace +sed 's/^ *//g' | + +# remove comments +grep -v '^#.*' | + +# remove trailing whitespace +sed 's/ *$//g' | + +# remove extraneous whitespace +sed 's/ */ /g' | + +# mask out subsections +( + stack_level=1 + mask_level=1 + + while read LINE ; do + try_print=1 + + if [ "$( prefix_tokens "$LINE" 10 )" '=' 'MASK PUSH ' ] ; then + tmp="$stack_level" + stack_level="$(( stack_level + 1 ))" + if [ "$mask_level" '=' "$tmp" ] ; then + mask_level="$stack_level" + fi + try_print=0 + elif [ "$( prefix_tokens "$LINE" 9 )" '=' 'MASK POP ' ] ; then + stack_level="$(( stack_level - 1 ))" + if [ "$mask_level" -gt "$stack_level" ] ; then + mask_level="$stack_level" + fi + try_print=0 + elif [ "$( prefix_tokens "$LINE" 5 )" '=' 'MASK ' ] ; then + if [ "$(( mask_level + 1 ))" -ge "$stack_level" ] ; then + mask_level="$stack_level" + eval "${LINE:5}" + if [ "$?" '!=' 0 ] ; then + mask_level="$(( mask_level - 1 ))" + fi + fi + try_print=0 + fi + + if [ "$stack_level" -lt 1 ] ; then + stack_level=1 + mask_level=0 + fi + + if [ "$try_print" '=' 1 -a "$mask_level" '=' "$stack_level" ] ; then + echo "$LINE" + fi + done +) + diff --git a/share/spack/docker/entrypoint.bash b/share/spack/docker/entrypoint.bash new file mode 100755 index 0000000000..58d6281084 --- /dev/null +++ b/share/spack/docker/entrypoint.bash @@ -0,0 +1,43 @@ +#! /usr/bin/env bash -e +# +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +if [ "$1" '=' 'docker-shell' ] ; then + if [ -t 0 ] ; then + exec bash -il + else + ( + echo -n "It looks like you're trying to run an intractive shell" + echo -n " session, but either no psuedo-TTY is allocateed for this" + echo -n " container's STDIN, or it is closed." + echo + + echo -n "Make sure you run docker with the --interactive and --tty" + echo -n " options." + echo + ) >&2 + + exit 1 + fi +else + exec 3>&1 + exec 4>&2 + + exec 1>&- + exec 2>&- + + source /etc/profile.d/spack.sh + source /etc/profile.d/handle-ssh.sh + + exec 1>&3 + exec 2>&4 + + exec 3>&- + exec 4>&- + + spack "$@" + exit $? +fi diff --git a/share/spack/docker/handle-prompt.sh b/share/spack/docker/handle-prompt.sh new file mode 100644 index 0000000000..d523ac3a31 --- /dev/null +++ b/share/spack/docker/handle-prompt.sh @@ -0,0 +1,169 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +if [ x$SPACK_PROMPT '!=' x0 ] ; then + +__tmp="`mktemp -d`" + +__trylock() { + local dir + dir="$__tmp/$1.lock" + mkdir "$dir" &>/dev/null + return $? +} + +__queue_init() { + local r + local w + + mkdir "$__tmp/$1.read.lock" ; r=$? + mkdir "$__tmp/$1.write.lock" ; w=$? + + if [ "$r" '=' '0' -a "$w" '=' '0' ] ; then + return 0 + else + return 1 + fi +} + +__queue_try_read() { + __trylock "$1.read" + return $? +} + +__queue_try_write() { + __trylock "$1.write" + return $? +} + +__queue_make_readable() { + rm -r "$__tmp/$1.read.lock" &>/dev/null + return $? +} + +__queue_make_writable() { + rm -r "$__tmp/$1.write.lock" &>/dev/null + return $? +} + +__read() { + cat "$__tmp/$1" 2> /dev/null + return $? +} + +__write() { + cat > "$__tmp/$1" 2> /dev/null + return $? +} + +__revparse_head() { + head="`git -C /spack rev-parse $@ HEAD 2>/dev/null`" + result="$?" + if [ "$result" '!=' '0' ] ; then + head="`git --git-dir=/spack/.git \\ + --work-tree=/spack rev-parse $@ HEAD 2>/dev/null`" + result="$?" + fi + + echo "$head" + return $result +} + +__git_head() { + head="`__revparse_head --abbrev-ref`" + if [ "$?" '=' '0' ] ; then + if [ "$head" '=' 'HEAD' ] ; then + head="`__revparse_head | cut -c1-8`..." + fi + + echo "$head" + fi +} + +__update_prompt() { + local prompt + prompt='' + linux_distro="$DOCKERFILE_DISTRO" + if [ -n "$linux_distro" ] ; then + linux_distro='\[\e[1;34m\][\[\e[0;34m\]'"$linux_distro"'\[\e[1;34m\]]' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$linux_distro" + fi + + git_head="`__git_head`" + + if [ -n "$git_head" ] ; then + git_head='\[\e[1;32m\](\[\e[0;32m\]'"$git_head"'\[\e[1;32m\])' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$git_head" + fi + + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt"'\[\e[0;m\]\W: ' + echo "$prompt" | __write prompt +} + +set -m +( + __queue_init query + __queue_init prompt + + __update_prompt + __queue_make_readable prompt + + __queue_make_writable query + + while sleep 0.010 ; do + last_q_time='' + + while sleep 0.010 ; do + q_time="`date +%s%N`" + if __queue_try_read query ; then + last_q_time="$q_time" + __queue_make_writable query + fi + + if [ -n "$last_q_time" -a \ + "$(( (q_time - last_q_time)/10000000 > 100 ))" '=' '1' ] ; then + break + fi + done + + __update_prompt + __queue_make_readable prompt + done +) &>/dev/null & +set +m + +__update_prompt_main_first_call=1 +__update_prompt_main() { + if [ "$__update_prompt_main_first_call" '=' '1' ] ; then + while sleep 0.001 ; do + if __queue_try_read prompt ; then + PS1="`__read prompt`" + break + fi + done + __update_prompt_main_first_call=0 + else + if __queue_try_read prompt ; then + PS1="`__read prompt`" + fi + fi + + if __queue_try_write query ; then + __queue_make_readable query + fi +} + +PROMPT_COMMAND=__update_prompt_main + +fi # [ x$SPACK_PROMPT '!=' x0 ] diff --git a/share/spack/docker/handle-ssh.sh b/share/spack/docker/handle-ssh.sh new file mode 100644 index 0000000000..f2139897cb --- /dev/null +++ b/share/spack/docker/handle-ssh.sh @@ -0,0 +1,41 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +uid="`id -u`" +if [ "$uid" '=' '0' ] ; then + for key_type in dsa ecdsa ed25519 rsa ; do + private_key_file="/etc/ssh/ssh_host_${key_type}_key" + public_key_file="$private_key_file.pub" + + if [ '!' -f "$private_key_file" ] ; then + ssh-keygen \ + -q -t "$key_type" -N "" -f "$private_key_file" + chmod 600 "$private_key_file" + chmod 644 "$public_key_file" + fi + done + + mkdir -p /var/run/sshd + + pgrep -u 0 -U 0 sshd &> /dev/null + if [ '!' "$?" '=' '0' ] ; then + nohup /usr/sbin/sshd -f /etc/ssh/sshd_config < /dev/null &> /dev/null + fi +fi + +if [ '!' -f "$HOME/.ssh/id_rsa" ] ; then + ssh-keygen \ + -t rsa -C "spack.developer@docker.host" -N "" -f "$HOME/.ssh/id_rsa" + cat "$HOME/.ssh/id_rsa.pub" >> "$HOME/.ssh/authorized_keys" + chmod 600 "$HOME/.ssh/authorized_keys" + + docker_ip="`ip address show dev eth0 | + grep inet | + cut -d' ' -f 6 | + cut -d/ -f 1`" + + ssh-keyscan -t rsa 127.0.0.1 localhost "$docker_ip" "`hostname`" \ + > "$HOME/.ssh/known_hosts" 2> /dev/null +fi diff --git a/share/spack/docker/modules.yaml b/share/spack/docker/modules.yaml new file mode 100644 index 0000000000..02907ecc40 --- /dev/null +++ b/share/spack/docker/modules.yaml @@ -0,0 +1,6 @@ +modules: + enable: + - lmod + lmod: + core_compilers: + - gcc diff --git a/share/spack/docker/push-image.sh b/share/spack/docker/push-image.sh new file mode 120000 index 0000000000..9e5dfdf52d --- /dev/null +++ b/share/spack/docker/push-image.sh @@ -0,0 +1 @@ +./build-image.sh \ No newline at end of file diff --git a/share/spack/docker/render-image-template.sh b/share/spack/docker/render-image-template.sh new file mode 120000 index 0000000000..9e5dfdf52d --- /dev/null +++ b/share/spack/docker/render-image-template.sh @@ -0,0 +1 @@ +./build-image.sh \ No newline at end of file diff --git a/share/spack/docker/run-image.sh b/share/spack/docker/run-image.sh new file mode 120000 index 0000000000..b3fd71be24 --- /dev/null +++ b/share/spack/docker/run-image.sh @@ -0,0 +1 @@ +build-image.sh \ No newline at end of file -- cgit v1.2.3-70-g09d2