diff options
author | Omar Padron <omar.padron@kitware.com> | 2018-02-27 03:36:14 -0500 |
---|---|---|
committer | Todd Gamblin <tgamblin@llnl.gov> | 2018-02-27 00:36:14 -0800 |
commit | 3a9cd2614e05b7d59febd4bcf7ce76a607bc2939 (patch) | |
tree | 7df5873c7e4cd2a336aa2ed51ee30af5961b1655 | |
parent | 072e78dd52c0c6ebc25af01656cd57eb1814397b (diff) | |
download | spack-3a9cd2614e05b7d59febd4bcf7ce76a607bc2939.tar.gz spack-3a9cd2614e05b7d59febd4bcf7ce76a607bc2939.tar.bz2 spack-3a9cd2614e05b7d59febd4bcf7ce76a607bc2939.tar.xz spack-3a9cd2614e05b7d59febd4bcf7ce76a607bc2939.zip |
Docker-Based Developer Resource (#5871)
* add docker-based development resources
* incorporate feedback from @ax3l
* fix docs/improve ssh handling
* experiment with custom prompt
* add arch/fix missing core_compilers key
* add centos/minor tweaks
* make prompt experiment optional
* workaround problem with latest fedora docker image
* add docker documentation page to index toc
* try another documentation fix
* switch arch linux base to base/archlinux
* update the git urls in the Dockerfiles
* add opensuse
* switch CUSTOM_PROMPT variable to simply "PROMPT"
25 files changed, 1308 insertions, 0 deletions
diff --git a/lib/spack/docs/docker_for_developers.rst b/lib/spack/docs/docker_for_developers.rst new file mode 100644 index 0000000000..cb08b7503e --- /dev/null +++ b/lib/spack/docs/docker_for_developers.rst @@ -0,0 +1,39 @@ +.. _docker_for_developers: + +===================== +Docker for Developers +===================== + +This guide is intended for people who want to use our prepared docker +environments to work on developing Spack or working on spack packages. It is +meant to serve as the companion documentation for the :ref:`packaging-guide`. + +-------- +Overview +-------- + +To get started, all you need is the latest version of ``docker`` and +``docker-compose``. + +.. code-block:: console + + $ cd share/spack/docker + $ docker-compose run --rm ubuntu + +This command should drop you into an interactive shell where you can run spack +within an isolated docker container running ubuntu. The copy of spack being +used should be tied to the working copy of your cloned git repo, so any changes +you make should be immediately reflected in the running docker container. Feel +free to add or modify any packages or to hack on spack, itself. Your contained +copy of spack should immediately reflect all changes. + +To work within a container running a different linux distro, change the "ubuntu" +argument to any one of the services listed under the ``docker-compose.yml`` +file. + +.. code-block:: console + + $ docker-compose config --services + fedora + ubuntu + $ docker-compose run --rm fedora diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 2ca737bb72..ea80e2c21e 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -74,6 +74,7 @@ or refer to the full manual below. contribution_guide packaging_guide developer_guide + docker_for_developers Spack API Docs <spack> LLNL API Docs <llnl> diff --git a/lib/spack/docs/workflows.rst b/lib/spack/docs/workflows.rst index 4c674f92e4..0d8ffb7a14 100644 --- a/lib/spack/docs/workflows.rst +++ b/lib/spack/docs/workflows.rst @@ -1254,6 +1254,28 @@ Just use the `docker bootstraping mechanism <http://singularity.lbl.gov/quicksta %runscript exec /bin/bash -l +"""""""""""""""""""""" +Docker for Development +"""""""""""""""""""""" + +For examples of how we use docker in development, see +:ref:`docker_for_developers`. + +""""""""""""""""""""""""" +Docker on Windows and OSX +""""""""""""""""""""""""" + +On Mac OS and Windows, docker runs on a hypervisor that is not allocated much +memory by default, and some spack packages may fail to build due to lack of +memory. To work around this issue, consider configuring your docker installation +to use more of your host memory. In some cases, you can also ease the memory +pressure on parallel builds by limiting the parallelism in your config.yaml. + +.. code-block:: yaml + + config: + build_jobs: 2 + ------------------ Upstream Bug Fixes ------------------ diff --git a/share/spack/docker/.env b/share/spack/docker/.env new file mode 100644 index 0000000000..924ceb3f73 --- /dev/null +++ b/share/spack/docker/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=spack diff --git a/share/spack/docker/docker-compose.yml b/share/spack/docker/docker-compose.yml new file mode 100644 index 0000000000..f050a86700 --- /dev/null +++ b/share/spack/docker/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3' +services: + arch: + build: ./spack_arch + volumes: + - '../../..:/spack' + environment: + PROMPT: "${PROMPT:-0}" + centos: + build: ./spack_centos + volumes: + - '../../..:/spack' + environment: + PROMPT: "${PROMPT:-0}" + fedora: + build: ./spack_fedora + volumes: + - '../../..:/spack' + environment: + PROMPT: "${PROMPT:-0}" + opensuse: + build: ./spack_opensuse + volumes: + - '../../..:/spack' + environment: + PROMPT: "${PROMPT:-0}" + ubuntu: + build: ./spack_ubuntu + volumes: + - '../../..:/spack' + environment: + PROMPT: "${PROMPT:-0}" diff --git a/share/spack/docker/spack_arch/Dockerfile b/share/spack/docker/spack_arch/Dockerfile new file mode 100644 index 0000000000..973b16f125 --- /dev/null +++ b/share/spack/docker/spack_arch/Dockerfile @@ -0,0 +1,59 @@ +FROM base/archlinux +MAINTAINER Omar Padron <omar.padron@kitware.com> + +ENV SPACK_ROOT=/spack \ + FORCE_UNSAFE_CONFIGURE=1 \ + DISTRO=arch + +RUN pacman -Sy --noconfirm \ + base-devel \ + ca-certificates \ + curl \ + gcc \ + gcc-fortran \ + git \ + gnupg2 \ + iproute2 \ + make \ + openssh \ + python \ + sudo \ + tcl && \ + git clone --depth 1 git://github.com/spack/spack.git /spack && \ + echo 'nobody ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nobody-sudo && \ + sudo -u nobody git clone --depth 1 \ + https://aur.archlinux.org/lua-posix.git /tmp/lua-posix && \ + sudo -u nobody git clone --depth 1 \ + https://aur.archlinux.org/lmod.git /tmp/lmod && \ + ( cd /tmp/lua-posix ; sudo -u nobody makepkg -si --asdeps --noconfirm ) && \ + ( cd /tmp/lmod ; sudo -u nobody makepkg -si --noconfirm ) && \ + rm -rf /tmp/lua-posix /tmp/lmod /spack/.git /etc/sudoers.d/nobody-sudo + +RUN ( cd /usr/share/lmod ; ln -s $( ls -d ./* | head -n 1 ) ./lmod ) + +RUN echo "source /usr/share/lmod/lmod/init/bash" \ + > /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/setup-env.sh" \ + >> /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/spack-completion.bash" \ + >> /etc/profile.d/spack.sh + +COPY handle-ssh.sh /etc/profile.d/handle-ssh.sh +COPY handle-prompt.sh /etc/profile.d/handle-prompt.sh.source + +RUN ( \ + echo "export DISTRO=$DISTRO" ; \ + echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ + echo "then" ; \ + echo "source /etc/profile.d/handle-prompt.sh.source" ; \ + echo "fi" ; \ +) > /etc/profile.d/handle-prompt.sh + +RUN mkdir -p /root/.spack +COPY modules.yaml /root/.spack/modules.yaml + +RUN rm -rf /root/*.* + +WORKDIR /root +ENTRYPOINT ["bash"] +CMD ["-l"] diff --git a/share/spack/docker/spack_arch/handle-prompt.sh b/share/spack/docker/spack_arch/handle-prompt.sh new file mode 100644 index 0000000000..bf62224a60 --- /dev/null +++ b/share/spack/docker/spack_arch/handle-prompt.sh @@ -0,0 +1,143 @@ +__tmp="`mktemp -d`" + +__trylock() { + local dir + dir="$__tmp/$1.lock" + mkdir "$dir" &>/dev/null + return $? +} + +__queue_init() { + local r + local w + + mkdir "$__tmp/$1.read.lock" ; r=$? + mkdir "$__tmp/$1.write.lock" ; w=$? + + if [ "$r" '=' '0' -a "$w" '=' '0' ] ; then + return 0 + else + return 1 + fi +} + +__queue_try_read() { + __trylock "$1.read" + return $? +} + +__queue_try_write() { + __trylock "$1.write" + return $? +} + +__queue_make_readable() { + rm -r "$__tmp/$1.read.lock" &>/dev/null + return $? +} + +__queue_make_writable() { + rm -r "$__tmp/$1.write.lock" &>/dev/null + return $? +} + +__read() { + cat "$__tmp/$1" 2> /dev/null + return $? +} + +__write() { + cat > "$__tmp/$1" 2> /dev/null + return $? +} + +__update_prompt() { + local prompt + prompt='' + linux_distro="$DISTRO" + if [ -n "$linux_distro" ] ; then + linux_distro='\[\e[1;34m\][\[\e[0;34m\]'"$linux_distro"'\[\e[1;34m\]]' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$linux_distro" + fi + + git_head="`git -C /spack rev-parse --abbrev-ref HEAD 2>/dev/null`" + if [ "$?" '=' '0' ] ; then + if [ "$git_head" '=' 'HEAD' ] ; then + git_head="`git -C /spack rev-parse HEAD 2>/dev/null | cut -c1-8`..." + fi + else + git_head='' + fi + + if [ -n "$git_head" ] ; then + git_head='\[\e[1;32m\](\[\e[0;32m\]'"$git_head"'\[\e[1;32m\])' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$git_head" + fi + + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt"'\[\e[0;m\]\W: ' + echo "$prompt" | __write prompt +} + +set -m +( + __queue_init query + __queue_init prompt + + __update_prompt + __queue_make_readable prompt + + __queue_make_writable query + + while sleep 0.010 ; do + last_q_time='' + + while sleep 0.010 ; do + q_time="`date +%s%N`" + if __queue_try_read query ; then + last_q_time="$q_time" + __queue_make_writable query + fi + + if [ -n "$last_q_time" -a \ + "$(( (q_time - last_q_time)/10000000 > 100 ))" '=' '1' ] ; then + break + fi + done + + __update_prompt + __queue_make_readable prompt + done +) &>/dev/null & +set +m + +__update_prompt_main_first_call=1 +__update_prompt_main() { + if [ "$__update_prompt_main_first_call" '=' '1' ] ; then + while sleep 0.001 ; do + if __queue_try_read prompt ; then + PS1="`__read prompt`" + break + fi + done + __update_prompt_main_first_call=0 + else + if __queue_try_read prompt ; then + PS1="`__read prompt`" + fi + fi + + if __queue_try_write query ; then + __queue_make_readable query + fi +} + +PROMPT_COMMAND=__update_prompt_main diff --git a/share/spack/docker/spack_arch/handle-ssh.sh b/share/spack/docker/spack_arch/handle-ssh.sh new file mode 100644 index 0000000000..f13c10e212 --- /dev/null +++ b/share/spack/docker/spack_arch/handle-ssh.sh @@ -0,0 +1,36 @@ +uid="`id -u`" +if [ "$uid" '=' '0' ] ; then + for key_type in dsa ecdsa ed25519 rsa ; do + private_key_file="/etc/ssh/ssh_host_${key_type}_key" + public_key_file="$private_key_file.pub" + + if [ '!' -f "$private_key_file" ] ; then + ssh-keygen \ + -q -t "$key_type" -N "" -f "$private_key_file" + chmod 600 "$private_key_file" + chmod 644 "$public_key_file" + fi + done + + mkdir -p /var/run/sshd + + pgrep -u 0 -U 0 sshd &> /dev/null + if [ '!' "$?" '=' '0' ] ; then + nohup /usr/sbin/sshd -f /etc/ssh/sshd_config < /dev/null &> /dev/null + fi +fi + +if [ '!' -f "$HOME/.ssh/id_rsa" ] ; then + ssh-keygen \ + -t rsa -C "spack.developer@docker.host" -N "" -f "$HOME/.ssh/id_rsa" + cat "$HOME/.ssh/id_rsa.pub" >> "$HOME/.ssh/authorized_keys" + chmod 600 "$HOME/.ssh/authorized_keys" + + docker_ip="`ip address show dev eth0 | + grep inet | + cut -d' ' -f 6 | + cut -d/ -f 1`" + + ssh-keyscan -t rsa 127.0.0.1 localhost "$docker_ip" "`hostname`" \ + > "$HOME/.ssh/known_hosts" 2> /dev/null +fi diff --git a/share/spack/docker/spack_arch/modules.yaml b/share/spack/docker/spack_arch/modules.yaml new file mode 100644 index 0000000000..02907ecc40 --- /dev/null +++ b/share/spack/docker/spack_arch/modules.yaml @@ -0,0 +1,6 @@ +modules: + enable: + - lmod + lmod: + core_compilers: + - gcc diff --git a/share/spack/docker/spack_centos/Dockerfile b/share/spack/docker/spack_centos/Dockerfile new file mode 100644 index 0000000000..a1069128f5 --- /dev/null +++ b/share/spack/docker/spack_centos/Dockerfile @@ -0,0 +1,57 @@ +FROM centos +MAINTAINER Omar Padron <omar.padron@kitware.com> + +ENV SPACK_ROOT=/spack \ + FORCE_UNSAFE_CONFIGURE=1 \ + DISTRO=centos + +RUN yum update -y && \ + yum install -y epel-release && \ + yum update -y && \ + yum groupinstall -y "Development Tools" && \ + yum install -y \ + curl \ + findutils \ + gcc-c++ \ + gcc \ + gcc-gfortran \ + git \ + gnupg2 \ + hostname \ + iproute \ + Lmod \ + make \ + patch \ + openssh-server \ + python \ + tcl && \ + git clone --depth 1 git://github.com/spack/spack.git /spack && \ + rm -rf /spack/.git /var/cache/yum && yum clean all + +RUN echo "source /usr/share/lmod/lmod/init/bash" \ + > /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/setup-env.sh" \ + >> /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/spack-completion.bash" \ + >> /etc/profile.d/spack.sh +COPY handle-ssh.sh /etc/profile.d/handle-ssh.sh +COPY handle-prompt.sh /etc/profile.d/handle-prompt.sh.source + +RUN ( \ + echo "export DISTRO=$DISTRO" ; \ + echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ + echo "then" ; \ + echo "source /etc/profile.d/handle-prompt.sh.source" ; \ + echo "fi" ; \ +) > /etc/profile.d/handle-prompt.sh + +RUN mkdir -p /root/.spack +COPY modules.yaml /root/.spack/modules.yaml + +RUN rm -f /run/nologin + +RUN rm -rf /root/*.* + +WORKDIR /root +ENTRYPOINT ["bash"] +CMD ["-l"] diff --git a/share/spack/docker/spack_centos/handle-prompt.sh b/share/spack/docker/spack_centos/handle-prompt.sh new file mode 100644 index 0000000000..15ef53e316 --- /dev/null +++ b/share/spack/docker/spack_centos/handle-prompt.sh @@ -0,0 +1,143 @@ +__tmp="`mktemp -d`" + +__trylock() { + local dir + dir="$__tmp/$1.lock" + mkdir "$dir" &>/dev/null + return $? +} + +__queue_init() { + local r + local w + + mkdir "$__tmp/$1.read.lock" ; r=$? + mkdir "$__tmp/$1.write.lock" ; w=$? + + if [ "$r" '=' '0' -a "$w" '=' '0' ] ; then + return 0 + else + return 1 + fi +} + +__queue_try_read() { + __trylock "$1.read" + return $? +} + +__queue_try_write() { + __trylock "$1.write" + return $? +} + +__queue_make_readable() { + rm -r "$__tmp/$1.read.lock" &>/dev/null + return $? +} + +__queue_make_writable() { + rm -r "$__tmp/$1.write.lock" &>/dev/null + return $? +} + +__read() { + cat "$__tmp/$1" 2> /dev/null + return $? +} + +__write() { + cat > "$__tmp/$1" 2> /dev/null + return $? +} + +__update_prompt() { + local prompt + prompt='' + linux_distro="$DISTRO" + if [ -n "$linux_distro" ] ; then + linux_distro='\[\e[1;34m\][\[\e[0;34m\]'"$linux_distro"'\[\e[1;34m\]]' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$linux_distro" + fi + + git_head="`git --git-dir=/spack/.git --work-tree=/spack rev-parse --abbrev-ref HEAD 2>/dev/null`" + if [ "$?" '=' '0' ] ; then + if [ "$git_head" '=' 'HEAD' ] ; then + git_head="`git --git-dir=/spack/.git --work-tree=/spack rev-parse HEAD 2>/dev/null | cut -c1-8`..." + fi + else + git_head='' + fi + + if [ -n "$git_head" ] ; then + git_head='\[\e[1;32m\](\[\e[0;32m\]'"$git_head"'\[\e[1;32m\])' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$git_head" + fi + + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt"'\[\e[0;m\]\W: ' + echo "$prompt" | __write prompt +} + +set -m +( + __queue_init query + __queue_init prompt + + __update_prompt + __queue_make_readable prompt + + __queue_make_writable query + + while sleep 0.010 ; do + last_q_time='' + + while sleep 0.010 ; do + q_time="`date +%s%N`" + if __queue_try_read query ; then + last_q_time="$q_time" + __queue_make_writable query + fi + + if [ -n "$last_q_time" -a \ + "$(( (q_time - last_q_time)/10000000 > 100 ))" '=' '1' ] ; then + break + fi + done + + __update_prompt + __queue_make_readable prompt + done +) &>/dev/null & +set +m + +__update_prompt_main_first_call=1 +__update_prompt_main() { + if [ "$__update_prompt_main_first_call" '=' '1' ] ; then + while sleep 0.001 ; do + if __queue_try_read prompt ; then + PS1="`__read prompt`" + break + fi + done + __update_prompt_main_first_call=0 + else + if __queue_try_read prompt ; then + PS1="`__read prompt`" + fi + fi + + if __queue_try_write query ; then + __queue_make_readable query + fi +} + +PROMPT_COMMAND=__update_prompt_main diff --git a/share/spack/docker/spack_centos/handle-ssh.sh b/share/spack/docker/spack_centos/handle-ssh.sh new file mode 100644 index 0000000000..f13c10e212 --- /dev/null +++ b/share/spack/docker/spack_centos/handle-ssh.sh @@ -0,0 +1,36 @@ +uid="`id -u`" +if [ "$uid" '=' '0' ] ; then + for key_type in dsa ecdsa ed25519 rsa ; do + private_key_file="/etc/ssh/ssh_host_${key_type}_key" + public_key_file="$private_key_file.pub" + + if [ '!' -f "$private_key_file" ] ; then + ssh-keygen \ + -q -t "$key_type" -N "" -f "$private_key_file" + chmod 600 "$private_key_file" + chmod 644 "$public_key_file" + fi + done + + mkdir -p /var/run/sshd + + pgrep -u 0 -U 0 sshd &> /dev/null + if [ '!' "$?" '=' '0' ] ; then + nohup /usr/sbin/sshd -f /etc/ssh/sshd_config < /dev/null &> /dev/null + fi +fi + +if [ '!' -f "$HOME/.ssh/id_rsa" ] ; then + ssh-keygen \ + -t rsa -C "spack.developer@docker.host" -N "" -f "$HOME/.ssh/id_rsa" + cat "$HOME/.ssh/id_rsa.pub" >> "$HOME/.ssh/authorized_keys" + chmod 600 "$HOME/.ssh/authorized_keys" + + docker_ip="`ip address show dev eth0 | + grep inet | + cut -d' ' -f 6 | + cut -d/ -f 1`" + + ssh-keyscan -t rsa 127.0.0.1 localhost "$docker_ip" "`hostname`" \ + > "$HOME/.ssh/known_hosts" 2> /dev/null +fi diff --git a/share/spack/docker/spack_centos/modules.yaml b/share/spack/docker/spack_centos/modules.yaml new file mode 100644 index 0000000000..02907ecc40 --- /dev/null +++ b/share/spack/docker/spack_centos/modules.yaml @@ -0,0 +1,6 @@ +modules: + enable: + - lmod + lmod: + core_compilers: + - gcc diff --git a/share/spack/docker/spack_fedora/Dockerfile b/share/spack/docker/spack_fedora/Dockerfile new file mode 100644 index 0000000000..5e0d984a04 --- /dev/null +++ b/share/spack/docker/spack_fedora/Dockerfile @@ -0,0 +1,56 @@ +FROM fedora:24 +MAINTAINER Omar Padron <omar.padron@kitware.com> + +ENV SPACK_ROOT=/spack \ + FORCE_UNSAFE_CONFIGURE=1 \ + DISTRO=fedora + +RUN dnf update -y && \ + dnf group install -y "C Development Tools and Libraries" && \ + dnf install -y \ + @development-tools \ + curl \ + findutils \ + gcc-c++ \ + gcc \ + gcc-gfortran \ + git \ + gnupg2 \ + hostname \ + iproute \ + Lmod \ + make \ + patch \ + openssh-server \ + python \ + tcl && \ + git clone --depth 1 git://github.com/spack/spack.git /spack && \ + rm -rf /spack/.git && dnf clean all + +RUN echo "source /usr/share/lmod/lmod/init/bash" \ + > /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/setup-env.sh" \ + >> /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/spack-completion.bash" \ + >> /etc/profile.d/spack.sh +COPY handle-ssh.sh /etc/profile.d/handle-ssh.sh +COPY handle-prompt.sh /etc/profile.d/handle-prompt.sh.source + +RUN ( \ + echo "export DISTRO=$DISTRO" ; \ + echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ + echo "then" ; \ + echo "source /etc/profile.d/handle-prompt.sh.source" ; \ + echo "fi" ; \ +) > /etc/profile.d/handle-prompt.sh + +RUN mkdir -p /root/.spack +COPY modules.yaml /root/.spack/modules.yaml + +RUN rm -f /run/nologin + +RUN rm -rf /root/*.* + +WORKDIR /root +ENTRYPOINT ["bash"] +CMD ["-l"] diff --git a/share/spack/docker/spack_fedora/handle-prompt.sh b/share/spack/docker/spack_fedora/handle-prompt.sh new file mode 100644 index 0000000000..bf62224a60 --- /dev/null +++ b/share/spack/docker/spack_fedora/handle-prompt.sh @@ -0,0 +1,143 @@ +__tmp="`mktemp -d`" + +__trylock() { + local dir + dir="$__tmp/$1.lock" + mkdir "$dir" &>/dev/null + return $? +} + +__queue_init() { + local r + local w + + mkdir "$__tmp/$1.read.lock" ; r=$? + mkdir "$__tmp/$1.write.lock" ; w=$? + + if [ "$r" '=' '0' -a "$w" '=' '0' ] ; then + return 0 + else + return 1 + fi +} + +__queue_try_read() { + __trylock "$1.read" + return $? +} + +__queue_try_write() { + __trylock "$1.write" + return $? +} + +__queue_make_readable() { + rm -r "$__tmp/$1.read.lock" &>/dev/null + return $? +} + +__queue_make_writable() { + rm -r "$__tmp/$1.write.lock" &>/dev/null + return $? +} + +__read() { + cat "$__tmp/$1" 2> /dev/null + return $? +} + +__write() { + cat > "$__tmp/$1" 2> /dev/null + return $? +} + +__update_prompt() { + local prompt + prompt='' + linux_distro="$DISTRO" + if [ -n "$linux_distro" ] ; then + linux_distro='\[\e[1;34m\][\[\e[0;34m\]'"$linux_distro"'\[\e[1;34m\]]' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$linux_distro" + fi + + git_head="`git -C /spack rev-parse --abbrev-ref HEAD 2>/dev/null`" + if [ "$?" '=' '0' ] ; then + if [ "$git_head" '=' 'HEAD' ] ; then + git_head="`git -C /spack rev-parse HEAD 2>/dev/null | cut -c1-8`..." + fi + else + git_head='' + fi + + if [ -n "$git_head" ] ; then + git_head='\[\e[1;32m\](\[\e[0;32m\]'"$git_head"'\[\e[1;32m\])' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$git_head" + fi + + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt"'\[\e[0;m\]\W: ' + echo "$prompt" | __write prompt +} + +set -m +( + __queue_init query + __queue_init prompt + + __update_prompt + __queue_make_readable prompt + + __queue_make_writable query + + while sleep 0.010 ; do + last_q_time='' + + while sleep 0.010 ; do + q_time="`date +%s%N`" + if __queue_try_read query ; then + last_q_time="$q_time" + __queue_make_writable query + fi + + if [ -n "$last_q_time" -a \ + "$(( (q_time - last_q_time)/10000000 > 100 ))" '=' '1' ] ; then + break + fi + done + + __update_prompt + __queue_make_readable prompt + done +) &>/dev/null & +set +m + +__update_prompt_main_first_call=1 +__update_prompt_main() { + if [ "$__update_prompt_main_first_call" '=' '1' ] ; then + while sleep 0.001 ; do + if __queue_try_read prompt ; then + PS1="`__read prompt`" + break + fi + done + __update_prompt_main_first_call=0 + else + if __queue_try_read prompt ; then + PS1="`__read prompt`" + fi + fi + + if __queue_try_write query ; then + __queue_make_readable query + fi +} + +PROMPT_COMMAND=__update_prompt_main diff --git a/share/spack/docker/spack_fedora/handle-ssh.sh b/share/spack/docker/spack_fedora/handle-ssh.sh new file mode 100644 index 0000000000..f13c10e212 --- /dev/null +++ b/share/spack/docker/spack_fedora/handle-ssh.sh @@ -0,0 +1,36 @@ +uid="`id -u`" +if [ "$uid" '=' '0' ] ; then + for key_type in dsa ecdsa ed25519 rsa ; do + private_key_file="/etc/ssh/ssh_host_${key_type}_key" + public_key_file="$private_key_file.pub" + + if [ '!' -f "$private_key_file" ] ; then + ssh-keygen \ + -q -t "$key_type" -N "" -f "$private_key_file" + chmod 600 "$private_key_file" + chmod 644 "$public_key_file" + fi + done + + mkdir -p /var/run/sshd + + pgrep -u 0 -U 0 sshd &> /dev/null + if [ '!' "$?" '=' '0' ] ; then + nohup /usr/sbin/sshd -f /etc/ssh/sshd_config < /dev/null &> /dev/null + fi +fi + +if [ '!' -f "$HOME/.ssh/id_rsa" ] ; then + ssh-keygen \ + -t rsa -C "spack.developer@docker.host" -N "" -f "$HOME/.ssh/id_rsa" + cat "$HOME/.ssh/id_rsa.pub" >> "$HOME/.ssh/authorized_keys" + chmod 600 "$HOME/.ssh/authorized_keys" + + docker_ip="`ip address show dev eth0 | + grep inet | + cut -d' ' -f 6 | + cut -d/ -f 1`" + + ssh-keyscan -t rsa 127.0.0.1 localhost "$docker_ip" "`hostname`" \ + > "$HOME/.ssh/known_hosts" 2> /dev/null +fi diff --git a/share/spack/docker/spack_fedora/modules.yaml b/share/spack/docker/spack_fedora/modules.yaml new file mode 100644 index 0000000000..02907ecc40 --- /dev/null +++ b/share/spack/docker/spack_fedora/modules.yaml @@ -0,0 +1,6 @@ +modules: + enable: + - lmod + lmod: + core_compilers: + - gcc diff --git a/share/spack/docker/spack_opensuse/Dockerfile b/share/spack/docker/spack_opensuse/Dockerfile new file mode 100644 index 0000000000..9bc31dcf0c --- /dev/null +++ b/share/spack/docker/spack_opensuse/Dockerfile @@ -0,0 +1,65 @@ +FROM opensuse +MAINTAINER Omar Padron <omar.padron@kitware.com> + +ENV SPACK_ROOT=/spack \ + FORCE_UNSAFE_CONFIGURE=1 \ + DISTRO=opensuse + +RUN zypper -n ref && \ + zypper -n up --skip-interactive --no-recommends && \ + zypper -n install -l --no-recommends --type pattern \ + devel_basis \ + devel_C_C++ && \ + zypper -n install -l --no-recommends \ + bash \ + bash-completion \ + ca-certificates \ + curl \ + findutils \ + gcc \ + gcc-locale \ + gcc-c++ \ + gcc-fortran \ + git \ + glibc-locale \ + gpg2 \ + hostname \ + iproute \ + lua-lmod \ + make \ + patch \ + openssh \ + python \ + python-xml \ + tcl && \ + git clone --depth 1 git://github.com/spack/spack.git /spack && \ + zypper clean && \ + rm -rf /spack/.git /var/cache/zypp/* + +RUN echo "source /usr/share/lmod/lmod/init/bash" \ + > /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/setup-env.sh" \ + >> /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/spack-completion.bash" \ + >> /etc/profile.d/spack.sh +COPY handle-ssh.sh /etc/profile.d/handle-ssh.sh +COPY handle-prompt.sh /etc/profile.d/handle-prompt.sh.source + +RUN ( \ + echo "export DISTRO=$DISTRO" ; \ + echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ + echo "then" ; \ + echo "source /etc/profile.d/handle-prompt.sh.source" ; \ + echo "fi" ; \ +) > /etc/profile.d/handle-prompt.sh + +RUN mkdir -p /root/.spack +COPY modules.yaml /root/.spack/modules.yaml + +RUN rm -f /run/nologin + +RUN rm -rf /root/*.* + +WORKDIR /root +ENTRYPOINT ["bash"] +CMD ["-l"] diff --git a/share/spack/docker/spack_opensuse/handle-prompt.sh b/share/spack/docker/spack_opensuse/handle-prompt.sh new file mode 100644 index 0000000000..15ef53e316 --- /dev/null +++ b/share/spack/docker/spack_opensuse/handle-prompt.sh @@ -0,0 +1,143 @@ +__tmp="`mktemp -d`" + +__trylock() { + local dir + dir="$__tmp/$1.lock" + mkdir "$dir" &>/dev/null + return $? +} + +__queue_init() { + local r + local w + + mkdir "$__tmp/$1.read.lock" ; r=$? + mkdir "$__tmp/$1.write.lock" ; w=$? + + if [ "$r" '=' '0' -a "$w" '=' '0' ] ; then + return 0 + else + return 1 + fi +} + +__queue_try_read() { + __trylock "$1.read" + return $? +} + +__queue_try_write() { + __trylock "$1.write" + return $? +} + +__queue_make_readable() { + rm -r "$__tmp/$1.read.lock" &>/dev/null + return $? +} + +__queue_make_writable() { + rm -r "$__tmp/$1.write.lock" &>/dev/null + return $? +} + +__read() { + cat "$__tmp/$1" 2> /dev/null + return $? +} + +__write() { + cat > "$__tmp/$1" 2> /dev/null + return $? +} + +__update_prompt() { + local prompt + prompt='' + linux_distro="$DISTRO" + if [ -n "$linux_distro" ] ; then + linux_distro='\[\e[1;34m\][\[\e[0;34m\]'"$linux_distro"'\[\e[1;34m\]]' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$linux_distro" + fi + + git_head="`git --git-dir=/spack/.git --work-tree=/spack rev-parse --abbrev-ref HEAD 2>/dev/null`" + if [ "$?" '=' '0' ] ; then + if [ "$git_head" '=' 'HEAD' ] ; then + git_head="`git --git-dir=/spack/.git --work-tree=/spack rev-parse HEAD 2>/dev/null | cut -c1-8`..." + fi + else + git_head='' + fi + + if [ -n "$git_head" ] ; then + git_head='\[\e[1;32m\](\[\e[0;32m\]'"$git_head"'\[\e[1;32m\])' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$git_head" + fi + + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt"'\[\e[0;m\]\W: ' + echo "$prompt" | __write prompt +} + +set -m +( + __queue_init query + __queue_init prompt + + __update_prompt + __queue_make_readable prompt + + __queue_make_writable query + + while sleep 0.010 ; do + last_q_time='' + + while sleep 0.010 ; do + q_time="`date +%s%N`" + if __queue_try_read query ; then + last_q_time="$q_time" + __queue_make_writable query + fi + + if [ -n "$last_q_time" -a \ + "$(( (q_time - last_q_time)/10000000 > 100 ))" '=' '1' ] ; then + break + fi + done + + __update_prompt + __queue_make_readable prompt + done +) &>/dev/null & +set +m + +__update_prompt_main_first_call=1 +__update_prompt_main() { + if [ "$__update_prompt_main_first_call" '=' '1' ] ; then + while sleep 0.001 ; do + if __queue_try_read prompt ; then + PS1="`__read prompt`" + break + fi + done + __update_prompt_main_first_call=0 + else + if __queue_try_read prompt ; then + PS1="`__read prompt`" + fi + fi + + if __queue_try_write query ; then + __queue_make_readable query + fi +} + +PROMPT_COMMAND=__update_prompt_main diff --git a/share/spack/docker/spack_opensuse/handle-ssh.sh b/share/spack/docker/spack_opensuse/handle-ssh.sh new file mode 100644 index 0000000000..f13c10e212 --- /dev/null +++ b/share/spack/docker/spack_opensuse/handle-ssh.sh @@ -0,0 +1,36 @@ +uid="`id -u`" +if [ "$uid" '=' '0' ] ; then + for key_type in dsa ecdsa ed25519 rsa ; do + private_key_file="/etc/ssh/ssh_host_${key_type}_key" + public_key_file="$private_key_file.pub" + + if [ '!' -f "$private_key_file" ] ; then + ssh-keygen \ + -q -t "$key_type" -N "" -f "$private_key_file" + chmod 600 "$private_key_file" + chmod 644 "$public_key_file" + fi + done + + mkdir -p /var/run/sshd + + pgrep -u 0 -U 0 sshd &> /dev/null + if [ '!' "$?" '=' '0' ] ; then + nohup /usr/sbin/sshd -f /etc/ssh/sshd_config < /dev/null &> /dev/null + fi +fi + +if [ '!' -f "$HOME/.ssh/id_rsa" ] ; then + ssh-keygen \ + -t rsa -C "spack.developer@docker.host" -N "" -f "$HOME/.ssh/id_rsa" + cat "$HOME/.ssh/id_rsa.pub" >> "$HOME/.ssh/authorized_keys" + chmod 600 "$HOME/.ssh/authorized_keys" + + docker_ip="`ip address show dev eth0 | + grep inet | + cut -d' ' -f 6 | + cut -d/ -f 1`" + + ssh-keyscan -t rsa 127.0.0.1 localhost "$docker_ip" "`hostname`" \ + > "$HOME/.ssh/known_hosts" 2> /dev/null +fi diff --git a/share/spack/docker/spack_opensuse/modules.yaml b/share/spack/docker/spack_opensuse/modules.yaml new file mode 100644 index 0000000000..02907ecc40 --- /dev/null +++ b/share/spack/docker/spack_opensuse/modules.yaml @@ -0,0 +1,6 @@ +modules: + enable: + - lmod + lmod: + core_compilers: + - gcc diff --git a/share/spack/docker/spack_ubuntu/Dockerfile b/share/spack/docker/spack_ubuntu/Dockerfile new file mode 100644 index 0000000000..8c1caac61b --- /dev/null +++ b/share/spack/docker/spack_ubuntu/Dockerfile @@ -0,0 +1,51 @@ +FROM ubuntu +MAINTAINER Omar Padron <omar.padron@kitware.com> + +ENV DEBIAN_FRONTEND=noninteractive \ + SPACK_ROOT=/spack \ + FORCE_UNSAFE_CONFIGURE=1 \ + DISTRO=ubuntu + +RUN apt-get -yqq update && apt-get -yqq install \ + build-essential \ + ca-certificates \ + curl \ + g++ \ + gcc \ + gfortran \ + git \ + gnupg2 \ + lmod \ + make \ + openssh-server \ + python \ + tcl && \ + git clone --depth 1 git://github.com/spack/spack.git /spack && \ + rm -rf /spack/.git && rm -rf /var/lib/apt/lists/* + +RUN echo "source /usr/share/lmod/lmod/init/bash" \ + > /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/setup-env.sh" \ + >> /etc/profile.d/spack.sh +RUN echo "source /spack/share/spack/spack-completion.bash" \ + >> /etc/profile.d/spack.sh +COPY handle-ssh.sh /etc/profile.d/handle-ssh.sh +COPY handle-prompt.sh /etc/profile.d/handle-prompt.sh.source + + +RUN ( \ + echo "export DISTRO=$DISTRO" ; \ + echo "if [ x\$PROMPT '!=' 'x' -a x\$PROMPT '!=' 'x0' ]" ; \ + echo "then" ; \ + echo "source /etc/profile.d/handle-prompt.sh.source" ; \ + echo "fi" ; \ +) > /etc/profile.d/handle-prompt.sh + +RUN mkdir -p /root/.spack +COPY modules.yaml /root/.spack/modules.yaml + +RUN rm -rf /root/*.* + +WORKDIR /root +ENTRYPOINT ["bash"] +CMD ["-l"] diff --git a/share/spack/docker/spack_ubuntu/handle-prompt.sh b/share/spack/docker/spack_ubuntu/handle-prompt.sh new file mode 100644 index 0000000000..bf62224a60 --- /dev/null +++ b/share/spack/docker/spack_ubuntu/handle-prompt.sh @@ -0,0 +1,143 @@ +__tmp="`mktemp -d`" + +__trylock() { + local dir + dir="$__tmp/$1.lock" + mkdir "$dir" &>/dev/null + return $? +} + +__queue_init() { + local r + local w + + mkdir "$__tmp/$1.read.lock" ; r=$? + mkdir "$__tmp/$1.write.lock" ; w=$? + + if [ "$r" '=' '0' -a "$w" '=' '0' ] ; then + return 0 + else + return 1 + fi +} + +__queue_try_read() { + __trylock "$1.read" + return $? +} + +__queue_try_write() { + __trylock "$1.write" + return $? +} + +__queue_make_readable() { + rm -r "$__tmp/$1.read.lock" &>/dev/null + return $? +} + +__queue_make_writable() { + rm -r "$__tmp/$1.write.lock" &>/dev/null + return $? +} + +__read() { + cat "$__tmp/$1" 2> /dev/null + return $? +} + +__write() { + cat > "$__tmp/$1" 2> /dev/null + return $? +} + +__update_prompt() { + local prompt + prompt='' + linux_distro="$DISTRO" + if [ -n "$linux_distro" ] ; then + linux_distro='\[\e[1;34m\][\[\e[0;34m\]'"$linux_distro"'\[\e[1;34m\]]' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$linux_distro" + fi + + git_head="`git -C /spack rev-parse --abbrev-ref HEAD 2>/dev/null`" + if [ "$?" '=' '0' ] ; then + if [ "$git_head" '=' 'HEAD' ] ; then + git_head="`git -C /spack rev-parse HEAD 2>/dev/null | cut -c1-8`..." + fi + else + git_head='' + fi + + if [ -n "$git_head" ] ; then + git_head='\[\e[1;32m\](\[\e[0;32m\]'"$git_head"'\[\e[1;32m\])' + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt$git_head" + fi + + if [ -n "$prompt" ] ; then + prompt="$prompt " + fi + prompt="$prompt"'\[\e[0;m\]\W: ' + echo "$prompt" | __write prompt +} + +set -m +( + __queue_init query + __queue_init prompt + + __update_prompt + __queue_make_readable prompt + + __queue_make_writable query + + while sleep 0.010 ; do + last_q_time='' + + while sleep 0.010 ; do + q_time="`date +%s%N`" + if __queue_try_read query ; then + last_q_time="$q_time" + __queue_make_writable query + fi + + if [ -n "$last_q_time" -a \ + "$(( (q_time - last_q_time)/10000000 > 100 ))" '=' '1' ] ; then + break + fi + done + + __update_prompt + __queue_make_readable prompt + done +) &>/dev/null & +set +m + +__update_prompt_main_first_call=1 +__update_prompt_main() { + if [ "$__update_prompt_main_first_call" '=' '1' ] ; then + while sleep 0.001 ; do + if __queue_try_read prompt ; then + PS1="`__read prompt`" + break + fi + done + __update_prompt_main_first_call=0 + else + if __queue_try_read prompt ; then + PS1="`__read prompt`" + fi + fi + + if __queue_try_write query ; then + __queue_make_readable query + fi +} + +PROMPT_COMMAND=__update_prompt_main diff --git a/share/spack/docker/spack_ubuntu/handle-ssh.sh b/share/spack/docker/spack_ubuntu/handle-ssh.sh new file mode 100644 index 0000000000..f13c10e212 --- /dev/null +++ b/share/spack/docker/spack_ubuntu/handle-ssh.sh @@ -0,0 +1,36 @@ +uid="`id -u`" +if [ "$uid" '=' '0' ] ; then + for key_type in dsa ecdsa ed25519 rsa ; do + private_key_file="/etc/ssh/ssh_host_${key_type}_key" + public_key_file="$private_key_file.pub" + + if [ '!' -f "$private_key_file" ] ; then + ssh-keygen \ + -q -t "$key_type" -N "" -f "$private_key_file" + chmod 600 "$private_key_file" + chmod 644 "$public_key_file" + fi + done + + mkdir -p /var/run/sshd + + pgrep -u 0 -U 0 sshd &> /dev/null + if [ '!' "$?" '=' '0' ] ; then + nohup /usr/sbin/sshd -f /etc/ssh/sshd_config < /dev/null &> /dev/null + fi +fi + +if [ '!' -f "$HOME/.ssh/id_rsa" ] ; then + ssh-keygen \ + -t rsa -C "spack.developer@docker.host" -N "" -f "$HOME/.ssh/id_rsa" + cat "$HOME/.ssh/id_rsa.pub" >> "$HOME/.ssh/authorized_keys" + chmod 600 "$HOME/.ssh/authorized_keys" + + docker_ip="`ip address show dev eth0 | + grep inet | + cut -d' ' -f 6 | + cut -d/ -f 1`" + + ssh-keyscan -t rsa 127.0.0.1 localhost "$docker_ip" "`hostname`" \ + > "$HOME/.ssh/known_hosts" 2> /dev/null +fi diff --git a/share/spack/docker/spack_ubuntu/modules.yaml b/share/spack/docker/spack_ubuntu/modules.yaml new file mode 100644 index 0000000000..02907ecc40 --- /dev/null +++ b/share/spack/docker/spack_ubuntu/modules.yaml @@ -0,0 +1,6 @@ +modules: + enable: + - lmod + lmod: + core_compilers: + - gcc |