From 7a81c37bdef5151bd4503af46ef3ccb5ef65ae04 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 26 Nov 2019 10:11:29 -0700 Subject: Package Index: Build in Dockerhub (#13810) * Package Index: Build in Dockerhub Prepare to build the package index service, packages.spack.io, on Dockerhub. Local build (in spack root dir): ``` docker build -t spack/packages.spack.io:latest -f share/spack/docker/package-index/Dockerfile . ``` Local test: ``` docker run -p 8080:80 spack/packages.spack.io:latest ``` * Travis-CI: Remove Docker Remove leftover docker stages from Travis-CI. * Simplify Split Call --- .travis.yml | 12 ------ share/spack/docker/package-index/Dockerfile | 30 +++++++++++++++ share/spack/docker/package-index/README.rst | 45 +++++++++++++++++++++++ share/spack/docker/package-index/cors-header.conf | 3 ++ share/spack/docker/package-index/split.sh | 20 ++++++++++ share/spack/packages/Dockerfile | 12 ------ share/spack/packages/build-image.sh | 17 --------- share/spack/packages/cors-header.conf | 3 -- share/spack/packages/push-image.sh | 1 - share/spack/packages/split.sh | 20 ---------- share/spack/qa/run-docker-tests | 45 ----------------------- 11 files changed, 98 insertions(+), 110 deletions(-) create mode 100644 share/spack/docker/package-index/Dockerfile create mode 100644 share/spack/docker/package-index/README.rst create mode 100644 share/spack/docker/package-index/cors-header.conf create mode 100755 share/spack/docker/package-index/split.sh delete mode 100644 share/spack/packages/Dockerfile delete mode 100755 share/spack/packages/build-image.sh delete mode 100644 share/spack/packages/cors-header.conf delete mode 120000 share/spack/packages/push-image.sh delete mode 100755 share/spack/packages/split.sh delete mode 100755 share/spack/qa/run-docker-tests diff --git a/.travis.yml b/.travis.yml index 691ee49695..55d1f13c54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -102,20 +102,11 @@ jobs: os: linux language: python env: [ TEST_SUITE=build, 'SPEC=mpich' ] - - python: '3.8' - stage: 'docker build' - os: linux - language: python - env: TEST_SUITE=docker - allow_failures: - - env: TEST_SUITE=docker stages: - 'style checks' - 'unit tests + documentation' - 'build tests' - - name: 'docker build' - if: type = push AND branch IN (develop, master) #============================================================================= @@ -199,9 +190,6 @@ before_script: #============================================================================= # Building #============================================================================= -services: - - docker - script: - share/spack/qa/run-$TEST_SUITE-tests diff --git a/share/spack/docker/package-index/Dockerfile b/share/spack/docker/package-index/Dockerfile new file mode 100644 index 0000000000..3ffafe90e8 --- /dev/null +++ b/share/spack/docker/package-index/Dockerfile @@ -0,0 +1,30 @@ +# prepare the package index in form of JSON files +FROM ubuntu:18.04 AS build-env + +ENV SPACK_ROOT=/opt/spack \ + DEBIAN_FRONTEND=noninteractive + +COPY bin $SPACK_ROOT/bin +COPY etc $SPACK_ROOT/etc +COPY lib $SPACK_ROOT/lib +COPY share $SPACK_ROOT/share +COPY var $SPACK_ROOT/var + +RUN apt-get -yqq update \ + && apt-get -yqq install \ + bash jq python \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build +# single, large index file +RUN $SPACK_ROOT/bin/spack list --format version_json > packages.json +# individual packages split into a tree of :firstLetter/:packageName.json +RUN $SPACK_ROOT/share/spack/docker/package-index/split.sh + +# nginx web service +FROM nginx:mainline-alpine +MAINTAINER Spack Maintainers +COPY --from=build-env --chown=nginx:nginx /build/packages /build/packages.json /usr/share/nginx/html/api/ +COPY share/spack/docker/package-index/cors-header.conf /etc/nginx/conf.d/ + +CMD ["nginx", "-g", "daemon off;"] diff --git a/share/spack/docker/package-index/README.rst b/share/spack/docker/package-index/README.rst new file mode 100644 index 0000000000..af5cde8025 --- /dev/null +++ b/share/spack/docker/package-index/README.rst @@ -0,0 +1,45 @@ +============================================ +The packages.spack.io Package Index REST API +============================================ + +This directory provides the docker recipe for the Spack package index on https://packages.spack.io + +On each merge to ``develop``, DockerHub builds a new image ``spack/packages.spack.io`` which is configured in: + https://cloud.docker.com/u/spack/repository/docker/spack/packages.spack.io/builds/edit + +------------ +The REST API +------------ + +The API is a simple, file-based JSON index. +A specific package can be queried via the URI syntax: +``https://packages.spack.io/api/:firstLetter/:packageName.json`` +which will return a HTTP status code ``200`` with a JSON file for all valid packages (content from ``spack list --format version_json``) and HTTP status code ``404`` for all other package names. + +Examples: + +- https://packages.spack.io/api/a/adios2.json +- https://packages.spack.io/api/p/py-pandas.json + +There is also the full index available at once under https://packages.spack.io/api/packages.json + +Current down-stream dependencies are, e.g. the https://shields.io service: + +- https://shields.io/category/version +- https://github.com/badges/shields/pull/3536 + +-------------------- +Local Build and Test +-------------------- + +Execute in your local Spack source root directory: + +.. code-block:: bash + + docker build -t spack/packages.spack.io:latest -f share/spack/docker/package-index/Dockerfile . + +Startup a local HTTP server on http://localhost:8080 via: + +.. code-block:: bash + + docker run -p 8080:80 spack/packages.spack.io:latest diff --git a/share/spack/docker/package-index/cors-header.conf b/share/spack/docker/package-index/cors-header.conf new file mode 100644 index 0000000000..77e34b60a4 --- /dev/null +++ b/share/spack/docker/package-index/cors-header.conf @@ -0,0 +1,3 @@ +# potentially add even more for preflight support +# https://enable-cors.org/server_nginx.html +add_header 'Access-Control-Allow-Origin' '*'; diff --git a/share/spack/docker/package-index/split.sh b/share/spack/docker/package-index/split.sh new file mode 100755 index 0000000000..2c499926cb --- /dev/null +++ b/share/spack/docker/package-index/split.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# split the package index in a small file tree of +# /p/package.json +# files with sub-directories grouped by the initial letter of the packages + +base_dir=$(pwd)/packages/ + +for pkg in $(cat packages.json | jq -c '.[]') +do + name="$(echo ${pkg} | jq -r '.name')"; + first_letter=${name::1} + mkdir -p ${base_dir}${first_letter}/ + echo ${pkg} > ${base_dir}${first_letter}/${name}.json +done diff --git a/share/spack/packages/Dockerfile b/share/spack/packages/Dockerfile deleted file mode 100644 index 3b9bd89ec2..0000000000 --- a/share/spack/packages/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:18.04 AS build-env -WORKDIR /build -RUN apt-get update && apt-get install -y jq -COPY packages.json ./ -COPY split.sh ./ -RUN /build/split.sh - -FROM nginx:mainline-alpine -COPY --from=build-env --chown=nginx:nginx /build/packages /build/packages.json /usr/share/nginx/html/api/ -COPY cors-header.conf /etc/nginx/conf.d/ - -CMD ["nginx", "-g", "daemon off;"] diff --git a/share/spack/packages/build-image.sh b/share/spack/packages/build-image.sh deleted file mode 100755 index adde5842ab..0000000000 --- a/share/spack/packages/build-image.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -script="$( basename "$0" )" -cd "$( dirname "$0" )" - -export IMAGE="spack/packages.spack.io:latest" - -if [ "$script" '=' 'push-image.sh' ] ; then - docker push "${IMAGE}" -else - docker build --no-cache --force-rm -t "${IMAGE}" . -fi diff --git a/share/spack/packages/cors-header.conf b/share/spack/packages/cors-header.conf deleted file mode 100644 index 77e34b60a4..0000000000 --- a/share/spack/packages/cors-header.conf +++ /dev/null @@ -1,3 +0,0 @@ -# potentially add even more for preflight support -# https://enable-cors.org/server_nginx.html -add_header 'Access-Control-Allow-Origin' '*'; diff --git a/share/spack/packages/push-image.sh b/share/spack/packages/push-image.sh deleted file mode 120000 index b3fd71be24..0000000000 --- a/share/spack/packages/push-image.sh +++ /dev/null @@ -1 +0,0 @@ -build-image.sh \ No newline at end of file diff --git a/share/spack/packages/split.sh b/share/spack/packages/split.sh deleted file mode 100755 index 2c499926cb..0000000000 --- a/share/spack/packages/split.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# split the package index in a small file tree of -# /p/package.json -# files with sub-directories grouped by the initial letter of the packages - -base_dir=$(pwd)/packages/ - -for pkg in $(cat packages.json | jq -c '.[]') -do - name="$(echo ${pkg} | jq -r '.name')"; - first_letter=${name::1} - mkdir -p ${base_dir}${first_letter}/ - echo ${pkg} > ${base_dir}${first_letter}/${name}.json -done diff --git a/share/spack/qa/run-docker-tests b/share/spack/qa/run-docker-tests deleted file mode 100755 index 769384c90e..0000000000 --- a/share/spack/qa/run-docker-tests +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -e -# -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# -# Description: -# Runs Spack docker tests. This builds a docker image for each of the -# configurations in share/spack/docker/config. -# -# Usage: -# run-docker-tests -# - -__login_attempted=0 -__login_success=1 -ensure_docker_login() { - if [ "$__login_attempted" '!=' '0' ] ; then - return $__login_success - fi - - echo "$DOCKER_PASSWORD" | \ - docker login -u "$DOCKER_USERNAME" --password-stdin - - if [ $? '=' '0' ] ; then - __login_success=0 - fi - - __login_attempted=1 - return $__login_success -} - -this_dir=$(cd $(dirname $0) && pwd) -SPACK_BIN="${this_dir}/../../../bin/spack" - -# packages.spack.io service -${SPACK_BIN} list --format version_json > ${this_dir}/../packages/packages.json -./share/spack/packages/build-image.sh -if [ "$TEST_SUITE" '=' "docker" -a \ - "$TRAVIS_EVENT_TYPE" != "pull_request" ] && ensure_docker_login ; then - ./share/spack/packages/push-image.sh -fi - -- cgit v1.2.3-60-g2f50