diff options
| -rw-r--r-- | .gitlab-ci.yml | 14 | ||||
| -rwxr-xr-x | bin/generate-gitlab-ci-yml.sh | 91 | ||||
| -rwxr-xr-x | bin/rebuild-index.sh | 13 | ||||
| -rwxr-xr-x | bin/rebuild-package.sh | 399 | ||||
| -rw-r--r-- | lib/spack/docs/index.rst | 1 | ||||
| -rw-r--r-- | lib/spack/docs/pipelines.rst | 439 | ||||
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 15 | ||||
| -rw-r--r-- | lib/spack/spack/ci.py (renamed from lib/spack/spack/cmd/release_jobs.py) | 452 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/buildcache.py | 99 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/ci.py | 482 | ||||
| -rw-r--r-- | lib/spack/spack/package.py | 3 | ||||
| -rw-r--r-- | lib/spack/spack/schema/gitlab_ci.py | 58 | ||||
| -rw-r--r-- | lib/spack/spack/test/ci.py | 167 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/ci.py | 582 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/release_jobs.py | 128 | ||||
| -rw-r--r-- | var/spack/gpg.mock/keys/package-signing-key | 1 |
16 files changed, 2166 insertions, 778 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 3fc9597a7c..0000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -generate ci jobs: - script: - - "./bin/generate-gitlab-ci-yml.sh" - tags: - - "spack-pre-ci" - artifacts: - paths: - - ci-generation - when: always diff --git a/bin/generate-gitlab-ci-yml.sh b/bin/generate-gitlab-ci-yml.sh deleted file mode 100755 index bb59199cf2..0000000000 --- a/bin/generate-gitlab-ci-yml.sh +++ /dev/null @@ -1,91 +0,0 @@ -#! /usr/bin/env bash - -# Remember where we are initially, it's the repo cloned by gitlab-ci -original_directory=$(pwd) -. "${original_directory}/share/spack/setup-env.sh" - -# Create a temporary working directory -temp_dir=$(mktemp -d) -trap 'rm -rf "$temp_dir"' INT TERM QUIT EXIT - -if [ -z "${DOWNSTREAM_CI_REPO}" ] ; then - echo "ERROR: missing variable: DOWNSTREAM_CI_REPO" >&2 - exit 1 -fi - -if [ -z "${SPACK_RELEASE_ENVIRONMENT_PATH}" ] ; then - echo "ERROR: missing variable: SPACK_RELEASE_ENVIRONMENT_PATH" >&2 - exit 1 -fi - -if [ -z "${CDASH_AUTH_TOKEN}" ] ; then - echo "WARNING: missing variable: CDASH_AUTH_TOKEN" >&2 -else - token_file="${temp_dir}/cdash_auth_token" - echo ${CDASH_AUTH_TOKEN} > ${token_file} -fi - -if [ -z "${SPACK_RELEASE_ENVIRONMENT_REPO}" ] ; then - echo "Assuming spack repo contains environment" >&2 - env_repo_dir=${original_directory} -else - echo "Cloning ${SPACK_RELEASE_ENVIRONMENT_REPO} into ${temp_dir}/envrepo" >&2 - cd ${temp_dir} - git clone ${SPACK_RELEASE_ENVIRONMENT_REPO} envrepo - cd envrepo - env_repo_dir=$(pwd) -fi - -current_branch="$CI_COMMIT_REF_NAME" - -# Because want to see generated gitlab-ci file as an artifact, -# we need to write it within the spack repo cloned by gitlab-ci. -gen_ci_dir="${original_directory}/ci-generation" -gen_ci_file="${gen_ci_dir}/.gitlab-ci.yml" -mkdir -p ${gen_ci_dir} - -env_dir="${env_repo_dir}/${SPACK_RELEASE_ENVIRONMENT_PATH}" - -if [ ! -f "${env_dir}/spack.yaml" ] ; then - echo "ERROR: Cannot find spack environment file in ${env_dir}" - exit 1 -fi - -cd $env_dir - -# The next commands generates the .gitlab-ci.yml (and optionally creates a -# buildgroup in cdash) -RELEASE_JOBS_ARGS=("--output-file" "${gen_ci_file}") -if [ ! -z "${token_file}" ]; then - RELEASE_JOBS_ARGS+=("--cdash-credentials" "${token_file}") -fi - -spack release-jobs "${RELEASE_JOBS_ARGS[@]}" - -if [[ $? -ne 0 ]]; then - echo "spack release-jobs command failed" - exit 1 -fi - -cd "$original_directory" -mv .git "$temp_dir/original-git-dir" -git init . - -git config user.email "robot@spack.io" -git config user.name "Spack Build Bot" - -cp ${gen_ci_file} "${original_directory}/.gitlab-ci.yml" -git add . - -echo "git commit" -commit_message="Auto-generated commit testing" -commit_message="${commit_message} ${current_branch} (${CI_COMMIT_SHA})" -git commit --message="${commit_message}" - -echo "git push" -git remote add origin "$DOWNSTREAM_CI_REPO" -git push --force origin "master:multi-ci-${current_branch}" - -rm -rf .git -mv "$temp_dir/original-git-dir" .git -git reset --hard HEAD diff --git a/bin/rebuild-index.sh b/bin/rebuild-index.sh deleted file mode 100755 index c4811e1f29..0000000000 --- a/bin/rebuild-index.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -set -x - -SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin" -export PATH="${SPACK_BIN_DIR}:${PATH}" - -spack buildcache update-index -d "$MIRROR_URL" diff --git a/bin/rebuild-package.sh b/bin/rebuild-package.sh deleted file mode 100755 index 747cbd9875..0000000000 --- a/bin/rebuild-package.sh +++ /dev/null @@ -1,399 +0,0 @@ -#!/bin/bash - -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -### -### This script represents a gitlab-ci job, corresponding to a single release -### spec. As such this script must first decide whether or not the spec it -### has been assigned is up to date on the remote binary mirror. If it is -### not (i.e. the source code has changed in a way that caused a change in the -### full_hash of the spec), this script will build the package, create a -### binary cache for it, and then push all related files to the remote binary -### mirror. This script also optionally communicates with a remote CDash -### instance to share status on the package build process. -### -### The following environment variables are (possibly) used within this script -### in order for the various elements function properly. -### -### First are two defaults we rely on from gitlab: -### -### CI_PROJECT_DIR -### CI_JOB_NAME -### -### The following must be set up in the variables section of gitlab: -### -### AWS_ACCESS_KEY_ID -### AWS_SECRET_ACCESS_KEY -### SPACK_SIGNING_KEY -### -### SPACK_S3_UPLOAD_MIRROR_URL // only required in the short term for the cloud case -### -### The following variabes are defined by the ci generation process and are -### required: -### -### SPACK_ENABLE_CDASH -### SPACK_ROOT_SPEC -### SPACK_MIRROR_URL -### SPACK_JOB_SPEC_PKG_NAME -### SPACK_COMPILER_ACTION -### -### Finally, these variables are optionally defined by the ci generation -### process, and may or may not be present: -### -### SPACK_CDASH_BASE_URL -### SPACK_CDASH_PROJECT -### SPACK_CDASH_PROJECT_ENC -### SPACK_CDASH_BUILD_NAME -### SPACK_CDASH_SITE -### SPACK_RELATED_BUILDS -### SPACK_JOB_SPEC_BUILDGROUP -### - -shopt -s expand_aliases - -export FORCE_UNSAFE_CONFIGURE=1 - -TEMP_DIR="${CI_PROJECT_DIR}/jobs_scratch_dir" - -JOB_LOG_DIR="${TEMP_DIR}/logs" -SPEC_DIR="${TEMP_DIR}/specs" -LOCAL_MIRROR="${CI_PROJECT_DIR}/local_mirror" -BUILD_CACHE_DIR="${LOCAL_MIRROR}/build_cache" -SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin" - -if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - CDASH_UPLOAD_URL="${SPACK_CDASH_BASE_URL}/submit.php?project=${SPACK_CDASH_PROJECT_ENC}" - DEP_JOB_RELATEBUILDS_URL="${SPACK_CDASH_BASE_URL}/api/v1/relateBuilds.php" - declare -a JOB_DEPS_PKG_NAMES -fi - -export SPACK_ROOT=${CI_PROJECT_DIR} -# export PATH="${SPACK_BIN_DIR}:${PATH}" -export GNUPGHOME="${CI_PROJECT_DIR}/opt/spack/gpg" - -. "${CI_PROJECT_DIR}/share/spack/setup-env.sh" - -mkdir -p ${JOB_LOG_DIR} -mkdir -p ${SPEC_DIR} - -cleanup() { - set +x - - if [ -z "$exit_code" ] ; then - - exit_code=$1 - if [ -z "$exit_code" ] ; then - exit_code=0 - fi - - restore_io - - if [ "$( type -t finalize )" '=' 'function' ] ; then - finalize "$JOB_LOG_DIR/cdash_log.txt" - fi - - # We can clean these out later on, once we have a good sense for - # how the logging infrastructure is working - # rm -rf "$JOB_LOG_DIR" - fi - - \exit $exit_code -} - -alias exit='cleanup' - -begin_logging() { - trap "cleanup 1; \\exit \$exit_code" INT TERM QUIT - trap "cleanup 0; \\exit \$exit_code" EXIT - - rm -rf "$JOB_LOG_DIR/cdash_log.txt" - - # NOTE: Here, some redirects are set up - exec 3>&1 # fd 3 is now a dup of stdout - exec 4>&2 # fd 4 is now a dup of stderr - - # stdout and stderr are joined and redirected to the log - exec &> "$JOB_LOG_DIR/cdash_log.txt" - - set -x -} - -restore_io() { - exec >&- - exec 2>&- - - exec >&3 - exec 2>&4 - - exec 3>&- - exec 4>&- -} - -finalize() { - # If you define a finalize function: - # - it will always be called at the very end of the script - # - the log file will be passed in as the first argument, and - # - the code in this function will not be logged. - echo "The full log file is located at $1" - # TODO: send this log data to cdash! -} - -check_error() -{ - local last_exit_code=$1 - local last_cmd=$2 - if [[ ${last_exit_code} -ne 0 ]]; then - echo "${last_cmd} exited with code ${last_exit_code}" - echo "TERMINATING JOB" - exit 1 - else - echo "${last_cmd} completed successfully" - fi -} - -extract_build_id() -{ - LINES_TO_SEARCH=$1 - regex="buildSummary\.php\?buildid=([[:digit:]]+)" - SINGLE_LINE_OUTPUT=$(echo ${LINES_TO_SEARCH} | tr -d '\n') - - if [[ ${SINGLE_LINE_OUTPUT} =~ ${regex} ]]; then - echo "${BASH_REMATCH[1]}" - else - echo "NONE" - fi -} - -get_relate_builds_post_data() -{ - cat <<EOF -{ - "project": "${1}", - "buildid": ${2}, - "relatedid": ${3}, - "relationship": "depends on" -} -EOF -} - -gen_full_specs_for_job_and_deps() { - SPEC_YAML_PATH="${SPEC_DIR}/${SPACK_JOB_SPEC_PKG_NAME}.yaml" - local spec_names_to_save="${SPACK_JOB_SPEC_PKG_NAME}" - - if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - IFS=';' read -ra DEPS <<< "${SPACK_RELATED_BUILDS}" - for i in "${DEPS[@]}"; do - depPkgName="${i}" - spec_names_to_save="${spec_names_to_save} ${depPkgName}" - JOB_DEPS_PKG_NAMES+=("${depPkgName}") - done - fi - - if [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then - # This corresponds to a bootstrapping phase where we need to - # rely on any available compiler to build the package (i.e. the - # compiler needed to be stripped from the spec), and thus we need - # to concretize the root spec again. - spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec "${SPACK_ROOT_SPEC}" --yaml-dir "${SPEC_DIR}" - else - # in this case, either we're relying on Spack to install missing compiler - # bootstrapped in a previous phase, or else we only had one phase (like a - # site which already knows what compilers are available on it's runners), - # so we don't want to concretize that root spec again. The reason we need - # this in the first case (bootstrapped compiler), is that we can't concretize - # a spec at this point if we're going to ask spack to "install_missing_compilers". - tmp_dir=$(mktemp -d) - TMP_YAML_PATH="${tmp_dir}/root.yaml" - ROOT_SPEC_YAML=$(spack python -c "import base64 ; import zlib ; print(str(zlib.decompress(base64.b64decode('${SPACK_ROOT_SPEC}')).decode('utf-8')))") - echo "${ROOT_SPEC_YAML}" > "${TMP_YAML_PATH}" - spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec-yaml "${TMP_YAML_PATH}" --yaml-dir "${SPEC_DIR}" - rm -rf ${tmp_dir} - fi -} - -begin_logging - -echo "Running job for spec: ${CI_JOB_NAME}" - -# This should create the directory we referred to as GNUPGHOME earlier -spack gpg list - -# Importing the secret key using gpg2 directly should allow to -# sign and verify both -set +x -KEY_IMPORT_RESULT=`echo ${SPACK_SIGNING_KEY} | base64 --decode | gpg2 --import` -check_error $? "gpg2 --import" -set -x - -spack gpg list --trusted -spack gpg list --signing - -# To have spack install missing compilers, we need to add a custom -# configuration scope, then we pass that to the package installation -# command -CUSTOM_CONFIG_SCOPE_DIR="${TEMP_DIR}/config_scope" -mkdir -p "${CUSTOM_CONFIG_SCOPE_DIR}" -CUSTOM_CONFIG_SCOPE_ARG="" - -if [ "${SPACK_COMPILER_ACTION}" == "INSTALL_MISSING" ]; then - echo "Make sure bootstrapped compiler will be installed" - custom_config_file_path="${CUSTOM_CONFIG_SCOPE_DIR}/config.yaml" - cat <<CONFIG_STUFF > "${custom_config_file_path}" -config: - install_missing_compilers: true -CONFIG_STUFF - CUSTOM_CONFIG_SCOPE_ARG="-C ${CUSTOM_CONFIG_SCOPE_DIR}" - # Configure the binary mirror where, if needed, this jobs compiler - # was installed in binary pacakge form, then tell spack to - # install_missing_compilers. -elif [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then - echo "Just find any available compiler" - spack compiler find -else - echo "No compiler action to be taken" -fi - -# Finally, list the compilers spack knows about -echo "Compiler Configurations:" -spack config get compilers - -# Write full-deps yamls for this job spec and its dependencies -gen_full_specs_for_job_and_deps - -# Make the build_cache directory if it doesn't exist -mkdir -p "${BUILD_CACHE_DIR}" - -# Get buildcache name so we can write a CDash build id file in the right place. -# If we're unable to get the buildcache name, we may have encountered a problem -# concretizing the spec, or some other issue that will eventually cause the job -# to fail. -JOB_BUILD_CACHE_ENTRY_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${SPEC_YAML_PATH}"` -if [[ $? -ne 0 ]]; then - echo "ERROR, unable to get buildcache entry name for job ${CI_JOB_NAME}" - exit 1 -fi - -if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - # Whether we have to build the spec or download it pre-built, we expect to find - # the cdash build id file sitting in this location afterwards. - JOB_CDASH_ID_FILE="${BUILD_CACHE_DIR}/${JOB_BUILD_CACHE_ENTRY_NAME}.cdashid" -fi - -# Finally, we can check the spec we have been tasked with build against -# the built binary on the remote mirror to see if it needs to be rebuilt -spack -d buildcache check --spec-yaml "${SPEC_YAML_PATH}" --mirror-url "${SPACK_MIRROR_URL}" --rebuild-on-error - -if [[ $? -ne 0 ]]; then - # Configure mirror - spack mirror add local_artifact_mirror "file://${LOCAL_MIRROR}" - - if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - JOB_CDASH_ID="NONE" - - # Install package, using the buildcache from the local mirror to - # satisfy dependencies. - BUILD_ID_LINE=`spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage --cdash-upload-url "${CDASH_UPLOAD_URL}" --cdash-build "${SPACK_CDASH_BUILD_NAME}" --cdash-site "${SPACK_CDASH_SITE}" --cdash-track "${SPACK_JOB_SPEC_BUILDGROUP}" -f "${SPEC_YAML_PATH}" | grep "buildSummary\\.php"` - check_error $? "spack install" - - # By parsing the output of the "spack install" command, we can get the - # buildid generated for us by CDash - JOB_CDASH_ID=$(extract_build_id "${BUILD_ID_LINE}") - - # Write the .cdashid file to the buildcache as well - echo "${JOB_CDASH_ID}" >> ${JOB_CDASH_ID_FILE} - else - spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage -f "${SPEC_YAML_PATH}" - fi - - # Copy some log files into an artifact location, once we have a way - # to provide a spec.yaml file to more spack commands (e.g. "location") - # stage_dir=$(spack location --stage-dir -f "${SPEC_YAML_PATH}") - # build_log_file=$(find -L "${stage_dir}" | grep "spack-build\\.out") - # config_log_file=$(find -L "${stage_dir}" | grep "config\\.log") - # cp "${build_log_file}" "${JOB_LOG_DIR}/" - # cp "${config_log_file}" "${JOB_LOG_DIR}/" - - # Create buildcache entry for this package, reading the spec from the yaml - # file. - spack -d buildcache create --spec-yaml "${SPEC_YAML_PATH}" -a -f -d "${LOCAL_MIRROR}" --no-rebuild-index - check_error $? "spack buildcache create" - - # TODO: The upload-s3 command should eventually be replaced with something - # like: "spack buildcache put <mirror> <spec>", when that subcommand is - # properly implemented. - if [ ! -z "${SPACK_S3_UPLOAD_MIRROR_URL}" ] ; then - spack -d upload-s3 spec --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --endpoint-url "${SPACK_S3_UPLOAD_MIRROR_URL}" - check_error $? "spack upload-s3 spec" - else - spack -d buildcache copy --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --destination-url "${SPACK_MIRROR_URL}" - fi -else - echo "spec ${CI_JOB_NAME} is already up to date on remote mirror, downloading it" - - # Configure remote mirror so we can download buildcache entry - spack mirror add remote_binary_mirror ${SPACK_MIRROR_URL} - - # Now download it - BUILDCACHE_DL_ARGS=("--spec-yaml" "${SPEC_YAML_PATH}" "--path" "${BUILD_CACHE_DIR}/" ) - if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - BUILDCACHE_DL_ARGS+=( "--require-cdashid" ) - fi - spack -d buildcache download "${BUILDCACHE_DL_ARGS[@]}" - check_error $? "spack buildcache download" -fi - -# The next step is to relate this job to the jobs it depends on -if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - if [ -f "${JOB_CDASH_ID_FILE}" ]; then - JOB_CDASH_BUILD_ID=$(<${JOB_CDASH_ID_FILE}) - - if [ "${JOB_CDASH_BUILD_ID}" == "NONE" ]; then - echo "ERROR: unable to read this jobs id from ${JOB_CDASH_ID_FILE}" - exit 1 - fi - - # Now get CDash ids for dependencies and "relate" each dependency build - # with this jobs build - for DEP_PKG_NAME in "${JOB_DEPS_PKG_NAMES[@]}"; do - echo "Getting cdash id for dependency --> ${DEP_PKG_NAME} <--" - DEP_SPEC_YAML_PATH="${SPEC_DIR}/${DEP_PKG_NAME}.yaml" - DEP_JOB_BUILDCACHE_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${DEP_SPEC_YAML_PATH}"` - - if [[ $? -eq 0 ]]; then - DEP_JOB_ID_FILE="${BUILD_CACHE_DIR}/${DEP_JOB_BUILDCACHE_NAME}.cdashid" - echo "DEP_JOB_ID_FILE path = ${DEP_JOB_ID_FILE}" - - if [ -f "${DEP_JOB_ID_FILE}" ]; then - DEP_JOB_CDASH_BUILD_ID=$(<${DEP_JOB_ID_FILE}) - echo "File ${DEP_JOB_ID_FILE} contained value ${DEP_JOB_CDASH_BUILD_ID}" - echo "Relating builds -> ${SPACK_CDASH_BUILD_NAME} (buildid=${JOB_CDASH_BUILD_ID}) depends on ${DEP_PKG_NAME} (buildid=${DEP_JOB_CDASH_BUILD_ID})" - relateBuildsPostBody="$(get_relate_builds_post_data "${SPACK_CDASH_PROJECT}" ${JOB_CDASH_BUILD_ID} ${DEP_JOB_CDASH_BUILD_ID})" - relateBuildsResult=`curl "${DEP_JOB_RELATEBUILDS_URL}" -H "Content-Type: application/json" -H "Accept: application/json" -d "${relateBuildsPostBody}"` - echo "Result of curl request: ${relateBuildsResult}" - else - echo "ERROR: Did not find expected .cdashid file for dependency: ${DEP_JOB_ID_FILE}" - exit 1 - fi - else - echo "ERROR: Unable to get buildcache entry name for ${DEP_SPEC_NAME}" - exit 1 - fi - done - else - echo "ERROR: Did not find expected .cdashid file ${JOB_CDASH_ID_FILE}" - exit 1 - fi -fi - -# Show the size of the buildcache and a list of what's in it, directly -# in the gitlab log output -( - restore_io - du -sh ${BUILD_CACHE_DIR} - find ${BUILD_CACHE_DIR} -maxdepth 3 -type d -ls -) - -echo "End of rebuild package script" diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index a8534ec1bf..489c15645a 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -74,6 +74,7 @@ or refer to the full manual below. package_list chain extensions + pipelines .. toctree:: :maxdepth: 2 diff --git a/lib/spack/docs/pipelines.rst b/lib/spack/docs/pipelines.rst new file mode 100644 index 0000000000..f70b39a16d --- /dev/null +++ b/lib/spack/docs/pipelines.rst @@ -0,0 +1,439 @@ +.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _pipelines: + +========= +Pipelines +========= + +Spack provides commands that support generating and running automated build +pipelines designed for Gitlab CI. At the highest level it works like this: +provide a spack environment describing the set of packages you care about, +and include within that environment file a description of how those packages +should be mapped to Gitlab runners. Spack can then generate a ``.gitlab-ci.yml`` +file containing job descriptions for all your packages that can be run by a +properly configured Gitlab CI instance. When run, the generated pipeline will +build and deploy binaries, and it can optionally report to a CDash instance +regarding the health of the builds as they evolve over time. + +------------------------------ +Getting started with pipelines +------------------------------ + +It is fairly straightforward to get started with automated build pipelines. At +a minimum, you'll need to set up a Gitlab instance (more about Gitlab CI +`here <https://about.gitlab.com/product/continuous-integration/>`_) and configure +at least one `runner <https://docs.gitlab.com/runner/>`_. Then the basic steps +for setting up a build pipeline are as follows: + +#. Create a repository on your gitlab instance +#. Add a ``spack.yaml`` at the root containing your pipeline environment (see + below for details) +#. Add a ``.gitlab-ci.yml`` at the root containing a single job, similar to + this one: + + .. code-block:: yaml + + pipeline-job: + tags: + - <custom-tag> + ... + script: + - spack ci start + +#. Add any secrets required by the CI process to environment variables using the + CI web ui +#. Push a commit containing the ``spack.yaml`` and ``.gitlab-ci.yml`` mentioned above + to the gitlab repository + +The ``<custom-tag>``, above, is used to pick one of your configured runners, +while the use of the ``spack ci start`` command implies that runner has an +appropriate version of spack installed and configured for use. Of course, there +are myriad ways to customize the process. You can configure CDash reporting +on the progress of your builds, set up S3 buckets to mirror binaries built by +the pipeline, clone a custom spack repository/ref for use by the pipeline, and +more. + +While it is possible to set up pipelines on gitlab.com, the builds there are +limited to 60 minutes and generic hardware. It is also possible to +`hook up <https://about.gitlab.com/blog/2018/04/24/getting-started-gitlab-ci-gcp>`_ +Gitlab to Google Kubernetes Engine (`GKE <https://cloud.google.com/kubernetes-engine/>`_) +or Amazon Elastic Kubernetes Service (`EKS <https://aws.amazon.com/eks>`_), though those +topics are outside the scope of this document. + +----------------------------------- +Spack commands supporting pipelines +----------------------------------- + +Spack provides a command `ci` with sub-commands for doing various things related +to automated build pipelines. All of the ``spack ci ...`` commands must be run +from within a environment, as each one makes use of the environment for different +purposes. Additionally, some options to the commands (or conditions present in +the spack environment file) may require particular environment variables to be +set in order to function properly. Examples of these are typically secrets +needed for pipeline operation that should not be visible in a spack environment +file. These environment variables are described in more detail +:ref:`ci_environment_variables`. + +.. _cmd_spack_ci: + +^^^^^^^^^^^^^^^^^^ +``spack ci`` +^^^^^^^^^^^^^^^^^^ + +Super-command for functionality related to generating pipelines and executing +pipeline jobs. + +.. _cmd_spack_ci_start: + +^^^^^^^^^^^^^^^^^^ +``spack ci start`` +^^^^^^^^^^^^^^^^^^ + +Currently this command is a short-cut to first run ``spack ci generate``, followed +by ``spack ci pushyaml``. + +.. _cmd_spack_ci_generate: + +^^^^^^^^^^^^^^^^^^^^^ +``spack ci generate`` +^^^^^^^^^^^^^^^^^^^^^ + +Concretizes the specs in the active environment, stages them (as described in +:ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk. + +.. _cmd_spack_ci_pushyaml: + +^^^^^^^^^^^^^^^^^^^^^ +``spack ci pushyaml`` +^^^^^^^^^^^^^^^^^^^^^ + +Generates a commit containing the generated ``.gitlab-ci.yml`` and pushes it to a +``DOWNSTREAM_CI_REPO``, which is frequently the same repository. The branch +created has the same name as the current branch being tested, but has ``multi-ci-`` +prepended to the branch name. Once Gitlab CI has full support for dynamically +defined workloads, this command will be deprecated. + +.. _cmd_spack_ci_rebuild: + +^^^^^^^^^^^^^^^^^^^^ +``spack ci rebuild`` +^^^^^^^^^^^^^^^^^^^^ + +This sub-command is responsible for ensuring a single spec from the release +environment is up to date on the remote mirror configured in the environment, +and as such, corresponds to a single job in the ``.gitlab-ci.yml`` file. + +------------------------------------ +A pipeline-enabled spack environment +------------------------------------ + +Here's an example of a spack environment file that has been enhanced with +sections desribing a build pipeline: + +.. code-block:: yaml + + spack: + definitions: + - pkgs: + - readline@7.0 + - compilers: + - '%gcc@5.5.0' + - oses: + - os=ubuntu18.04 + - os=centos7 + specs: + - matrix: + - [$pkgs] + - [$compilers] + - [$oses] + mirrors: + cloud_gitlab: https://mirror.spack.io + gitlab-ci: + mappings: + - match: + - os=ubuntu18.04 + runner-attributes: + tags: + - spack-k8s + image: spack/spack_builder_ubuntu_18.04 + - match: + - os=centos7 + runner-attributes: + tags: + - spack-k8s + image: spack/spack_builder_centos_7 + cdash: + build-group: Release Testing + url: https://cdash.spack.io + project: Spack + site: Spack AWS Gitlab Instance + +Hopefully, the ``definitions``, ``specs``, ``mirrors``, etc. sections are already +familiar, as they are part of spack :ref:`environments`. So let's take a more +in-depth look some of the pipeline-related sections in that environment file +that might not be as familiar. + +The ``gitlab-ci`` section is used to configure how the pipeline workload should be +generated, mainly how the jobs for building specs should be assigned to the +configured runners on your instance. Each entry within the list of ``mappings`` +corresponds to a known gitlab runner, where the ``match`` section is used +in assigning a release spec to one of the runners, and the ``runner-attributes`` +section is used to configure the spec/job for that particular runner. + +There are other pipeline options you can configure within the ``gitlab-ci`` section +as well. The ``bootstrap`` section allows you to specify lists of specs from +your ``definitions`` that should be staged ahead of the environment's ``specs`` (this +section is described in more detail below). The ``enable-artifacts-buildcache`` key +takes a boolean and determines whether the pipeline uses artifacts to store and +pass along the buildcaches from one stage to the next (the default if you don't +provide this option is ``False``). The ``enable-debug-messages`` key takes a boolean +and allows you to choose whether the pipeline build jobs are run as ``spack -d ci rebuild`` +or just ``spack ci rebuild`` (the default is not to enable debug messages). The +``final-stage-rebuild-index`` section controls whether an extra job is added to the +end of your pipeline (in a stage by itself) which will regenerate the mirror's +buildcache index. Under normal operation, each pipeline job that rebuilds a package +will re-generate the mirror's buildcache index after the buildcache entry for that +job has been created and pushed to the mirror. Since jobs in the same stage can run in +parallel, there is the possibility that at the end of some stage, the index may not +reflect all the binaries in the buildcache. Adding the ``final-stage-rebuild-index`` +section ensures that at the end of the pipeline, the index will be in sync with the +binaries on the mirror. If the mirror lives in an S3 bucket, this job will need to +run on a machine with the Python ``boto3`` module installed, and consequently the +``final-stage-rebuild-index`` needs to specify a list of ``tags`` to pick a runner +satisfying that condition. It can also take an ``image`` key so Docker executor type +runners can pick the right image for the index regeneration job. + +The optional ``cdash`` section provides information that will be used by the +``spack ci generate`` command (invoked by ``spack ci start``) for reporting +to CDash. All the jobs generated from this environment will belong to a +"build group" within CDash that can be tracked over time. As the release +progresses, this build group may have jobs added or removed. The url, project, +and site are used to specify the CDash instance to which build results should +be reported. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Assignment of specs to runners +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``mappings`` section corresponds to a list of runners, and during assignment +of specs to runners, the list is traversed in order looking for matches, the +first runner that matches a release spec is assigned to build that spec. The +``match`` section within each runner mapping section is a list of specs, and +if any of those specs match the release spec (the ``spec.satisfies()`` method +is used), then that runner is considered a match. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Configuration of specs/jobs for a runner +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once a runner has been chosen to build a release spec, the ``runner-attributes`` +section provides information determining details of the job in the context of +the runner. The ``runner-attributes`` section must have a ``tags`` key, which +is a list containing at least one tag used to select the runner from among the +runners known to the gitlab instance. For Docker executor type runners, the +``image`` key is used to specify the Docker image used to build the release spec +(and could also appear as a dictionary with a ``name`` specifying the image name, +as well as an ``entrypoint`` to override whatever the default for that image is). +For other types of runners the ``variables`` key will be useful to pass any +information on to the runner that it needs to do its work (e.g. scheduler +parameters, etc.). + +.. _staging_algorithm: + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Summary of ``.gitlab-ci.yml`` generation algorithm +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +All specs yielded by the matrix (or all the specs in the environment) have their +dependencies computed, and the entire resulting set of specs are staged together +before being run through the ``gitlab-ci/mappings`` entries, where each staged +spec is assigned a runner. "Staging" is the name we have given to the process +of figuring out in what order the specs should be built, taking into consideration +Gitlab CI rules about jobs/stages. In the staging process the goal is to maximize +the number of jobs in any stage of the pipeline, while ensuring that the jobs in +any stage only depend on jobs in previous stages (since those jobs are guaranteed +to have completed already). As a runner is determined for a job, the information +in the ``runner-attributes`` is used to populate various parts of the job +description that will be used by Gitlab CI. Once all the jobs have been assigned +a runner, the ``.gitlab-ci.yml`` is written to disk. + +The short example provided above would result in the ``readline``, ``ncurses``, +and ``pkgconf`` packages getting staged and built on the runner chosen by the +``spack-k8s`` tag. In this example, we assume the runner is a Docker executor +type runner, and thus certain jobs will be run in the ``centos7`` container, +and others in the ``ubuntu-18.04`` container. The resulting ``.gitlab-ci.yml`` +will contain 6 jobs in three stages. Once the jobs have been generated, the +presence of a ``SPACK_CDASH_AUTH_TOKEN`` environment variable during the +``spack ci generate`` command would result in all of the jobs being put in a +build group on CDash called "Release Testing" (that group will be created if +it didn't already exist). + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Optional compiler bootstrapping +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Spack pipelines also have support for bootstrapping compilers on systems that +may not already have the desired compilers installed. The idea here is that +you can specify a list of things to bootstrap in your ``definitions``, and +spack will guarantee those will be installed in a phase of the pipeline before +your release specs, so that you can rely on those packages being available in +the binary mirror when you need them later on in the pipeline. At the moment +the only viable use-case for bootstrapping is to install compilers. + +Here's an example of what bootstrapping some compilers might look like: + +.. code-block:: yaml + + spack: + definitions: + - compiler-pkgs: + - 'llvm+clang@6.0.1 os=centos7' + - 'gcc@6.5.0 os=centos7' + - 'llvm+clang@6.0.1 os=ubuntu18.04' + - 'gcc@6.5.0 os=ubuntu18.04' + - pkgs: + - readline@7.0 + - compilers: + - '%gcc@5.5.0' + - '%gcc@6.5.0' + - '%gcc@7.3.0' + - '%clang@6.0.0' + - '%clang@6.0.1' + - oses: + - os=ubuntu18.04 + - os=centos7 + specs: + - matrix: + - [$pkgs] + - [$compilers] + - [$oses] + exclude: + - '%gcc@7.3.0 os=centos7' + - '%gcc@5.5.0 os=ubuntu18.04' + gitlab-ci: + bootstrap: + - name: compiler-pkgs + compiler-agnostic: true + mappings: + # mappings similar to the example higher up in this description + ... + +In the example above, we have added a list to the ``definitions`` called +``compiler-pkgs`` (you can add any number of these), which lists compiler packages +we want to be staged ahead of the full matrix of release specs (which consists +only of readline in our example). Then within the ``gitlab-ci`` section, we +have added a ``bootstrap`` section, which can contain a list of items, each +referring to a list in the ``definitions`` section. These items can either +be a dictionary or a string. If you supply a dictionary, it must have a name +key whose value must match one of the lists in definitions and it can have a +``compiler-agnostic`` key whose value is a boolean. If you supply a string, +then it needs to match one of the lists provided in ``definitions``. You can +think of the bootstrap list as an ordered list of pipeline "phases" that will +be staged before your actual release specs. While this introduces another +layer of bottleneck in the pipeline (all jobs in all stages of one phase must +complete before any jobs in the next phase can begin), it also means you are +guaranteed your bootstrapped compilers will be available when you need them. + +The ``compiler-agnostic`` key can be provided with each item in the +bootstrap list. It tells the ``spack ci generate`` command that any jobs staged +from that particular list should have the compiler removed from the spec, so +that any compiler available on the runner where the job is run can be used to +build the package. + +When including a bootstrapping phase as in the example above, the result is that +the bootstrapped compiler packages will be pushed to the binary mirror (and the +local artifacts mirror) before the actual release specs are built. In this case, +the jobs corresponding to subsequent release specs are configured to +``install_missing_compilers``, so that if spack is asked to install a package +with a compiler it doesn't know about, it can be quickly installed from the +binary mirror first. + +Since bootstrapping compilers is optional, those items can be left out of the +environment/stack file, and in that case no bootstrapping will be done (only the +specs will be staged for building) and the runners will be expected to already +have all needed compilers installed and configured for spack to use. + +------------------------------------- +Using a custom spack in your pipeline +------------------------------------- + +If your runners will not have a version of spack ready to invoke, or if for some +other reason you want to use a custom version of spack to run your pipelines, +this can be accomplished fairly simply. First, create CI environment variables +containing the url and branch/tag you want to clone (calling them, for example, +``SPACK_REPO`` and ``SPACK_REF``), use them to clone spack in your pre-ci +``before_script``, and finally pass those same values along to the workload +generation process via the ``spack-repo`` and ``spack-ref`` cli args. Here's +an example: + +.. code-block:: yaml + + pipeline-job: + tags: + - <some-other-tag> + before_script: + - git clone ${SPACK_REPO} --branch ${SPACK_REF} + - . ./spack/share/spack/setup-env.sh + script: + - spack ci start --spack-repo ${SPACK_REPO} --spack-ref ${SPACK_REF} <...args> + after_script: + - rm -rf ./spack + +If the ``spack ci start`` command receives those extra command line arguments, +then it adds similar ``before_script`` and ``after_script`` sections for each of +the ``spack ci rebuild`` jobs it generates (cloning and sourcing a custom +spack in the ``before_script`` and removing it again in the ``after_script``). +This gives you control over the version of spack used when the rebuild jobs +are actually run on the gitlab runner. + +.. _ci_environment_variables: + +-------------------------------------------------- +Environment variables affecting pipeline operation +-------------------------------------------------- + +Certain secrets and some other information should be provided to the pipeline +infrastructure via environment variables, usually for reasons of security, but +in some cases to support other pipeline use cases such as PR testing. The +environment variables used by the pipeline infrastructure are described here. + +^^^^^^^^^^^^^^^^^ +AWS_ACCESS_KEY_ID +^^^^^^^^^^^^^^^^^ + +Needed when binary mirror is an S3 bucket. + +^^^^^^^^^^^^^^^^^^^^^ +AWS_SECRET_ACCESS_KEY +^^^^^^^^^^^^^^^^^^^^^ + +Needed when binary mirror is an S3 bucket. + +^^^^^^^^^^^^^^^ +S3_ENDPOINT_URL +^^^^^^^^^^^^^^^ + +Needed when binary mirror is an S3 bucket that is *not* on AWS. + +^^^^^^^^^^^^^^^^^ +CDASH_AUTH_TOKEN +^^^^^^^^^^^^^^^^^ + +Needed in order to report build groups to CDash. + +^^^^^^^^^^^^^^^^^ +SPACK_SIGNING_KEY +^^^^^^^^^^^^^^^^^ + +Needed to sign/verify binary packages from the remote binary mirror. + +^^^^^^^^^^^^^^^^^^ +DOWNSTREAM_CI_REPO +^^^^^^^^^^^^^^^^^^ + +Needed until Gitlab CI supports dynamic job generation. Can contain connection +credentials, and could be the same repository or a different one. diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 6f3efc2acf..37190a549c 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -803,10 +803,10 @@ def needs_rebuild(spec, mirror_url, rebuild_on_errors=False): try: _, _, yaml_file = web_util.read_from_url(file_path) yaml_contents = codecs.getreader('utf-8')(yaml_file).read() - except URLError as url_err: + except (URLError, web_util.SpackWebError) as url_err: err_msg = [ 'Unable to determine whether {0} needs rebuilding,', - ' caught URLError attempting to read from {1}.', + ' caught exception attempting to read from {1}.', ] tty.error(''.join(err_msg).format(spec.short_spec, file_path)) tty.debug(url_err) @@ -908,11 +908,16 @@ def _download_buildcache_entry(mirror_root, descriptions): return True -def download_buildcache_entry(file_descriptions): - if not spack.mirror.MirrorCollection(): - tty.die("Please add a spack mirror to allow " + +def download_buildcache_entry(file_descriptions, mirror_url=None): + if not mirror_url and not spack.mirror.MirrorCollection(): + tty.die("Please provide or add a spack mirror to allow " + "download of buildcache entries.") + if mirror_url: + mirror_root = os.path.join( + mirror_url, _build_cache_relative_path) + return _download_buildcache_entry(mirror_root, file_descriptions) + for mirror in spack.mirror.MirrorCollection().values(): mirror_root = os.path.join( mirror.fetch_url, diff --git a/lib/spack/spack/cmd/release_jobs.py b/lib/spack/spack/ci.py index cdd3b10dc3..ed06524073 100644 --- a/lib/spack/spack/cmd/release_jobs.py +++ b/lib/spack/spack/ci.py @@ -4,7 +4,11 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import base64 +import datetime import json +import os +import shutil +import tempfile import zlib from six import iteritems @@ -14,31 +18,36 @@ from six.moves.urllib.request import build_opener, HTTPHandler, Request import llnl.util.tty as tty -import spack.environment as ev +import spack +import spack.binary_distribution as bindist +import spack.cmd.buildcache as buildcache import spack.compilers as compilers +import spack.config as cfg +import spack.environment as ev from spack.dependency import all_deptypes from spack.error import SpackError import spack.hash_types as ht +from spack.main import SpackCommand +import spack.repo from spack.spec import Spec import spack.util.spack_yaml as syaml +import spack.util.web as web_util -description = "generate release build set as .gitlab-ci.yml" -section = "build" -level = "long" +spack_gpg = SpackCommand('gpg') +spack_compiler = SpackCommand('compiler') -def setup_parser(subparser): - subparser.add_argument( - '-o', '--output-file', default=".gitlab-ci.yml", - help="path to output file to write") - subparser.add_argument( - '-p', '--print-summary', action='store_true', default=False, - help="Print summary of staged jobs to standard output") +class TemporaryDirectory(object): + def __init__(self): + self.temporary_directory = tempfile.mkdtemp() - subparser.add_argument( - '--cdash-credentials', default=None, - help="Path to file containing CDash authentication token") + def __enter__(self): + return self.temporary_directory + + def __exit__(self, exc_type, exc_value, exc_traceback): + shutil.rmtree(self.temporary_directory) + return False def _create_buildgroup(opener, headers, url, project, group_name, group_type): @@ -131,6 +140,10 @@ def get_job_name(phase, strip_compiler, spec, osarch, build_group): format_args.append(spec.name) item_idx += 1 + format_str += '/{{{0}}}'.format(item_idx) + format_args.append(spec.dag_hash(7)) + item_idx += 1 + format_str += ' {{{0}}}'.format(item_idx) format_args.append(spec.version) item_idx += 1 @@ -368,6 +381,10 @@ def compute_spec_deps(spec_list): rkey, rlabel = spec_deps_key_label(spec) for s in spec.traverse(deptype=deptype): + if s.external: + tty.msg('Will not stage external pkg: {0}'.format(s)) + continue + skey, slabel = spec_deps_key_label(s) spec_labels[slabel] = { 'spec': get_spec_string(s), @@ -377,6 +394,10 @@ def compute_spec_deps(spec_list): for d in s.dependencies(deptype=deptype): dkey, dlabel = spec_deps_key_label(d) + if d.external: + tty.msg('Will not stage external dep: {0}'.format(d)) + continue + append_dep(slabel, dlabel) for l, d in spec_labels.items(): @@ -406,15 +427,28 @@ def find_matching_config(spec, ci_mappings): return None -def release_jobs(parser, args): - env = ev.get_env(args, 'release-jobs', required=True) +def pkg_name_from_spec_label(spec_label): + return spec_label[:spec_label.index('/')] + + +def generate_gitlab_ci_yaml(env, print_summary, output_file, + custom_spack_repo=None, custom_spack_ref=None): + # FIXME: What's the difference between one that opens with 'spack' + # and one that opens with 'env'? This will only handle the former. + with spack.concretize.disable_compiler_existence_check(): + env.concretize() yaml_root = ev.config_dict(env.yaml) if 'gitlab-ci' not in yaml_root: tty.die('Environment yaml does not have "gitlab-ci" section') - ci_mappings = yaml_root['gitlab-ci']['mappings'] + gitlab_ci = yaml_root['gitlab-ci'] + ci_mappings = gitlab_ci['mappings'] + + final_job_config = None + if 'final-stage-rebuild-index' in gitlab_ci: + final_job_config = gitlab_ci['final-stage-rebuild-index'] build_group = None enable_cdash_reporting = False @@ -426,23 +460,40 @@ def release_jobs(parser, args): build_group = ci_cdash['build-group'] cdash_url = ci_cdash['url'] cdash_project = ci_cdash['project'] - proj_enc = urlencode({'project': cdash_project}) - eq_idx = proj_enc.find('=') + 1 - cdash_project_enc = proj_enc[eq_idx:] cdash_site = ci_cdash['site'] - if args.cdash_credentials: - with open(args.cdash_credentials) as fd: - cdash_auth_token = fd.read() - cdash_auth_token = cdash_auth_token.strip() + if 'SPACK_CDASH_AUTH_TOKEN' in os.environ: + tty.verbose("Using CDash auth token from environment") + cdash_auth_token = os.environ.get('SPACK_CDASH_AUTH_TOKEN') + + # Make sure we use a custom spack if necessary + before_script = None + after_script = None + if custom_spack_repo: + if not custom_spack_ref: + custom_spack_ref = 'master' + before_script = [ + ('git clone "{0}" --branch "{1}" --depth 1 ' + '--single-branch'.format(custom_spack_repo, custom_spack_ref)), + # Next line just shows spack version in pipeline output + 'pushd ./spack && git rev-parse HEAD && popd', + '. "./spack/share/spack/setup-env.sh"', + ] + after_script = [ + 'rm -rf "./spack"' + ] ci_mirrors = yaml_root['mirrors'] mirror_urls = [url for url in ci_mirrors.values()] + enable_artifacts_buildcache = False + if 'enable-artifacts-buildcache' in gitlab_ci: + enable_artifacts_buildcache = gitlab_ci['enable-artifacts-buildcache'] + bootstrap_specs = [] phases = [] - if 'bootstrap' in yaml_root['gitlab-ci']: - for phase in yaml_root['gitlab-ci']['bootstrap']: + if 'bootstrap' in gitlab_ci: + for phase in gitlab_ci['bootstrap']: try: phase_name = phase.get('name') strip_compilers = phase.get('compiler-agnostic') @@ -469,9 +520,11 @@ def release_jobs(parser, args): staged_phases = {} for phase in phases: phase_name = phase['name'] - staged_phases[phase_name] = stage_spec_jobs(env.spec_lists[phase_name]) + with spack.concretize.disable_compiler_existence_check(): + staged_phases[phase_name] = stage_spec_jobs( + env.spec_lists[phase_name]) - if args.print_summary: + if print_summary: for phase in phases: phase_name = phase['name'] tty.msg('Stages for phase "{0}"'.format(phase_name)) @@ -498,10 +551,12 @@ def release_jobs(parser, args): stage_id += 1 for spec_label in stage_jobs: - release_spec = spec_labels[spec_label]['spec'] root_spec = spec_labels[spec_label]['rootSpec'] + pkg_name = pkg_name_from_spec_label(spec_label) + release_spec = root_spec[pkg_name] - runner_attribs = find_matching_config(root_spec, ci_mappings) + runner_attribs = find_matching_config( + release_spec, ci_mappings) if not runner_attribs: tty.warn('No match found for {0}, skipping it'.format( @@ -529,7 +584,11 @@ def release_jobs(parser, args): job_name = get_job_name(phase_name, strip_compilers, release_spec, osname, build_group) - job_scripts = ['./bin/rebuild-package.sh'] + debug_flag = '' + if 'enable-debug-messages' in gitlab_ci: + debug_flag = '-d ' + + job_scripts = ['spack {0}ci rebuild'.format(debug_flag)] compiler_action = 'NONE' if len(phases) > 1: @@ -538,7 +597,6 @@ def release_jobs(parser, args): compiler_action = 'INSTALL_MISSING' job_vars = { - 'SPACK_MIRROR_URL': mirror_urls[0], 'SPACK_ROOT_SPEC': format_root_spec( root_spec, main_phase, strip_compilers), 'SPACK_JOB_SPEC_PKG_NAME': release_spec.name, @@ -547,11 +605,13 @@ def release_jobs(parser, args): job_dependencies = [] if spec_label in dependencies: - job_dependencies = ( - [get_job_name(phase_name, strip_compilers, - spec_labels[dep_label]['spec'], - osname, build_group) - for dep_label in dependencies[spec_label]]) + for dep_label in dependencies[spec_label]: + dep_pkg = pkg_name_from_spec_label(dep_label) + dep_spec = spec_labels[dep_label]['rootSpec'][dep_pkg] + dep_job_name = get_job_name( + phase_name, strip_compilers, dep_spec, osname, + build_group) + job_dependencies.append(dep_job_name) # This next section helps gitlab make sure the right # bootstrapped compiler exists in the artifacts buildcache by @@ -585,34 +645,38 @@ def release_jobs(parser, args): [spec_labels[d]['spec'].name for d in dependencies[spec_label]]) - job_vars['SPACK_CDASH_BASE_URL'] = cdash_url - job_vars['SPACK_CDASH_PROJECT'] = cdash_project - job_vars['SPACK_CDASH_PROJECT_ENC'] = cdash_project_enc job_vars['SPACK_CDASH_BUILD_NAME'] = cdash_build_name - job_vars['SPACK_CDASH_SITE'] = cdash_site - job_vars['SPACK_RELATED_BUILDS'] = ';'.join(related_builds) - job_vars['SPACK_JOB_SPEC_BUILDGROUP'] = build_group - - job_vars['SPACK_ENABLE_CDASH'] = str(enable_cdash_reporting) + job_vars['SPACK_RELATED_BUILDS_CDASH'] = ';'.join( + related_builds) variables.update(job_vars) + artifact_paths = [ + 'jobs_scratch_dir', + 'cdash_report', + ] + + if enable_artifacts_buildcache: + artifact_paths.append('local_mirror/build_cache') + job_object = { 'stage': stage_name, 'variables': variables, 'script': job_scripts, 'tags': tags, 'artifacts': { - 'paths': [ - 'jobs_scratch_dir', - 'cdash_report', - 'local_mirror/build_cache', - ], + 'paths': artifact_paths, 'when': 'always', }, 'dependencies': job_dependencies, } + if before_script: + job_object['before_script'] = before_script + + if after_script: + job_object['after_script'] = after_script + if image_name: job_object['image'] = image_name if image_entry is not None: @@ -624,7 +688,7 @@ def release_jobs(parser, args): output_object[job_name] = job_object job_id += 1 - tty.msg('{0} build jobs generated in {1} stages'.format( + tty.debug('{0} build jobs generated in {1} stages'.format( job_id, stage_id)) # Use "all_job_names" to populate the build group for this set @@ -637,20 +701,280 @@ def release_jobs(parser, args): else: tty.warn('Unable to populate buildgroup without CDash credentials') - # Add an extra, final job to regenerate the index - final_stage = 'stage-rebuild-index' - final_job = { - 'stage': final_stage, - 'variables': { - 'MIRROR_URL': mirror_urls[0], - }, - 'script': './bin/rebuild-index.sh', - 'tags': ['spack-post-ci'] # may want a runner to handle this - } - output_object['rebuild-index'] = final_job - stage_names.append(final_stage) + if final_job_config: + # Add an extra, final job to regenerate the index + final_stage = 'stage-rebuild-index' + final_job = { + 'stage': final_stage, + 'script': 'spack buildcache update-index -d {0}'.format( + mirror_urls[0]), + 'tags': final_job_config['tags'] + } + if 'image' in final_job_config: + final_job['image'] = final_job_config['image'] + if before_script: + final_job['before_script'] = before_script + if after_script: + final_job['after_script'] = after_script + output_object['rebuild-index'] = final_job + stage_names.append(final_stage) output_object['stages'] = stage_names - with open(args.output_file, 'w') as outf: + with open(output_file, 'w') as outf: outf.write(syaml.dump_config(output_object, default_flow_style=True)) + + +def url_encode_string(input_string): + encoded_keyval = urlencode({'donotcare': input_string}) + eq_idx = encoded_keyval.find('=') + 1 + encoded_value = encoded_keyval[eq_idx:] + return encoded_value + + +def import_signing_key(base64_signing_key): + if not base64_signing_key: + tty.warn('No key found for signing/verifying packages') + return + + tty.debug('ci.import_signing_key() will attempt to import a key') + + # This command has the side-effect of creating the directory referred + # to as GNUPGHOME in setup_environment() + list_output = spack_gpg('list', output=str) + + tty.debug('spack gpg list:') + tty.debug(list_output) + + decoded_key = base64.b64decode(base64_signing_key) + if isinstance(decoded_key, bytes): + decoded_key = decoded_key.decode('utf8') + + with TemporaryDirectory() as tmpdir: + sign_key_path = os.path.join(tmpdir, 'signing_key') + with open(sign_key_path, 'w') as fd: + fd.write(decoded_key) + + key_import_output = spack_gpg('trust', sign_key_path, output=str) + tty.debug('spack gpg trust {0}'.format(sign_key_path)) + tty.debug(key_import_output) + + # Now print the keys we have for verifying and signing + trusted_keys_output = spack_gpg('list', '--trusted', output=str) + signing_keys_output = spack_gpg('list', '--signing', output=str) + + tty.debug('spack gpg list --trusted') + tty.debug(trusted_keys_output) + tty.debug('spack gpg list --signing') + tty.debug(signing_keys_output) + + +def configure_compilers(compiler_action, scope=None): + if compiler_action == 'INSTALL_MISSING': + tty.debug('Make sure bootstrapped compiler will be installed') + config = cfg.get('config') + config['install_missing_compilers'] = True + cfg.set('config', config) + elif compiler_action == 'FIND_ANY': + tty.debug('Just find any available compiler') + find_args = ['find'] + if scope: + find_args.extend(['--scope', scope]) + output = spack_compiler(*find_args) + tty.debug('spack compiler find') + tty.debug(output) + output = spack_compiler('list') + tty.debug('spack compiler list') + tty.debug(output) + else: + tty.debug('No compiler action to be taken') + + return None + + +def get_concrete_specs(root_spec, job_name, related_builds, compiler_action): + spec_map = { + 'root': None, + 'deps': {}, + } + + if compiler_action == 'FIND_ANY': + # This corresponds to a bootstrapping phase where we need to + # rely on any available compiler to build the package (i.e. the + # compiler needed to be stripped from the spec when we generated + # the job), and thus we need to concretize the root spec again. + tty.debug('About to concretize {0}'.format(root_spec)) + concrete_root = Spec(root_spec).concretized() + tty.debug('Resulting concrete root: {0}'.format(concrete_root)) + else: + # in this case, either we're relying on Spack to install missing + # compiler bootstrapped in a previous phase, or else we only had one + # phase (like a site which already knows what compilers are available + # on it's runners), so we don't want to concretize that root spec + # again. The reason we take this path in the first case (bootstrapped + # compiler), is that we can't concretize a spec at this point if we're + # going to ask spack to "install_missing_compilers". + concrete_root = Spec.from_yaml( + str(zlib.decompress(base64.b64decode(root_spec)).decode('utf-8'))) + + spec_map['root'] = concrete_root + spec_map[job_name] = concrete_root[job_name] + + if related_builds: + for dep_job_name in related_builds.split(';'): + spec_map['deps'][dep_job_name] = concrete_root[dep_job_name] + + return spec_map + + +def register_cdash_build(build_name, base_url, project, site, track): + url = base_url + '/api/v1/addBuild.php' + time_stamp = datetime.datetime.now().strftime('%Y%m%d-%H%M') + build_stamp = '{0}-{1}'.format(time_stamp, track) + payload = { + "project": project, + "site": site, + "name": build_name, + "stamp": build_stamp, + } + + tty.debug('Registering cdash build to {0}, payload:'.format(url)) + tty.debug(payload) + + enc_data = json.dumps(payload).encode('utf-8') + + headers = { + 'Content-Type': 'application/json', + } + + opener = build_opener(HTTPHandler) + + request = Request(url, data=enc_data, headers=headers) + + response = opener.open(request) + response_code = response.getcode() + + if response_code != 200 and response_code != 201: + msg = 'Adding build failed (response code = {0}'.format(response_code) + raise SpackError(msg) + + response_text = response.read() + response_json = json.loads(response_text) + build_id = response_json['buildid'] + + return (build_id, build_stamp) + + +def relate_cdash_builds(spec_map, cdash_base_url, job_build_id, cdash_project, + cdashids_mirror_url): + if not job_build_id: + return + + dep_map = spec_map['deps'] + + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + + cdash_api_url = '{0}/api/v1/relateBuilds.php'.format(cdash_base_url) + + for dep_pkg_name in dep_map: + tty.debug('Fetching cdashid file for {0}'.format(dep_pkg_name)) + dep_spec = dep_map[dep_pkg_name] + dep_build_id = read_cdashid_from_mirror(dep_spec, cdashids_mirror_url) + + payload = { + "project": cdash_project, + "buildid": job_build_id, + "relatedid": dep_build_id, + "relationship": "depends on" + } + + enc_data = json.dumps(payload).encode('utf-8') + + opener = build_opener(HTTPHandler) + + request = Request(cdash_api_url, data=enc_data, headers=headers) + + response = opener.open(request) + response_code = response.getcode() + + if response_code != 200 and response_code != 201: + msg = 'Relate builds ({0} -> {1}) failed (resp code = {2})'.format( + job_build_id, dep_build_id, response_code) + raise SpackError(msg) + + response_text = response.read() + tty.debug('Relate builds response: {0}'.format(response_text)) + + +def write_cdashid_to_mirror(cdashid, spec, mirror_url): + if not spec.concrete: + tty.die('Can only write cdashid for concrete spec to mirror') + + with TemporaryDirectory() as tmpdir: + local_cdash_path = os.path.join(tmpdir, 'job.cdashid') + with open(local_cdash_path, 'w') as fd: + fd.write(cdashid) + + buildcache_name = bindist.tarball_name(spec, '') + cdashid_file_name = '{0}.cdashid'.format(buildcache_name) + remote_url = os.path.join( + mirror_url, bindist.build_cache_relative_path(), cdashid_file_name) + + tty.debug('pushing cdashid to url') + tty.debug(' local file path: {0}'.format(local_cdash_path)) + tty.debug(' remote url: {0}'.format(remote_url)) + web_util.push_to_url(local_cdash_path, remote_url) + + +def read_cdashid_from_mirror(spec, mirror_url): + if not spec.concrete: + tty.die('Can only read cdashid for concrete spec from mirror') + + buildcache_name = bindist.tarball_name(spec, '') + cdashid_file_name = '{0}.cdashid'.format(buildcache_name) + url = os.path.join( + mirror_url, bindist.build_cache_relative_path(), cdashid_file_name) + + resp_url, resp_headers, response = web_util.read_from_url(url) + contents = response.fp.read() + + return int(contents) + + +def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id): + if mirror_url: + tty.debug('Creating buildcache') + buildcache._createtarball(env, yaml_path, None, mirror_url, None, + True, True, False, False, True, False) + if build_id: + tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( + build_id, mirror_url)) + write_cdashid_to_mirror(build_id, spec, mirror_url) + + +def copy_stage_logs_to_artifacts(job_spec, job_log_dir): + try: + job_pkg = spack.repo.get(job_spec) + tty.debug('job package: {0}'.format(job_pkg)) + stage_dir = job_pkg.stage.path + tty.debug('stage dir: {0}'.format(stage_dir)) + build_env_src = os.path.join(stage_dir, 'spack-build-env.txt') + build_out_src = os.path.join(stage_dir, 'spack-build-out.txt') + build_env_dst = os.path.join( + job_log_dir, 'spack-build-env.txt') + build_out_dst = os.path.join( + job_log_dir, 'spack-build-out.txt') + tty.debug('Copying logs to artifacts:') + tty.debug(' 1: {0} -> {1}'.format( + build_env_src, build_env_dst)) + shutil.copyfile(build_env_src, build_env_dst) + tty.debug(' 2: {0} -> {1}'.format( + build_out_src, build_out_dst)) + shutil.copyfile(build_out_src, build_out_dst) + except Exception as inst: + msg = ('Unable to copy build logs from stage to artifacts ' + 'due to exception: {0}').format(inst) + tty.error(msg) diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index 46d1bf3628..5baa63af85 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -303,19 +303,18 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False): return specs_from_cli -def createtarball(args): - """create a binary package from an existing install""" - if args.spec_yaml: +def _createtarball(env, spec_yaml, packages, directory, key, no_deps, force, + rel, unsigned, allow_root, no_rebuild_index): + if spec_yaml: packages = set() - tty.msg('createtarball, reading spec from {0}'.format(args.spec_yaml)) - with open(args.spec_yaml, 'r') as fd: + with open(spec_yaml, 'r') as fd: yaml_text = fd.read() tty.debug('createtarball read spec yaml:') tty.debug(yaml_text) s = Spec.from_yaml(yaml_text) packages.add('/{0}'.format(s.dag_hash())) - elif args.packages: - packages = args.packages + elif packages: + packages = packages else: tty.die("build cache file creation requires at least one" + " installed package argument or else path to a" + @@ -324,18 +323,15 @@ def createtarball(args): specs = set() outdir = '.' - if args.directory: - outdir = args.directory + if directory: + outdir = directory mirror = spack.mirror.MirrorCollection().lookup(outdir) outdir = url_util.format(mirror.push_url) signkey = None - if args.key: - signkey = args.key - - # restrict matching to current environment if one is active - env = ev.get_env(args, 'buildcache create') + if key: + signkey = key matches = find_matching_specs(pkgs, env=env) @@ -350,7 +346,7 @@ def createtarball(args): else: tty.debug('adding matching spec %s' % match.format()) specs.add(match) - if args.no_deps is True: + if no_deps is True: continue tty.debug('recursing dependencies') for d, node in match.traverse(order='post', @@ -368,14 +364,25 @@ def createtarball(args): for spec in specs: tty.msg('creating binary cache file for package %s ' % spec.format()) try: - bindist.build_tarball(spec, outdir, args.force, args.rel, - args.unsigned, args.allow_root, signkey, - not args.no_rebuild_index) + bindist.build_tarball(spec, outdir, force, rel, + unsigned, allow_root, signkey, + not no_rebuild_index) except Exception as e: tty.warn('%s' % e) pass +def createtarball(args): + """create a binary package from an existing install""" + + # restrict matching to current environment if one is active + env = ev.get_env(args, 'buildcache create') + + _createtarball(env, args.spec_yaml, args.packages, args.directory, + args.key, args.no_deps, args.force, args.rel, args.unsigned, + args.allow_root, args.no_rebuild_index) + + def installtarball(args): """install from a binary package""" if not args.packages: @@ -477,6 +484,32 @@ def check_binaries(args): configured_mirrors, specs, args.output_file, args.rebuild_on_error)) +def download_buildcache_files(concrete_spec, local_dest, require_cdashid, + mirror_url=None): + tarfile_name = bindist.tarball_name(concrete_spec, '.spack') + tarball_dir_name = bindist.tarball_directory_name(concrete_spec) + tarball_path_name = os.path.join(tarball_dir_name, tarfile_name) + local_tarball_path = os.path.join(local_dest, tarball_dir_name) + + files_to_fetch = [ + { + 'url': tarball_path_name, + 'path': local_tarball_path, + 'required': True, + }, { + 'url': bindist.tarball_name(concrete_spec, '.spec.yaml'), + 'path': local_dest, + 'required': True, + }, { + 'url': bindist.tarball_name(concrete_spec, '.cdashid'), + 'path': local_dest, + 'required': require_cdashid, + }, + ] + + return bindist.download_buildcache_entry(files_to_fetch, mirror_url) + + def get_tarball(args): """Download buildcache entry from a remote mirror to local folder. This command uses the process exit code to indicate its result, specifically, @@ -493,34 +526,10 @@ def get_tarball(args): sys.exit(0) spec = get_concrete_spec(args) + result = download_buildcache_files(spec, args.path, args.require_cdashid) - tarfile_name = bindist.tarball_name(spec, '.spack') - tarball_dir_name = bindist.tarball_directory_name(spec) - tarball_path_name = os.path.join(tarball_dir_name, tarfile_name) - local_tarball_path = os.path.join(args.path, tarball_dir_name) - - files_to_fetch = [ - { - 'url': tarball_path_name, - 'path': local_tarball_path, - 'required': True, - }, { - 'url': bindist.tarball_name(spec, '.spec.yaml'), - 'path': args.path, - 'required': True, - }, { - 'url': bindist.tarball_name(spec, '.cdashid'), - 'path': args.path, - 'required': args.require_cdashid, - }, - ] - - result = bindist.download_buildcache_entry(files_to_fetch) - - if result: - sys.exit(0) - - sys.exit(1) + if not result: + sys.exit(1) def get_concrete_spec(args): diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py new file mode 100644 index 0000000000..22c555d85a --- /dev/null +++ b/lib/spack/spack/cmd/ci.py @@ -0,0 +1,482 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import shutil +import sys + +from six.moves.urllib.parse import urlencode + +import llnl.util.tty as tty + +import spack.binary_distribution as bindist +import spack.ci as spack_ci +import spack.cmd.buildcache as buildcache +import spack.environment as ev +import spack.hash_types as ht +import spack.util.executable as exe + + +description = "manage continuous integration pipelines" +section = "build" +level = "long" + + +def get_env_var(variable_name): + if variable_name in os.environ: + return os.environ.get(variable_name) + return None + + +def setup_parser(subparser): + setup_parser.parser = subparser + subparsers = subparser.add_subparsers(help='CI sub-commands') + + start = subparsers.add_parser('start', help=ci_start.__doc__) + start.add_argument( + '--output-file', default=None, + help="Absolute path to file where generated jobs file should be " + + "written. The default is .gitlab-ci.yml in the root of the " + + "repository.") + start.add_argument( + '--copy-to', default=None, + help="Absolute path of additional location where generated jobs " + + "yaml file should be copied. Default is not to copy.") + start.add_argument( + '--spack-repo', default=None, + help="Provide a url for this argument if a custom spack repo " + + "should be cloned as a step in each generated job.") + start.add_argument( + '--spack-ref', default=None, + help="Provide a git branch or tag if a custom spack branch " + + "should be checked out as a step in each generated job. " + + "This argument is ignored if no --spack-repo is provided.") + start.add_argument( + '--downstream-repo', default=None, + help="Url to repository where commit containing jobs yaml file " + + "should be pushed.") + start.add_argument( + '--branch-name', default='default-branch', + help="Name of current branch, used in generation of pushed commit.") + start.add_argument( + '--commit-sha', default='none', + help="SHA of current commit, used in generation of pushed commit.") + start.set_defaults(func=ci_start) + + # Dynamic generation of the jobs yaml from a spack environment + generate = subparsers.add_parser('generate', help=ci_generate.__doc__) + generate.add_argument( + '--output-file', default=None, + help="Absolute path to file where generated jobs file should be " + + "written. The default is .gitlab-ci.yml in the root of the " + + "repository.") + generate.add_argument( + '--copy-to', default=None, + help="Absolute path of additional location where generated jobs " + + "yaml file should be copied. Default is not to copy.") + generate.add_argument( + '--spack-repo', default=None, + help="Provide a url for this argument if a custom spack repo " + + "should be cloned as a step in each generated job.") + generate.add_argument( + '--spack-ref', default=None, + help="Provide a git branch or tag if a custom spack branch " + + "should be checked out as a step in each generated job. " + + "This argument is ignored if no --spack-repo is provided.") + generate.set_defaults(func=ci_generate) + + # Commit and push jobs yaml to a downstream CI repo + pushyaml = subparsers.add_parser('pushyaml', help=ci_pushyaml.__doc__) + pushyaml.add_argument( + '--downstream-repo', default=None, + help="Url to repository where commit containing jobs yaml file " + + "should be pushed.") + pushyaml.add_argument( + '--branch-name', default='default-branch', + help="Name of current branch, used in generation of pushed commit.") + pushyaml.add_argument( + '--commit-sha', default='none', + help="SHA of current commit, used in generation of pushed commit.") + pushyaml.set_defaults(func=ci_pushyaml) + + # Check a spec against mirror. Rebuild, create buildcache and push to + # mirror (if necessary). + rebuild = subparsers.add_parser('rebuild', help=ci_rebuild.__doc__) + rebuild.set_defaults(func=ci_rebuild) + + +def ci_generate(args): + """Generate jobs file from a spack environment file containing CI info. + Before invoking this command, you can set the environment variable + SPACK_CDASH_AUTH_TOKEN to contain the CDash authorization token + for creating a build group for the generated workload and registering + all generated jobs under that build group. If this environment + variable is not set, no build group will be created on CDash.""" + env = ev.get_env(args, 'ci generate', required=True) + + output_file = args.output_file + copy_yaml_to = args.copy_to + spack_repo = args.spack_repo + spack_ref = args.spack_ref + + if not output_file: + gen_ci_dir = os.getcwd() + output_file = os.path.join(gen_ci_dir, '.gitlab-ci.yml') + else: + gen_ci_dir = os.path.dirname(output_file) + if not os.path.exists(gen_ci_dir): + os.makedirs(gen_ci_dir) + + # Generate the jobs + spack_ci.generate_gitlab_ci_yaml( + env, True, output_file, spack_repo, spack_ref) + + if copy_yaml_to: + copy_to_dir = os.path.dirname(copy_yaml_to) + if not os.path.exists(copy_to_dir): + os.makedirs(copy_to_dir) + shutil.copyfile(output_file, copy_yaml_to) + + +def ci_pushyaml(args): + """Push the generated jobs yaml file to a remote repository. The file + (.gitlab-ci.yaml) is expected to be in the current directory, which + should be the root of the repository.""" + downstream_repo = args.downstream_repo + branch_name = args.branch_name + commit_sha = args.commit_sha + + if not downstream_repo: + tty.die('No downstream repo to push to, exiting') + + working_dir = os.getcwd() + jobs_yaml = os.path.join(working_dir, '.gitlab-ci.yml') + git_dir = os.path.join(working_dir, '.git') + + if not os.path.exists(jobs_yaml): + tty.die('.gitlab-ci.yml must exist in current directory') + + if not os.path.exists(git_dir): + tty.die('.git directory must exist in current directory') + + # Create a temporary working directory + with spack_ci.TemporaryDirectory() as temp_dir: + git = exe.which('git', required=True) + + # Push a commit with the generated file to the downstream ci repo + saved_git_dir = os.path.join(temp_dir, 'original-git-dir') + + shutil.move('.git', saved_git_dir) + + git('init', '.') + + git('config', 'user.email', 'robot@spack.io') + git('config', 'user.name', 'Spack Build Bot') + + git('add', '.') + + # If the environment contains a spack directory, do not commit + # or push it with any other generated products + if os.path.exists('./spack') and os.path.isdir('./spack'): + git('rm', '-rf', '--cached', 'spack') + + tty.msg('git commit') + commit_message = '{0} {1} ({2})'.format( + 'Auto-generated commit testing', branch_name, commit_sha) + + git('commit', '-m', '{0}'.format(commit_message)) + + tty.msg('git push') + git('remote', 'add', 'downstream', downstream_repo) + push_to_branch = 'master:multi-ci-{0}'.format(branch_name) + git('push', '--force', 'downstream', push_to_branch) + + shutil.rmtree('.git') + shutil.move(saved_git_dir, '.git') + git('reset', '--hard', 'HEAD') + + +def ci_rebuild(args): + """This command represents a gitlab-ci job, corresponding to a single + release spec. As such it must first decide whether or not the spec it + has been assigned to build is up to date on the remote binary mirror. + If it is not (i.e. the full_hash of the spec as computed locally does + not match the one stored in the metadata on the mirror), this script + will build the package, create a binary cache for it, and then push all + related files to the remote binary mirror. This script also + communicates with a remote CDash instance to share status on the package + build process. + + The spec to be built by this job is represented by essentially two + pieces of information: 1) a root spec (possibly already concrete, but + maybe still needing to be concretized) and 2) a package name used to + index that root spec (once the root is, for certain, concrete).""" + env = ev.get_env(args, 'ci rebuild', required=True) + yaml_root = ev.config_dict(env.yaml) + + # The following environment variables should defined in the CI + # infrastructre (or some other external source) in the case that the + # remote mirror is an S3 bucket. The AWS keys are used to upload + # buildcache entries to S3 using the boto3 api. + # + # AWS_ACCESS_KEY_ID + # AWS_SECRET_ACCESS_KEY + # S3_ENDPOINT_URL (only needed for non-AWS S3 implementations) + # + # If present, we will import the SPACK_SIGNING_KEY using the + # "spack gpg trust" command, so it can be used both for verifying + # dependency buildcache entries and signing the buildcache entry we create + # for our target pkg. + # + # SPACK_SIGNING_KEY + + ci_artifact_dir = get_env_var('CI_PROJECT_DIR') + signing_key = get_env_var('SPACK_SIGNING_KEY') + root_spec = get_env_var('SPACK_ROOT_SPEC') + job_spec_pkg_name = get_env_var('SPACK_JOB_SPEC_PKG_NAME') + compiler_action = get_env_var('SPACK_COMPILER_ACTION') + cdash_build_name = get_env_var('SPACK_CDASH_BUILD_NAME') + related_builds = get_env_var('SPACK_RELATED_BUILDS_CDASH') + + gitlab_ci = None + if 'gitlab-ci' in yaml_root: + gitlab_ci = yaml_root['gitlab-ci'] + + if not gitlab_ci: + tty.die('spack ci rebuild requires an env containing gitlab-ci cfg') + + enable_cdash = False + if 'cdash' in yaml_root: + enable_cdash = True + ci_cdash = yaml_root['cdash'] + job_spec_buildgroup = ci_cdash['build-group'] + cdash_base_url = ci_cdash['url'] + cdash_project = ci_cdash['project'] + proj_enc = urlencode({'project': cdash_project}) + eq_idx = proj_enc.find('=') + 1 + cdash_project_enc = proj_enc[eq_idx:] + cdash_site = ci_cdash['site'] + tty.debug('cdash_base_url = {0}'.format(cdash_base_url)) + tty.debug('cdash_project = {0}'.format(cdash_project)) + tty.debug('cdash_project_enc = {0}'.format(cdash_project_enc)) + tty.debug('cdash_build_name = {0}'.format(cdash_build_name)) + tty.debug('cdash_site = {0}'.format(cdash_site)) + tty.debug('related_builds = {0}'.format(related_builds)) + tty.debug('job_spec_buildgroup = {0}'.format(job_spec_buildgroup)) + + remote_mirror_url = None + if 'mirrors' in yaml_root: + ci_mirrors = yaml_root['mirrors'] + mirror_urls = [url for url in ci_mirrors.values()] + remote_mirror_url = mirror_urls[0] + + if not remote_mirror_url: + tty.die('spack ci rebuild requires an env containing a mirror') + + tty.debug('ci_artifact_dir = {0}'.format(ci_artifact_dir)) + tty.debug('root_spec = {0}'.format(root_spec)) + tty.debug('remote_mirror_url = {0}'.format(remote_mirror_url)) + tty.debug('job_spec_pkg_name = {0}'.format(job_spec_pkg_name)) + tty.debug('compiler_action = {0}'.format(compiler_action)) + + spack_cmd = exe.which('spack') + + os.environ['FORCE_UNSAFE_CONFIGURE'] = '1' + + cdash_report_dir = os.path.join(ci_artifact_dir, 'cdash_report') + temp_dir = os.path.join(ci_artifact_dir, 'jobs_scratch_dir') + job_log_dir = os.path.join(temp_dir, 'logs') + spec_dir = os.path.join(temp_dir, 'specs') + + local_mirror_dir = os.path.join(ci_artifact_dir, 'local_mirror') + build_cache_dir = os.path.join(local_mirror_dir, 'build_cache') + + enable_artifacts_mirror = False + artifact_mirror_url = None + if 'enable-artifacts-buildcache' in gitlab_ci: + enable_artifacts_mirror = gitlab_ci['enable-artifacts-buildcache'] + if enable_artifacts_mirror: + artifact_mirror_url = 'file://' + local_mirror_dir + mirror_msg = 'artifact buildcache enabled, mirror url: {0}'.format( + artifact_mirror_url) + tty.debug(mirror_msg) + + # Clean out scratch directory from last stage + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + + if os.path.exists(cdash_report_dir): + shutil.rmtree(cdash_report_dir) + + os.makedirs(job_log_dir) + os.makedirs(spec_dir) + + job_spec_yaml_path = os.path.join( + spec_dir, '{0}.yaml'.format(job_spec_pkg_name)) + job_log_file = os.path.join(job_log_dir, 'pipeline_log.txt') + + cdash_build_id = None + cdash_build_stamp = None + + with open(job_log_file, 'w') as log_fd: + os.dup2(log_fd.fileno(), sys.stdout.fileno()) + os.dup2(log_fd.fileno(), sys.stderr.fileno()) + + current_directory = os.getcwd() + tty.debug('Current working directory: {0}, Contents:'.format( + current_directory)) + directory_list = os.listdir(current_directory) + for next_entry in directory_list: + tty.debug(' {0}'.format(next_entry)) + + # Make a copy of the environment file, so we can overwrite the changed + # version in between the two invocations of "spack install" + env_src_path = os.path.join(current_directory, 'spack.yaml') + env_dst_path = os.path.join(current_directory, 'spack.yaml_BACKUP') + shutil.copyfile(env_src_path, env_dst_path) + + tty.debug('job concrete spec path: {0}'.format(job_spec_yaml_path)) + + if signing_key: + spack_ci.import_signing_key(signing_key) + + spack_ci.configure_compilers(compiler_action) + + spec_map = spack_ci.get_concrete_specs( + root_spec, job_spec_pkg_name, related_builds, compiler_action) + + job_spec = spec_map[job_spec_pkg_name] + + tty.debug('Here is the concrete spec: {0}'.format(job_spec)) + + with open(job_spec_yaml_path, 'w') as fd: + fd.write(job_spec.to_yaml(hash=ht.build_hash)) + + tty.debug('Done writing concrete spec') + + # DEBUG + with open(job_spec_yaml_path) as fd: + tty.debug('Wrote spec file, read it back. Contents:') + tty.debug(fd.read()) + + # DEBUG the root spec + root_spec_yaml_path = os.path.join(spec_dir, 'root.yaml') + with open(root_spec_yaml_path, 'w') as fd: + fd.write(spec_map['root'].to_yaml(hash=ht.build_hash)) + + if bindist.needs_rebuild(job_spec, remote_mirror_url, True): + # Binary on remote mirror is not up to date, we need to rebuild + # it. + # + # FIXME: ensure mirror precedence causes this local mirror to + # be chosen ahead of the remote one when installing deps + if enable_artifacts_mirror: + mirror_add_output = spack_cmd( + 'mirror', 'add', 'local_mirror', artifact_mirror_url) + tty.debug('spack mirror add:') + tty.debug(mirror_add_output) + + mirror_list_output = spack_cmd('mirror', 'list') + tty.debug('listing spack mirrors:') + tty.debug(mirror_list_output) + + # 2) build up install arguments + install_args = ['-d', '-v', '-k', 'install', '--keep-stage'] + + # 3) create/register a new build on CDash (if enabled) + cdash_args = [] + if enable_cdash: + tty.debug('Registering build with CDash') + (cdash_build_id, + cdash_build_stamp) = spack_ci.register_cdash_build( + cdash_build_name, cdash_base_url, cdash_project, + cdash_site, job_spec_buildgroup) + + cdash_upload_url = '{0}/submit.php?project={1}'.format( + cdash_base_url, cdash_project_enc) + + cdash_args = [ + '--cdash-upload-url', cdash_upload_url, + '--cdash-build', cdash_build_name, + '--cdash-site', cdash_site, + '--cdash-buildstamp', cdash_build_stamp, + ] + + spec_cli_arg = [job_spec_yaml_path] + + tty.debug('Installing package') + + try: + # Two-pass install is intended to avoid spack trying to + # install from buildcache even though the locally computed + # full hash is different than the one stored in the spec.yaml + # file on the remote mirror. + first_pass_args = install_args + [ + '--cache-only', + '--only', + 'dependencies', + ] + first_pass_args.extend(spec_cli_arg) + tty.debug('First pass install arguments: {0}'.format( + first_pass_args)) + spack_cmd(*first_pass_args) + + # Overwrite the changed environment file so it doesn't + # the next install invocation. + shutil.copyfile(env_dst_path, env_src_path) + + second_pass_args = install_args + [ + '--no-cache', + '--only', + 'package', + ] + second_pass_args.extend(cdash_args) + second_pass_args.extend(spec_cli_arg) + tty.debug('Second pass install arguments: {0}'.format( + second_pass_args)) + spack_cmd(*second_pass_args) + except Exception as inst: + tty.error('Caught exception during install:') + tty.error(inst) + + spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) + + # 4) create buildcache on remote mirror + spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, + remote_mirror_url, cdash_build_id) + + # 5) create another copy of that buildcache on "local artifact + # mirror" (only done if cash reporting is enabled) + spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, + artifact_mirror_url, cdash_build_id) + + # 6) relate this build to its dependencies on CDash (if enabled) + if enable_cdash: + spack_ci.relate_cdash_builds( + spec_map, cdash_base_url, cdash_build_id, cdash_project, + artifact_mirror_url or remote_mirror_url) + else: + # There is nothing to do here unless "local artifact mirror" is + # enabled, in which case, we need to download the buildcache to + # the local artifacts directory to be used by dependent jobs in + # subsequent stages + tty.debug('No need to rebuild {0}'.format(job_spec_pkg_name)) + if enable_artifacts_mirror: + tty.debug('Getting {0} buildcache'.format(job_spec_pkg_name)) + tty.debug('Downloading to {0}'.format(build_cache_dir)) + buildcache.download_buildcache_files( + job_spec, build_cache_dir, True, remote_mirror_url) + + +def ci_start(args): + """Kicks of the CI process (currently just calls ci_generate() then + ci_push())""" + ci_generate(args) + ci_pushyaml(args) + + +def ci(parser, args): + if args.func: + args.func(args) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index f7dd6601ea..ba3cf2961b 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1669,7 +1669,8 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): spack.hooks.post_install(self.spec) return elif kwargs.get('cache_only', False): - tty.die('No binary for %s found and cache-only specified') + tty.die('No binary for %s found and cache-only specified' + % self.name) tty.msg('No binary for %s found: installing from source' % self.name) diff --git a/lib/spack/spack/schema/gitlab_ci.py b/lib/spack/spack/schema/gitlab_ci.py index f11c9f07d8..9d0ba8a77b 100644 --- a/lib/spack/spack/schema/gitlab_ci.py +++ b/lib/spack/spack/schema/gitlab_ci.py @@ -9,6 +9,24 @@ :lines: 13- """ +image_schema = { + 'oneOf': [ + { + 'type': 'string' + }, { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'entrypoint': { + 'type': 'array', + 'items': { + 'type': 'string', + }, + }, + }, + }, + ], +} #: Properties for inclusion in other schemas properties = { @@ -58,24 +76,7 @@ properties = { 'additionalProperties': True, 'required': ['tags'], 'properties': { - 'image': { - 'oneOf': [ - { - 'type': 'string' - }, { - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'entrypoint': { - 'type': 'array', - 'items': { - 'type': 'string', - }, - }, - }, - }, - ], - }, + 'image': image_schema, 'tags': { 'type': 'array', 'default': [], @@ -95,6 +96,27 @@ properties = { }, }, }, + 'enable-artifacts-buildcache': { + 'type': 'boolean', + 'default': False, + }, + 'enable-debug-messages': { + 'type': 'boolean', + 'default': False, + }, + 'final-stage-rebuild-index': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['tags'], + 'properties': { + 'image': image_schema, + 'tags': { + 'type': 'array', + 'default': [], + 'items': {'type': 'string'} + }, + }, + }, }, }, } diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py new file mode 100644 index 0000000000..fd8fd7d79d --- /dev/null +++ b/lib/spack/spack/test/ci.py @@ -0,0 +1,167 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import pytest +from six.moves.urllib.error import URLError + +import spack.ci as ci +import spack.main as spack_main +import spack.config as cfg +import spack.paths as spack_paths +import spack.spec as spec +import spack.util.gpg as gpg_util +import spack.util.web as web_util + + +@pytest.fixture(scope='function') +def testing_gpg_directory(tmpdir): + old_gpg_path = gpg_util.GNUPGHOME + gpg_util.GNUPGHOME = str(tmpdir.join('gpg')) + yield + gpg_util.GNUPGHOME = old_gpg_path + + +@pytest.fixture +def tmp_scope(): + """Creates a temporary configuration scope""" + base_name = 'internal-testing-scope' + current_overrides = set( + x.name for x in + cfg.config.matching_scopes(r'^{0}'.format(base_name))) + + num_overrides = 0 + scope_name = base_name + while scope_name in current_overrides: + scope_name = '{0}{1}'.format(base_name, num_overrides) + num_overrides += 1 + + with cfg.override(cfg.InternalConfigScope(scope_name)): + yield scope_name + + +def test_urlencode_string(): + s = 'Spack Test Project' + + s_enc = ci.url_encode_string(s) + + assert(s_enc == 'Spack+Test+Project') + + +def test_import_signing_key(testing_gpg_directory): + signing_key_dir = spack_paths.mock_gpg_keys_path + signing_key_path = os.path.join(signing_key_dir, 'package-signing-key') + with open(signing_key_path) as fd: + signing_key = fd.read() + + # Just make sure this does not raise any exceptions + ci.import_signing_key(signing_key) + + +def test_configure_compilers(mutable_config): + + def assert_missing(config): + assert('install_missing_compilers' not in config or + config['install_missing_compilers'] is False) + + def assert_present(config): + assert('install_missing_compilers' in config and + config['install_missing_compilers'] is True) + + original_config = cfg.get('config') + assert_missing(original_config) + + ci.configure_compilers('FIND_ANY', scope='site') + + second_config = cfg.get('config') + assert_missing(second_config) + + ci.configure_compilers('INSTALL_MISSING') + last_config = cfg.get('config') + assert_present(last_config) + + +def test_get_concrete_specs(config, mock_packages): + root_spec = ( + 'eJztkk1uwyAQhfc5BbuuYjWObSKuUlURYP5aDBjjBPv0RU7iRI6qpKuqUtnxzZvRwHud' + 'YxSt1oCMyuVoBdI5MN8paxDYZK/ZbkLYU3kqAuA0Dtz6BgGtTB8XdG87BCgzwXbwXArY' + 'CxYQiLtqXxUTpLZxSjN/mWlwwxAQlJ7v8wpFtsvK1UXSOUyTjvRKB2Um7LBPhZD0l1md' + 'xJ7VCATfszOiXGOR9np7vwDn7lCMS8SXQNf3RCtyBTVzzNTMUMXmfWrFeR+UngEAEncS' + 'ASjKwZcid7ERNldthBxjX46mMD2PsJnlYXDs2rye3l+vroOkJJ54SXgZPklLRQmx61sm' + 'cgKNVFRO0qlpf2pojq1Ro7OG56MY+Bgc1PkIo/WkaT8OVcrDYuvZkJdtBl/+XCZ+NQBJ' + 'oKg1h6X/VdXRoyE2OWeH6lCXZdHGrauUZAWFw/YJ/0/39OefN3F4Kle3cXjYsF684ZqG' + 'Tbap/uPwbRx+YPStIQ8bvgA7G6YE' + ) + + dep_builds = 'diffutils;libiconv' + spec_map = ci.get_concrete_specs(root_spec, 'bzip2', dep_builds, 'NONE') + + assert('root' in spec_map and 'deps' in spec_map) + + nonconc_root_spec = 'archive-files' + dep_builds = '' + spec_map = ci.get_concrete_specs( + nonconc_root_spec, 'archive-files', dep_builds, 'FIND_ANY') + + assert('root' in spec_map and 'deps' in spec_map) + assert('archive-files' in spec_map) + + +def test_register_cdash_build(): + build_name = 'Some pkg' + base_url = 'http://cdash.fake.org' + project = 'spack' + site = 'spacktests' + track = 'Experimental' + + with pytest.raises(URLError): + ci.register_cdash_build(build_name, base_url, project, site, track) + + +def test_relate_cdash_builds(config, mock_packages): + root_spec = ( + 'eJztkk1uwyAQhfc5BbuuYjWObSKuUlURYP5aDBjjBPv0RU7iRI6qpKuqUtnxzZvRwHud' + 'YxSt1oCMyuVoBdI5MN8paxDYZK/ZbkLYU3kqAuA0Dtz6BgGtTB8XdG87BCgzwXbwXArY' + 'CxYQiLtqXxUTpLZxSjN/mWlwwxAQlJ7v8wpFtsvK1UXSOUyTjvRKB2Um7LBPhZD0l1md' + 'xJ7VCATfszOiXGOR9np7vwDn7lCMS8SXQNf3RCtyBTVzzNTMUMXmfWrFeR+UngEAEncS' + 'ASjKwZcid7ERNldthBxjX46mMD2PsJnlYXDs2rye3l+vroOkJJ54SXgZPklLRQmx61sm' + 'cgKNVFRO0qlpf2pojq1Ro7OG56MY+Bgc1PkIo/WkaT8OVcrDYuvZkJdtBl/+XCZ+NQBJ' + 'oKg1h6X/VdXRoyE2OWeH6lCXZdHGrauUZAWFw/YJ/0/39OefN3F4Kle3cXjYsF684ZqG' + 'Tbap/uPwbRx+YPStIQ8bvgA7G6YE' + ) + + dep_builds = 'diffutils;libiconv' + spec_map = ci.get_concrete_specs(root_spec, 'bzip2', dep_builds, 'NONE') + cdash_api_url = 'http://cdash.fake.org' + job_build_id = '42' + cdash_project = 'spack' + cdashids_mirror_url = 'https://my.fake.mirror' + + with pytest.raises(web_util.SpackWebError): + ci.relate_cdash_builds(spec_map, cdash_api_url, job_build_id, + cdash_project, cdashids_mirror_url) + + # Just make sure passing None for build id doesn't throw exceptions + ci.relate_cdash_builds(spec_map, cdash_api_url, None, cdash_project, + cdashids_mirror_url) + + +def test_read_write_cdash_ids(config, tmp_scope, tmpdir, mock_packages): + working_dir = tmpdir.join('working_dir') + mirror_dir = working_dir.join('mirror') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + mirror_cmd = spack_main.SpackCommand('mirror') + mirror_cmd('add', '--scope', tmp_scope, 'test_mirror', mirror_url) + + mock_spec = spec.Spec('archive-files').concretized() + orig_cdashid = '42' + + ci.write_cdashid_to_mirror(orig_cdashid, mock_spec, mirror_url) + + # Now read it back + read_cdashid = ci.read_cdashid_from_mirror(mock_spec, mirror_url) + + assert(str(read_cdashid) == orig_cdashid) diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py new file mode 100644 index 0000000000..c914f565fc --- /dev/null +++ b/lib/spack/spack/test/cmd/ci.py @@ -0,0 +1,582 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import filecmp +import os +import pytest + +import llnl.util.filesystem as fs + +import spack +import spack.ci as ci +import spack.config +import spack.environment as ev +import spack.hash_types as ht +import spack.util.gpg as gpg_util +from spack.main import SpackCommand +import spack.paths as spack_paths +import spack.repo as repo +from spack.spec import Spec +from spack.test.conftest import MockPackage, MockPackageMultiRepo +import spack.util.executable as exe +import spack.util.spack_yaml as syaml + + +ci_cmd = SpackCommand('ci') +env_cmd = SpackCommand('env') +mirror_cmd = SpackCommand('mirror') +gpg_cmd = SpackCommand('gpg') +install_cmd = SpackCommand('install') +buildcache_cmd = SpackCommand('buildcache') +git = exe.which('git', required=True) + + +@pytest.fixture(scope='function') +def testing_gpg_directory(tmpdir): + old_gpg_path = gpg_util.GNUPGHOME + gpg_util.GNUPGHOME = str(tmpdir.join('gpg')) + yield + gpg_util.GNUPGHOME = old_gpg_path + + +@pytest.fixture() +def env_deactivate(): + yield + spack.environment._active_environment = None + os.environ.pop('SPACK_ENV', None) + + +def initialize_new_repo(repo_path, initial_commit=False): + if not os.path.exists(repo_path): + os.makedirs(repo_path) + + with fs.working_dir(repo_path): + init_args = ['init', '.'] + # if not initial_commit: + # init_args.append('--bare') + + git(*init_args) + + if initial_commit: + readme_contents = "This is the project README\n" + readme_path = os.path.join(repo_path, 'README.md') + with open(readme_path, 'w') as fd: + fd.write(readme_contents) + git('add', '.') + git('commit', '-m', 'Project initial commit') + + +def get_repo_status(repo_path): + with fs.working_dir(repo_path): + output = git('rev-parse', '--abbrev-ref', 'HEAD', output=str) + current_branch = output.split()[0] + + output = git('rev-parse', 'HEAD', output=str) + current_sha = output.split()[0] + + return current_branch, current_sha + + +def set_env_var(key, val): + os.environ[key] = val + + +def test_specs_staging(config): + """Make sure we achieve the best possible staging for the following +spec DAG:: + + a + /| + c b + |\ + e d + |\ + f g + +In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage, +and then 'd', 'b', and 'a' to be put in the next three stages, respectively. + +""" + default = ('build', 'link') + + g = MockPackage('g', [], []) + f = MockPackage('f', [], []) + e = MockPackage('e', [], []) + d = MockPackage('d', [f, g], [default, default]) + c = MockPackage('c', [], []) + b = MockPackage('b', [d, e], [default, default]) + a = MockPackage('a', [b, c], [default, default]) + + mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g]) + + with repo.swap(mock_repo): + spec_a = Spec('a') + spec_a.concretize() + + spec_a_label = ci.spec_deps_key_label(spec_a)[1] + spec_b_label = ci.spec_deps_key_label(spec_a['b'])[1] + spec_c_label = ci.spec_deps_key_label(spec_a['c'])[1] + spec_d_label = ci.spec_deps_key_label(spec_a['d'])[1] + spec_e_label = ci.spec_deps_key_label(spec_a['e'])[1] + spec_f_label = ci.spec_deps_key_label(spec_a['f'])[1] + spec_g_label = ci.spec_deps_key_label(spec_a['g'])[1] + + spec_labels, dependencies, stages = ci.stage_spec_jobs([spec_a]) + + assert (len(stages) == 4) + + assert (len(stages[0]) == 4) + assert (spec_c_label in stages[0]) + assert (spec_e_label in stages[0]) + assert (spec_f_label in stages[0]) + assert (spec_g_label in stages[0]) + + assert (len(stages[1]) == 1) + assert (spec_d_label in stages[1]) + + assert (len(stages[2]) == 1) + assert (spec_b_label in stages[2]) + + assert (len(stages[3]) == 1) + assert (spec_a_label in stages[3]) + + +def test_ci_generate_with_env(tmpdir, mutable_mock_env_path, env_deactivate, + install_mockery, mock_packages): + """Make sure we can get a .gitlab-ci.yml from an environment file + which has the gitlab-ci, cdash, and mirrors sections.""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + definitions: + - bootstrap: + - cmake@3.4.3 + - old-gcc-pkgs: + - archive-files + - callpath + - hypre@0.2.15 + specs: + - matrix: + - [$old-gcc-pkgs] + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + bootstrap: + - name: bootstrap + compiler-agnostic: true + mappings: + - match: + - arch=test-debian6-x86_64 + runner-attributes: + tags: + - donotcare + image: donotcare + final-stage-rebuild-index: + image: donotcare + tags: [donotcare] + cdash: + build-group: Not important + url: https://my.fake.cdash + project: Not used + site: Nothing +""") + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outputfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + ci_cmd('generate', '--output-file', outputfile) + + with open(outputfile) as f: + contents = f.read() + yaml_contents = syaml.load(contents) + found_spec = False + for ci_key in yaml_contents.keys(): + if '(bootstrap)' in ci_key: + found_spec = True + assert('cmake' in ci_key) + assert(found_spec) + assert('stages' in yaml_contents) + assert(len(yaml_contents['stages']) == 6) + assert(yaml_contents['stages'][0] == 'stage-0') + assert(yaml_contents['stages'][5] == 'stage-rebuild-index') + + +def test_ci_generate_with_env_missing_section(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages): + """Make sure we get a reasonable message if we omit gitlab-ci section""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + mirrors: + some-mirror: https://my.fake.mirror +""") + + expect_out = 'Error: Environment yaml does not have "gitlab-ci" section' + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + + with ev.read('test'): + output = ci_cmd('generate', fail_on_error=False, output=str) + assert(expect_out in output) + + +def test_ci_generate_with_cdash_token(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages): + """Make sure we it doesn't break if we configure cdash""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + enable-artifacts-buildcache: True + enable-debug-messages: True + mappings: + - match: + - archive-files + runner-attributes: + tags: + - donotcare + image: donotcare + cdash: + build-group: Not important + url: https://my.fake.cdash + project: Not used + site: Nothing +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + + with ev.read('test'): + fake_token = 'notreallyatokenbutshouldnotmatter' + os.environ['SPACK_CDASH_AUTH_TOKEN'] = fake_token + copy_to_file = str(tmpdir.join('backup-ci.yml')) + output = ci_cmd('generate', '--copy-to', copy_to_file, output=str) + # That fake token should still have resulted in being unable to + # register build group with cdash, but the workload should + # still have been generated. + expect = 'Problem populating buildgroup' + assert(expect in output) + + dir_contents = os.listdir(tmpdir.strpath) + + print(dir_contents) + + assert('backup-ci.yml' in dir_contents) + + orig_file = str(tmpdir.join('.gitlab-ci.yml')) + + assert(filecmp.cmp(orig_file, copy_to_file) is True) + + +def test_ci_generate_with_external_pkg(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages): + """Make sure we do not generate jobs for external pkgs""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + - externaltest + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + mappings: + - match: + - archive-files + - externaltest + runner-attributes: + tags: + - donotcare + image: donotcare +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outputfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + ci_cmd('generate', '--output-file', outputfile) + + with open(outputfile) as f: + contents = f.read() + print('generated contents: ') + print(contents) + yaml_contents = syaml.load(contents) + for ci_key in yaml_contents.keys(): + if 'externaltool' in ci_key: + print('Erroneously staged "externaltool" pkg') + assert(False) + + +def test_ci_generate_debug_with_custom_spack(tmpdir, mutable_mock_env_path, + env_deactivate, install_mockery, + mock_packages): + """Make sure we generate cloning of spack in job script if needed""" + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + enable-artifacts-buildcache: True + enable-debug-messages: True + mappings: + - match: + - archive-files + runner-attributes: + tags: + - donotcare + image: donotcare +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + spack_repo = 'https://github.com/usera/spack.git' + spack_ref = 'custom-branch' + expected_clone_str = 'git clone "{0}"'.format(spack_repo) + + ci_cmd('generate', '--output-file', outfile, '--spack-repo', + spack_repo, '--spack-ref', spack_ref) + + with open(outfile) as f: + contents = f.read() + yaml_contents = syaml.load(contents) + for ci_key in yaml_contents.keys(): + if '(specs)' in ci_key: + next_job = yaml_contents[ci_key] + print(next_job) + assert('before_script' in next_job) + before_script = next_job['before_script'] + for step in before_script: + if expected_clone_str in step: + break + else: + msg = 'job "{0}" did not clone spack repo'.format( + ci_key) + print(msg) + assert(False) + + assert('script' in next_job) + script = next_job['script'] + for step in script: + if 'spack -d ci rebuild' in step: + break + else: + msg = 'job {0} missing rebuild command'.format( + ci_key) + print(msg) + assert(False) + + +def test_ci_rebuild_basic(tmpdir, mutable_mock_env_path, env_deactivate, + install_mockery, mock_packages, + testing_gpg_directory): + working_dir = tmpdir.join('working_dir') + + mirror_dir = working_dir.join('mirror') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + signing_key_dir = spack_paths.mock_gpg_keys_path + signing_key_path = os.path.join(signing_key_dir, 'package-signing-key') + with open(signing_key_path) as fd: + signing_key = fd.read() + + spack_yaml_contents = """ +spack: + definitions: + - packages: [archive-files] + specs: + - $packages + mirrors: + test-mirror: {0} + gitlab-ci: + enable-artifacts-buildcache: True + mappings: + - match: + - archive-files + runner-attributes: + tags: + - donotcare + image: donotcare + cdash: + build-group: Not important + url: https://my.fake.cdash + project: Not used + site: Nothing +""".format(mirror_url) + + print('spack.yaml:\n{0}\n'.format(spack_yaml_contents)) + + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write(spack_yaml_contents) + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + with ev.read('test'): + root_spec = ('eJyNjsGOwyAMRO/5Ct96alRFFK34ldUqcohJ6BJAQFHUry9Nk66' + 'UXNY3v5mxJ3qSojoDBjnqTGelDUVRQZlMIWpnBZya+nJa0Mv1Fg' + 'G8waRcmAQkimkHWxcF9NRptHyVEoaBkoD5i7ecLVC6yZd/YTtpc' + 'SIBg5Tr/mnA6mt9qTZL9CiLr7trk7StJyd/F81jKGoqoe2gVAaH' + '0uT7ZwPeH9A875HaA9MfidHdHxgxjgJuTGVtIrvfHGtynjkGyzi' + 'xRrkHy94t1lftvv1n4AkVK3kQ') + + # Create environment variables as gitlab would do it + set_env_var('CI_PROJECT_DIR', working_dir.strpath) + set_env_var('SPACK_SIGNING_KEY', signing_key) + set_env_var('SPACK_ROOT_SPEC', root_spec) + set_env_var('SPACK_JOB_SPEC_PKG_NAME', 'archive-files') + set_env_var('SPACK_COMPILER_ACTION', 'NONE') + set_env_var('SPACK_CDASH_BUILD_NAME', '(specs) archive-files') + set_env_var('SPACK_RELATED_BUILDS_CDASH', '') + + rebuild_output = ci_cmd( + 'rebuild', fail_on_error=False, output=str) + + print(rebuild_output) + + +def test_ci_pushyaml(tmpdir): + fake_yaml_contents = """generate ci jobs: + script: + - "./share/spack/qa/gitlab/generate-gitlab-ci-yml.sh" + tags: + - "spack-pre-ci" + artifacts: + paths: + - ci-generation + when: always + """ + local_repo_path = tmpdir.join('local_repo') + initialize_new_repo(local_repo_path.strpath, True) + + remote_repo_path = tmpdir.join('remote_repo') + initialize_new_repo(remote_repo_path.strpath) + + current_branch, current_sha = get_repo_status(local_repo_path.strpath) + + print('local repo info: {0}, {1}'.format(current_branch, current_sha)) + + local_jobs_yaml = local_repo_path.join('.gitlab-ci.yml') + with local_jobs_yaml.open('w') as f: + f.write(fake_yaml_contents) + + pushyaml_args = [ + 'pushyaml', + '--downstream-repo', remote_repo_path.strpath, + '--branch-name', current_branch, + '--commit-sha', current_sha, + ] + + with fs.working_dir(local_repo_path.strpath): + ci_cmd(*pushyaml_args) + + with fs.working_dir(remote_repo_path.strpath): + branch_to_checkout = 'multi-ci-{0}'.format(current_branch) + git('checkout', branch_to_checkout) + with open('.gitlab-ci.yml') as fd: + pushed_contents = fd.read() + assert pushed_contents == fake_yaml_contents + + +@pytest.mark.disable_clean_stage_check +def test_push_mirror_contents(tmpdir, mutable_mock_env_path, env_deactivate, + install_mockery, mock_packages, mock_fetch, + mock_stage, testing_gpg_directory): + working_dir = tmpdir.join('working_dir') + + mirror_dir = working_dir.join('mirror') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + signing_key_dir = spack_paths.mock_gpg_keys_path + signing_key_path = os.path.join(signing_key_dir, 'package-signing-key') + with open(signing_key_path) as fd: + signing_key = fd.read() + + ci.import_signing_key(signing_key) + + spack_yaml_contents = """ +spack: + definitions: + - packages: [patchelf] + specs: + - $packages + mirrors: + test-mirror: {0} +""".format(mirror_url) + + print('spack.yaml:\n{0}\n'.format(spack_yaml_contents)) + + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write(spack_yaml_contents) + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + with ev.read('test') as env: + spec_map = ci.get_concrete_specs( + 'patchelf', 'patchelf', '', 'FIND_ANY') + concrete_spec = spec_map['patchelf'] + spec_yaml = concrete_spec.to_yaml(hash=ht.build_hash) + yaml_path = str(tmpdir.join('spec.yaml')) + with open(yaml_path, 'w') as ypfd: + ypfd.write(spec_yaml) + + install_cmd('--keep-stage', yaml_path) + + # env, spec, yaml_path, mirror_url, build_id + ci.push_mirror_contents( + env, concrete_spec, yaml_path, mirror_url, '42') + + buildcache_list_output = buildcache_cmd('list', output=str) + + assert('patchelf' in buildcache_list_output) + + logs_dir = working_dir.join('logs_dir') + if not os.path.exists(logs_dir.strpath): + os.makedirs(logs_dir.strpath) + + ci.copy_stage_logs_to_artifacts(concrete_spec, logs_dir.strpath) + + logs_dir_list = os.listdir(logs_dir.strpath) + + assert('spack-build-env.txt' in logs_dir_list) + assert('spack-build-out.txt' in logs_dir_list) + + # Also just make sure that if something goes wrong with the + # stage logs copy, no exception is thrown + ci.copy_stage_logs_to_artifacts(None, logs_dir.strpath) + + dl_dir = working_dir.join('download_dir') + if not os.path.exists(dl_dir.strpath): + os.makedirs(dl_dir.strpath) + + buildcache_cmd('download', '--spec-yaml', yaml_path, '--path', + dl_dir.strpath, '--require-cdashid') + + dl_dir_list = os.listdir(dl_dir.strpath) + + assert(len(dl_dir_list) == 3) diff --git a/lib/spack/spack/test/cmd/release_jobs.py b/lib/spack/spack/test/cmd/release_jobs.py deleted file mode 100644 index 7a51354c1c..0000000000 --- a/lib/spack/spack/test/cmd/release_jobs.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os -import pytest -import re - -import spack -import spack.environment as ev -from spack import repo -from spack.cmd.release_jobs import stage_spec_jobs, spec_deps_key_label -from spack.main import SpackCommand -from spack.spec import Spec -from spack.test.conftest import MockPackage, MockPackageMultiRepo - - -env = SpackCommand('env') -release_jobs = SpackCommand('release-jobs') - - -@pytest.fixture() -def env_deactivate(): - yield - spack.environment._active_environment = None - os.environ.pop('SPACK_ENV', None) - - -def test_specs_staging(config): - """Make sure we achieve the best possible staging for the following -spec DAG:: - - a - /| - c b - |\ - e d - |\ - f g - -In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage, -and then 'd', 'b', and 'a' to be put in the next three stages, respectively. - -""" - default = ('build', 'link') - - g = MockPackage('g', [], []) - f = MockPackage('f', [], []) - e = MockPackage('e', [], []) - d = MockPackage('d', [f, g], [default, default]) - c = MockPackage('c', [], []) - b = MockPackage('b', [d, e], [default, default]) - a = MockPackage('a', [b, c], [default, default]) - - mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g]) - - with repo.swap(mock_repo): - spec_a = Spec('a') - spec_a.concretize() - - spec_a_label = spec_deps_key_label(spec_a)[1] - spec_b_label = spec_deps_key_label(spec_a['b'])[1] - spec_c_label = spec_deps_key_label(spec_a['c'])[1] - spec_d_label = spec_deps_key_label(spec_a['d'])[1] - spec_e_label = spec_deps_key_label(spec_a['e'])[1] - spec_f_label = spec_deps_key_label(spec_a['f'])[1] - spec_g_label = spec_deps_key_label(spec_a['g'])[1] - - spec_labels, dependencies, stages = stage_spec_jobs([spec_a]) - - assert (len(stages) == 4) - - assert (len(stages[0]) == 4) - assert (spec_c_label in stages[0]) - assert (spec_e_label in stages[0]) - assert (spec_f_label in stages[0]) - assert (spec_g_label in stages[0]) - - assert (len(stages[1]) == 1) - assert (spec_d_label in stages[1]) - - assert (len(stages[2]) == 1) - assert (spec_b_label in stages[2]) - - assert (len(stages[3]) == 1) - assert (spec_a_label in stages[3]) - - -def test_release_jobs_with_env(tmpdir, mutable_mock_env_path, env_deactivate, - install_mockery, mock_packages): - """Make sure we can get a .gitlab-ci.yml from an environment file - which has the gitlab-ci, cdash, and mirrors sections.""" - filename = str(tmpdir.join('spack.yaml')) - with open(filename, 'w') as f: - f.write("""\ -spack: - definitions: - - packages: [archive-files] - specs: - - $packages - mirrors: - some-mirror: https://my.fake.mirror - gitlab-ci: - mappings: - - match: - - archive-files - runner-attributes: - tags: - - donotcare - image: donotcare - cdash: - build-group: Not important - url: https://my.fake.cdash - project: Not used - site: Nothing -""") - with tmpdir.as_cwd(): - env('create', 'test', './spack.yaml') - outputfile = str(tmpdir.join('.gitlab-ci.yml')) - - with ev.read('test'): - release_jobs('--output-file', outputfile) - - with open(outputfile) as f: - contents = f.read().replace(os.linesep, '') - assert('archive-files' in contents) - assert(re.search(r'stages:\s*\[\s*stage-0', contents)) diff --git a/var/spack/gpg.mock/keys/package-signing-key b/var/spack/gpg.mock/keys/package-signing-key new file mode 100644 index 0000000000..43d5c4ed66 --- /dev/null +++ b/var/spack/gpg.mock/keys/package-signing-key @@ -0,0 +1 @@ +LS0tLS1CRUdJTiBQR1AgUFVCTElDIEtFWSBCTE9DSy0tLS0tCgptUUVOQkYzd0NMMEJDQUMyUURBemw4c1RLRmxHMWRsS09GNm9YckFqZUFyaVdtWWhNZUZFcXNhU3ZwUU5QekJSCkkvbDc5Zmd1OUk5bytxZDNCUmJlK1VGbFpKdnBDS051aFl3R1QyeTZYWGJjemNRRVVjRzBYRU5xd1prRlVaRlAKd1l4V0VZVURwTnZVeHpqUWR5RlhIcjdMa2svM3JncDlZVUJ4MTMzQTVJbEZtcFBCbVdIeFFObVRCelV2Qk5pRgpEM0pvNWZZaENQMzVyM080cVpxWE1hMHZPUFI2QS95TjVnUXRLdCtFemgwOGpYRHRQT0Y4U3pOQm9Kc0FTcnRTCmZ6Z2p0L0xtdXc3QWNqM1ZsR2JyS2wzTER3Q0NjUVB6Nk9RR0dSeFJsc3oyOHl5STA0a2pBM2FjYjkwUDlHT1cKenUyMjBsZ29Bb2x3M1dMdC9VMUNrNHdrR0dXSEpHQlpNQnovQUJFQkFBRzBNRk53WVdOcklFSjFhV3hrSUZCcApjR1ZzYVc1bElDaEVaVzF2SUV0bGVTa2dQR3RsZVVCemNHRmpheTVrWlcxdlBva0JUZ1FUQVFnQU9CWWhCT29YCjE3clFhY25GR3hab09XWHhrU2kvR0JHREJRSmQ4QWk5QWhzdkJRc0pDQWNDQmhVS0NRZ0xBZ1FXQWdNQkFoNEIKQWhlQUFBb0pFR1h4a1NpL0dCR0RybEVILzJPVEltVGxjYWtQNmtDa1hJcEIrdTYySXRocVMvcGo2eUg5VFppeAprQ01UT2xLaWRIV3ludXZidHFwZjJ5RGE3NTJnaTg5M2w5ckhGS2VYWkFQVXN0eENaanhxdUJqNHg4UWFacFZiClNBdFg2UGlOTjJnQldsRVhIM0RYZllMK1QzSTkrVW9nVlJOUWFTZmxaM2ovL3RqVDMwMjA1UzBsRXZuVnBSdXYKNm5iQkZ4V0pHTzRPZTRlby81ajhHeE9LRHRsS2t2TXc4SGtHcUczM3hiREhBODB2VXlwRFZMSkJ0OUpJRVF2agpVdmhaengxcmNPMFdjK2FqNVNKOWVOUkFEN05FMzFVWHRYTnVpNTAxc29IR3FVL3R3TXl2c3NSMFQzMUdlYlEvCmJrbXY0KzlMczhuU2tvTEdYVG5rbi9YT0piRnZUbXhmZFBrRVVUc0llRFBGS1phNUFRMEVYZkFJdlFFSUFLM2UKYk12TUl2QndxeEVFSTR1bmh1aGwvVjJJR1ZpYS8zeldPNm5aVlduMmdORFluNXdGcituYTJSQXRpKzZyTFIzZwo3RTVyWk9hcFAzdVc0b3I3QUd3WmFqc2pLdGoyR0xqWGZtRW16R204TFBqTUN2ZEFva2gzbnh3eHZLMFBoUHA5CjJOVEJIU2xuN0thODJuTUo4a0pqTEpBekxGb2dZN1Nvd0dUeVp0RHd6RDI0ckxaYnNCRXdkbm9GRWI4SFM5a1UKd0paWnBuMUkwYVBPNCtla2VoZ1FRKzNmLzAyZXRYcUtNOXd1cWVMZWVXZXo1Rlk5amUyYXN1ZVI2TnBabDVmUgpjK1loakdiNmtYb0NtZU1sNnIrN1I4YVA4Qjh0VHdrRFN1bFVZS0VJUE5wVUczTEI3K0paOEdSeFZNa3N4Z3V2Cld1VXpCN0k2UEcwZWxmNUpKaFVBRVFFQUFZa0NiQVFZQVFnQUlCWWhCT29YMTdyUWFjbkZHeFpvT1dYeGtTaS8KR0JHREJRSmQ4QWk5QWhzdUFVQUpFR1h4a1NpL0dCR0R3SFFnQkJrQkNBQWRGaUVFNlVlaWpBdkxQMmt0MC9WdgpaZTBzbGlXcXJXTUZBbDN3Q0wwQUNna1FaZTBzbGlXcXJXT1R1Z2YvYWVqTk0yZGVFdTQrQWo1TDdRWVh4aXhnClJPZ25DVkJ0ekN3S2lPSWNlS3hjM2RDQmpHVlZDTksvTmxNY205ZEY0N2YxMDl3cGVMc2tESjllZ1RWcG0yc3kKQXNJRE5HRWIwNzZFV21vdGloTmhJZWtHam1NLzdDL244NlpMeVEyL05ZNVpxSzJkWUVpbHZxTVhIbkY0aDdnOAozVzdpUzRQSzh5QXp1T1krcDVXQ29hcGdlLzVMeEcvNHNGOGdOMVJUc2xyYjdwVWxCMkNPbXoyTXI3MExxS2VmCnF5TkNhVkV6NkVSdFRhek1xTW1nbERCRDE5TmFnQkxBM1BpSklsQ1VWNDNqVDBOUXVSVjlxSCsvbitlT0xFeUYKTkdNMTNHYTRJVjVhZUp5RzNEbStLQnpvQ0VzRmtvMmFBRWJ4SEhodlpjYlY1RkozRk4xUEhodkJtZUZraFd5ZgpCLzlGdzgwanNVMHJESU1nMU5NWUtlK05mN0FOOEw1NXI3OWM2Q2FYVXd2NG1KSDRUSXpVaE5ZS0cyU3BwQVFZCk1wOVJtc0l6blM3cWdBZ2JLLzFBSysrZEdFLzZjUGZ6VXd3V1ozSFVmcy9hZG4rNVVNU2NaRmhUbHBmOWdBSmQKTkRwY0lQM084YnVTMEFSK2NwVm52M1I4TmJ2TlJURWhEUDg2TFhYcDZueld4SzZYdW51a2dua04vcFpsdGY5eApwaFZReEVRMGUzSHRaNi9UYWZXaVlzVEx0RmdMNTc4VTBjTm94bVA4M1MxRitUUlJXSiszdXh5SFUxWVZiSFVHCng3MzdPdDZ6ZWRTbFBJS0JFOUF5UU5mL3pMWUtYNTNmbStaZ0RMNFQ4cnVERGJrQnNRRkxiME5CQURpSDMvSDYKOHB0NERBSmVzOE93NzcrY011dExZWWluCj1TODd5Ci0tLS0tRU5EIFBHUCBQVUJMSUMgS0VZIEJMT0NLLS0tLS0KLS0tLS1CRUdJTiBQR1AgUFJJVkFURSBLRVkgQkxPQ0stLS0tLQoKbFFPWUJGM3dDTDBCQ0FDMlFEQXpsOHNUS0ZsRzFkbEtPRjZvWHJBamVBcmlXbVloTWVGRXFzYVN2cFFOUHpCUgpJL2w3OWZndTlJOW8rcWQzQlJiZStVRmxaSnZwQ0tOdWhZd0dUMnk2WFhiY3pjUUVVY0cwWEVOcXdaa0ZVWkZQCndZeFdFWVVEcE52VXh6alFkeUZYSHI3TGtrLzNyZ3A5WVVCeDEzM0E1SWxGbXBQQm1XSHhRTm1UQnpVdkJOaUYKRDNKbzVmWWhDUDM1cjNPNHFacVhNYTB2T1BSNkEveU41Z1F0S3QrRXpoMDhqWER0UE9GOFN6TkJvSnNBU3J0UwpmemdqdC9MbXV3N0FjajNWbEdicktsM0xEd0NDY1FQejZPUUdHUnhSbHN6Mjh5eUkwNGtqQTNhY2I5MFA5R09XCnp1MjIwbGdvQW9sdzNXTHQvVTFDazR3a0dHV0hKR0JaTUJ6L0FCRUJBQUVBQi8wYW15bE9SdUV6SlVkUFE0WHEKdzJyeU9veU5TUWVSdnUrODlkcTBteGZOTVh4TXFNWmxlaEtBYWNxM0ZDWGhoZ0l2cW5NSnp5cWdZajB1bW4rOQpjVXFkV3pmOHh3dEV0ZGRoYUF3V3lBZGhqT3pKYlh5QXY3azhrV2N4UG42SFJDUkRycmlUenQyOHUxbm9SeVNwCjVDb3oxR2s3NFVFM0E1ZUJnbUpkaFlHZDZJYlVFWk5vR2d1UDdYWDhQNmhyaW9lYkgrVGpXVHFWRkVFOXdwSW4KRmhSc3VtMktLWFM5cjNBUGFzM05RVjBRcHhORjdzWkRWM1BrWWVpR096Zzk1amtGb2x5cjhIb0FjSFVJMkwrKwpFbWVOZDZadkFMN2EvTk4vdUdHY05Tb3VtWTVJaVdSM0ZscGFlNmJ4UXZybDNYMlpGVmJZRW84U1VjWFNKalF0Cm1CWEJCQUROOGJ4ZHhwd3NPd0NYT2xTNWhWWW1xOFE4Q1dUaEVyOHpzK2gzY2dCSGlQZ0FmUUNJMGYzM2o1OXQKcTd2SERFc3NHTGg3WDJpdlF6M3BwZkVma3h0VHAzdEJ1WFNxczR1V1dYeXJHWDlIbGZRaDBCRFBQOXFLTlpPRQpXTUNHSXZtSUJsQnhJNEp2U1ZiOUtlSEtrZkRCcFdJUjFYUUZGK0RlSXE3Qlh2OGF2d1FBNG93eFhLTWxVMnpDCjh5aFNqd3hpaHo1R3gxNXpMUDJ5RzgwM2NZZ21xYjRYSGVUczBrbmlrMVprNi81SXNkZjVVc2hSU3BVY1hJdGsKM0RyeS9uYzNuR01lN2hmS21UdTBJSUNrend0MFpCUE12RnV2RVRjTFp5bm1ZdTdmRlpENXVnNmpxb09aVFZpTApUZ1NKMFNuY2FvWGVhSHppd2dtUENPTGZtUVFPemNFRC9BcGVmVWVPL2M2Y2dNazNwQ3pLaEtaYTlqTS80cm9hCnFLZ1VXNGxFWHdvVUdJZkhiS3RGMmdkZCtBWGlkWmtiV3lycGRuejh5SkY0Q1JlMTFuV2ZwVlQxblNucm0zM1IKM1djdDB3WnJOckVBWlNhVzF1NE5GU09OM2Z3NmVoeEI5d0tWYjk5dFJIUU0zakorRGlWNENTWFB5NTc0YmxKeApiaUY3SDNWTUVTQVNOTzYwTUZOd1lXTnJJRUoxYVd4a0lGQnBjR1ZzYVc1bElDaEVaVzF2SUV0bGVTa2dQR3RsCmVVQnpjR0ZqYXk1a1pXMXZQb2tCVGdRVEFRZ0FPQlloQk9vWDE3clFhY25GR3hab09XWHhrU2kvR0JHREJRSmQKOEFpOUFoc3ZCUXNKQ0FjQ0JoVUtDUWdMQWdRV0FnTUJBaDRCQWhlQUFBb0pFR1h4a1NpL0dCR0RybEVILzJPVApJbVRsY2FrUDZrQ2tYSXBCK3U2Mkl0aHFTL3BqNnlIOVRaaXhrQ01UT2xLaWRIV3ludXZidHFwZjJ5RGE3NTJnCmk4OTNsOXJIRktlWFpBUFVzdHhDWmp4cXVCajR4OFFhWnBWYlNBdFg2UGlOTjJnQldsRVhIM0RYZllMK1QzSTkKK1VvZ1ZSTlFhU2ZsWjNqLy90alQzMDIwNVMwbEV2blZwUnV2Nm5iQkZ4V0pHTzRPZTRlby81ajhHeE9LRHRsSwprdk13OEhrR3FHMzN4YkRIQTgwdlV5cERWTEpCdDlKSUVRdmpVdmhaengxcmNPMFdjK2FqNVNKOWVOUkFEN05FCjMxVVh0WE51aTUwMXNvSEdxVS90d015dnNzUjBUMzFHZWJRL2JrbXY0KzlMczhuU2tvTEdYVG5rbi9YT0piRnYKVG14ZmRQa0VVVHNJZURQRktaYWRBNWNFWGZBSXZRRUlBSzNlYk12TUl2QndxeEVFSTR1bmh1aGwvVjJJR1ZpYQovM3pXTzZuWlZXbjJnTkRZbjV3RnIrbmEyUkF0aSs2ckxSM2c3RTVyWk9hcFAzdVc0b3I3QUd3WmFqc2pLdGoyCkdMalhmbUVtekdtOExQak1DdmRBb2toM254d3h2SzBQaFBwOTJOVEJIU2xuN0thODJuTUo4a0pqTEpBekxGb2cKWTdTb3dHVHladER3ekQyNHJMWmJzQkV3ZG5vRkViOEhTOWtVd0paWnBuMUkwYVBPNCtla2VoZ1FRKzNmLzAyZQp0WHFLTTl3dXFlTGVlV2V6NUZZOWplMmFzdWVSNk5wWmw1ZlJjK1loakdiNmtYb0NtZU1sNnIrN1I4YVA4Qjh0ClR3a0RTdWxVWUtFSVBOcFVHM0xCNytKWjhHUnhWTWtzeGd1dld1VXpCN0k2UEcwZWxmNUpKaFVBRVFFQUFRQUgKOTA0YW5NVHY3c0lUMnNUS0Z5MmxFL1ZSMjM4b3BEb3BacHV0b1IrcmdiTVlDTVhJaWVxTW8zbHAxaGh1WFczWgpkMnIwbnpLYkM3aVNUdkkxMVk2Wk1wZGMwMXU5Y0lJR0N4VDl1TWZycGVmWm9Gb2pUc25EUHlOT21Tc1JMTENSClNDcytYU2sxbHVRQ3kwd2JpZ1lqY2JCZzNLUHFXUUlqaXFhZEo5QXhFLzdIYnFUcXZXd3owaWlUNGJndlNhWDMKRTRJbjFhZ2NhV1RQMFpjaTNuMWQxZGR5c1pIUjFMaXR0cm56TTVEb3lSeHc0K3ZjTWIzb2VBME9yaVRuR3dadApuN1pqR291TkFlUEh5R0ZHd1pFZWlwMllzVWJRQmZKbk9FSlB4UFlqRm1NU1IyT3I2MzdtMXlGM1pZdDRPMysrCjF3Qm05eVJUYXRPOHlsakhTY3VCd1FRQXdMVVdCM21CWmhqSGZjS0pkRmpRZG8vM2FFQWI1SENhMWVhVWFTTXUKVSs1RkMwV0luNnF1OWhDQWdtb05IeFdxbXkzM3lKaUtwbHRuNEpCKzF5MFNBNkFKTFo4TER5WXgxdVhPTHJEYgpmdUEwSVRickJRZCt6U0hHYVp5Tks4THZpNENNWXN0NVQxR1VZa1lVSWszd2I4MlJDZmFFUFhza2gybXBWQkVICjRRVUVBT2I1WTNXNzNqTG44S3hlU3AvL1FGbkw4QTlMTzJFMFFHNlp0aDRhd1J6THF0MVpTU0FhQXRCU1RLTHcKeFgxbVRGb1VjbVBjb3RwT1BvemlCU2daSzJ1VnoxeXdlNmpMUmg1bFUwZnRqekJiNGJ2OTB2K3BwM1ZNZEhnUQo0YytHN09vMERBcWxkR2FkY2dRVGgvMGdBKzR0L3psMTAyTjVoZHpsUUtUUy8zM1JBLzBjOHpWcnZ4MnorNDIvCncwMHNKWGJSSndZRUxKUWV5OVpoZklnUlNkczlhWE5pMEUyZEYrd0xtN0JOVytiQkQ5MlV3OG5FYmxVZWNpamEKWDVZcXhWdFhTVG5IVlZDN1FoeXgxVSt3UHFDZlJTYmJPL0FhUlBrMyt6dmRRc2Rkbk9DM3h6TzEvMkdLc3FENwoyZGNyVGFBZC9pRTBkMVB6VUNLYWdnRHh3RWZINFQ4OGlRSnNCQmdCQ0FBZ0ZpRUU2aGZYdXRCcHljVWJGbWc1ClpmR1JLTDhZRVlNRkFsM3dDTDBDR3k0QlFBa1FaZkdSS0w4WUVZUEFkQ0FFR1FFSUFCMFdJUVRwUjZLTUM4cy8KYVMzVDlXOWw3U3lXSmFxdFl3VUNYZkFJdlFBS0NSQmw3U3lXSmFxdFk1TzZCLzlwNk0weloxNFM3ajRDUGt2dApCaGZHTEdCRTZDY0pVRzNNTEFxSTRoeDRyRnpkMElHTVpWVUkwcjgyVXh5YjEwWGp0L1hUM0NsNHV5UU1uMTZCCk5XbWJheklDd2dNMFlSdlR2b1JhYWkyS0UyRWg2UWFPWXovc0wrZnpwa3ZKRGI4MWpsbW9yWjFnU0tXK294Y2UKY1hpSHVEemRidUpMZzhyeklETzQ1ajZubFlLaHFtQjcva3ZFYi9pd1h5QTNWRk95V3R2dWxTVUhZSTZiUFl5dgp2UXVvcDUrckkwSnBVVFBvUkcxTnJNeW95YUNVTUVQWDAxcUFFc0RjK0lraVVKUlhqZU5QUTFDNUZYMm9mNytmCjU0NHNUSVUwWXpYY1pyZ2hYbHA0bkliY09iNG9IT2dJU3dXU2pab0FSdkVjZUc5bHh0WGtVbmNVM1U4ZUc4R1oKNFdTRmJKOEgvMFhEelNPeFRTc01neURVMHhncDc0MS9zQTN3dm5tdnYxem9KcGRUQy9pWWtmaE1qTlNFMWdvYgpaS21rQkJneW4xR2F3ak9kTHVxQUNCc3IvVUFyNzUwWVQvcHc5L05UREJabmNkUit6OXAyZjdsUXhKeGtXRk9XCmwvMkFBbDAwT2x3Zy9jN3h1NUxRQkg1eWxXZS9kSHcxdTgxRk1TRU0vem90ZGVucWZOYkVycGU2ZTZTQ2VRMysKbG1XMS8zR21GVkRFUkRSN2NlMW5yOU5wOWFKaXhNdTBXQXZudnhUUncyakdZL3pkTFVYNU5GRlluN2U3SElkVApWaFZzZFFiSHZmczYzck41MUtVOGdvRVQwREpBMS8vTXRncGZuZCtiNW1BTXZoUHl1NE1OdVFHeEFVdHZRMEVBCk9JZmY4ZnJ5bTNnTUFsNnp3N0R2djV3eTYwdGhpS2M9Cj0xQndzCi0tLS0tRU5EIFBHUCBQUklWQVRFIEtFWSBCTE9DSy0tLS0tCg==
\ No newline at end of file |