diff options
Diffstat (limited to 'bin/rebuild-package.sh')
| -rwxr-xr-x | bin/rebuild-package.sh | 399 |
1 files changed, 0 insertions, 399 deletions
diff --git a/bin/rebuild-package.sh b/bin/rebuild-package.sh deleted file mode 100755 index 747cbd9875..0000000000 --- a/bin/rebuild-package.sh +++ /dev/null @@ -1,399 +0,0 @@ -#!/bin/bash - -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -### -### This script represents a gitlab-ci job, corresponding to a single release -### spec. As such this script must first decide whether or not the spec it -### has been assigned is up to date on the remote binary mirror. If it is -### not (i.e. the source code has changed in a way that caused a change in the -### full_hash of the spec), this script will build the package, create a -### binary cache for it, and then push all related files to the remote binary -### mirror. This script also optionally communicates with a remote CDash -### instance to share status on the package build process. -### -### The following environment variables are (possibly) used within this script -### in order for the various elements function properly. -### -### First are two defaults we rely on from gitlab: -### -### CI_PROJECT_DIR -### CI_JOB_NAME -### -### The following must be set up in the variables section of gitlab: -### -### AWS_ACCESS_KEY_ID -### AWS_SECRET_ACCESS_KEY -### SPACK_SIGNING_KEY -### -### SPACK_S3_UPLOAD_MIRROR_URL // only required in the short term for the cloud case -### -### The following variabes are defined by the ci generation process and are -### required: -### -### SPACK_ENABLE_CDASH -### SPACK_ROOT_SPEC -### SPACK_MIRROR_URL -### SPACK_JOB_SPEC_PKG_NAME -### SPACK_COMPILER_ACTION -### -### Finally, these variables are optionally defined by the ci generation -### process, and may or may not be present: -### -### SPACK_CDASH_BASE_URL -### SPACK_CDASH_PROJECT -### SPACK_CDASH_PROJECT_ENC -### SPACK_CDASH_BUILD_NAME -### SPACK_CDASH_SITE -### SPACK_RELATED_BUILDS -### SPACK_JOB_SPEC_BUILDGROUP -### - -shopt -s expand_aliases - -export FORCE_UNSAFE_CONFIGURE=1 - -TEMP_DIR="${CI_PROJECT_DIR}/jobs_scratch_dir" - -JOB_LOG_DIR="${TEMP_DIR}/logs" -SPEC_DIR="${TEMP_DIR}/specs" -LOCAL_MIRROR="${CI_PROJECT_DIR}/local_mirror" -BUILD_CACHE_DIR="${LOCAL_MIRROR}/build_cache" -SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin" - -if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - CDASH_UPLOAD_URL="${SPACK_CDASH_BASE_URL}/submit.php?project=${SPACK_CDASH_PROJECT_ENC}" - DEP_JOB_RELATEBUILDS_URL="${SPACK_CDASH_BASE_URL}/api/v1/relateBuilds.php" - declare -a JOB_DEPS_PKG_NAMES -fi - -export SPACK_ROOT=${CI_PROJECT_DIR} -# export PATH="${SPACK_BIN_DIR}:${PATH}" -export GNUPGHOME="${CI_PROJECT_DIR}/opt/spack/gpg" - -. "${CI_PROJECT_DIR}/share/spack/setup-env.sh" - -mkdir -p ${JOB_LOG_DIR} -mkdir -p ${SPEC_DIR} - -cleanup() { - set +x - - if [ -z "$exit_code" ] ; then - - exit_code=$1 - if [ -z "$exit_code" ] ; then - exit_code=0 - fi - - restore_io - - if [ "$( type -t finalize )" '=' 'function' ] ; then - finalize "$JOB_LOG_DIR/cdash_log.txt" - fi - - # We can clean these out later on, once we have a good sense for - # how the logging infrastructure is working - # rm -rf "$JOB_LOG_DIR" - fi - - \exit $exit_code -} - -alias exit='cleanup' - -begin_logging() { - trap "cleanup 1; \\exit \$exit_code" INT TERM QUIT - trap "cleanup 0; \\exit \$exit_code" EXIT - - rm -rf "$JOB_LOG_DIR/cdash_log.txt" - - # NOTE: Here, some redirects are set up - exec 3>&1 # fd 3 is now a dup of stdout - exec 4>&2 # fd 4 is now a dup of stderr - - # stdout and stderr are joined and redirected to the log - exec &> "$JOB_LOG_DIR/cdash_log.txt" - - set -x -} - -restore_io() { - exec >&- - exec 2>&- - - exec >&3 - exec 2>&4 - - exec 3>&- - exec 4>&- -} - -finalize() { - # If you define a finalize function: - # - it will always be called at the very end of the script - # - the log file will be passed in as the first argument, and - # - the code in this function will not be logged. - echo "The full log file is located at $1" - # TODO: send this log data to cdash! -} - -check_error() -{ - local last_exit_code=$1 - local last_cmd=$2 - if [[ ${last_exit_code} -ne 0 ]]; then - echo "${last_cmd} exited with code ${last_exit_code}" - echo "TERMINATING JOB" - exit 1 - else - echo "${last_cmd} completed successfully" - fi -} - -extract_build_id() -{ - LINES_TO_SEARCH=$1 - regex="buildSummary\.php\?buildid=([[:digit:]]+)" - SINGLE_LINE_OUTPUT=$(echo ${LINES_TO_SEARCH} | tr -d '\n') - - if [[ ${SINGLE_LINE_OUTPUT} =~ ${regex} ]]; then - echo "${BASH_REMATCH[1]}" - else - echo "NONE" - fi -} - -get_relate_builds_post_data() -{ - cat <<EOF -{ - "project": "${1}", - "buildid": ${2}, - "relatedid": ${3}, - "relationship": "depends on" -} -EOF -} - -gen_full_specs_for_job_and_deps() { - SPEC_YAML_PATH="${SPEC_DIR}/${SPACK_JOB_SPEC_PKG_NAME}.yaml" - local spec_names_to_save="${SPACK_JOB_SPEC_PKG_NAME}" - - if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - IFS=';' read -ra DEPS <<< "${SPACK_RELATED_BUILDS}" - for i in "${DEPS[@]}"; do - depPkgName="${i}" - spec_names_to_save="${spec_names_to_save} ${depPkgName}" - JOB_DEPS_PKG_NAMES+=("${depPkgName}") - done - fi - - if [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then - # This corresponds to a bootstrapping phase where we need to - # rely on any available compiler to build the package (i.e. the - # compiler needed to be stripped from the spec), and thus we need - # to concretize the root spec again. - spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec "${SPACK_ROOT_SPEC}" --yaml-dir "${SPEC_DIR}" - else - # in this case, either we're relying on Spack to install missing compiler - # bootstrapped in a previous phase, or else we only had one phase (like a - # site which already knows what compilers are available on it's runners), - # so we don't want to concretize that root spec again. The reason we need - # this in the first case (bootstrapped compiler), is that we can't concretize - # a spec at this point if we're going to ask spack to "install_missing_compilers". - tmp_dir=$(mktemp -d) - TMP_YAML_PATH="${tmp_dir}/root.yaml" - ROOT_SPEC_YAML=$(spack python -c "import base64 ; import zlib ; print(str(zlib.decompress(base64.b64decode('${SPACK_ROOT_SPEC}')).decode('utf-8')))") - echo "${ROOT_SPEC_YAML}" > "${TMP_YAML_PATH}" - spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec-yaml "${TMP_YAML_PATH}" --yaml-dir "${SPEC_DIR}" - rm -rf ${tmp_dir} - fi -} - -begin_logging - -echo "Running job for spec: ${CI_JOB_NAME}" - -# This should create the directory we referred to as GNUPGHOME earlier -spack gpg list - -# Importing the secret key using gpg2 directly should allow to -# sign and verify both -set +x -KEY_IMPORT_RESULT=`echo ${SPACK_SIGNING_KEY} | base64 --decode | gpg2 --import` -check_error $? "gpg2 --import" -set -x - -spack gpg list --trusted -spack gpg list --signing - -# To have spack install missing compilers, we need to add a custom -# configuration scope, then we pass that to the package installation -# command -CUSTOM_CONFIG_SCOPE_DIR="${TEMP_DIR}/config_scope" -mkdir -p "${CUSTOM_CONFIG_SCOPE_DIR}" -CUSTOM_CONFIG_SCOPE_ARG="" - -if [ "${SPACK_COMPILER_ACTION}" == "INSTALL_MISSING" ]; then - echo "Make sure bootstrapped compiler will be installed" - custom_config_file_path="${CUSTOM_CONFIG_SCOPE_DIR}/config.yaml" - cat <<CONFIG_STUFF > "${custom_config_file_path}" -config: - install_missing_compilers: true -CONFIG_STUFF - CUSTOM_CONFIG_SCOPE_ARG="-C ${CUSTOM_CONFIG_SCOPE_DIR}" - # Configure the binary mirror where, if needed, this jobs compiler - # was installed in binary pacakge form, then tell spack to - # install_missing_compilers. -elif [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then - echo "Just find any available compiler" - spack compiler find -else - echo "No compiler action to be taken" -fi - -# Finally, list the compilers spack knows about -echo "Compiler Configurations:" -spack config get compilers - -# Write full-deps yamls for this job spec and its dependencies -gen_full_specs_for_job_and_deps - -# Make the build_cache directory if it doesn't exist -mkdir -p "${BUILD_CACHE_DIR}" - -# Get buildcache name so we can write a CDash build id file in the right place. -# If we're unable to get the buildcache name, we may have encountered a problem -# concretizing the spec, or some other issue that will eventually cause the job -# to fail. -JOB_BUILD_CACHE_ENTRY_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${SPEC_YAML_PATH}"` -if [[ $? -ne 0 ]]; then - echo "ERROR, unable to get buildcache entry name for job ${CI_JOB_NAME}" - exit 1 -fi - -if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - # Whether we have to build the spec or download it pre-built, we expect to find - # the cdash build id file sitting in this location afterwards. - JOB_CDASH_ID_FILE="${BUILD_CACHE_DIR}/${JOB_BUILD_CACHE_ENTRY_NAME}.cdashid" -fi - -# Finally, we can check the spec we have been tasked with build against -# the built binary on the remote mirror to see if it needs to be rebuilt -spack -d buildcache check --spec-yaml "${SPEC_YAML_PATH}" --mirror-url "${SPACK_MIRROR_URL}" --rebuild-on-error - -if [[ $? -ne 0 ]]; then - # Configure mirror - spack mirror add local_artifact_mirror "file://${LOCAL_MIRROR}" - - if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - JOB_CDASH_ID="NONE" - - # Install package, using the buildcache from the local mirror to - # satisfy dependencies. - BUILD_ID_LINE=`spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage --cdash-upload-url "${CDASH_UPLOAD_URL}" --cdash-build "${SPACK_CDASH_BUILD_NAME}" --cdash-site "${SPACK_CDASH_SITE}" --cdash-track "${SPACK_JOB_SPEC_BUILDGROUP}" -f "${SPEC_YAML_PATH}" | grep "buildSummary\\.php"` - check_error $? "spack install" - - # By parsing the output of the "spack install" command, we can get the - # buildid generated for us by CDash - JOB_CDASH_ID=$(extract_build_id "${BUILD_ID_LINE}") - - # Write the .cdashid file to the buildcache as well - echo "${JOB_CDASH_ID}" >> ${JOB_CDASH_ID_FILE} - else - spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage -f "${SPEC_YAML_PATH}" - fi - - # Copy some log files into an artifact location, once we have a way - # to provide a spec.yaml file to more spack commands (e.g. "location") - # stage_dir=$(spack location --stage-dir -f "${SPEC_YAML_PATH}") - # build_log_file=$(find -L "${stage_dir}" | grep "spack-build\\.out") - # config_log_file=$(find -L "${stage_dir}" | grep "config\\.log") - # cp "${build_log_file}" "${JOB_LOG_DIR}/" - # cp "${config_log_file}" "${JOB_LOG_DIR}/" - - # Create buildcache entry for this package, reading the spec from the yaml - # file. - spack -d buildcache create --spec-yaml "${SPEC_YAML_PATH}" -a -f -d "${LOCAL_MIRROR}" --no-rebuild-index - check_error $? "spack buildcache create" - - # TODO: The upload-s3 command should eventually be replaced with something - # like: "spack buildcache put <mirror> <spec>", when that subcommand is - # properly implemented. - if [ ! -z "${SPACK_S3_UPLOAD_MIRROR_URL}" ] ; then - spack -d upload-s3 spec --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --endpoint-url "${SPACK_S3_UPLOAD_MIRROR_URL}" - check_error $? "spack upload-s3 spec" - else - spack -d buildcache copy --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --destination-url "${SPACK_MIRROR_URL}" - fi -else - echo "spec ${CI_JOB_NAME} is already up to date on remote mirror, downloading it" - - # Configure remote mirror so we can download buildcache entry - spack mirror add remote_binary_mirror ${SPACK_MIRROR_URL} - - # Now download it - BUILDCACHE_DL_ARGS=("--spec-yaml" "${SPEC_YAML_PATH}" "--path" "${BUILD_CACHE_DIR}/" ) - if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - BUILDCACHE_DL_ARGS+=( "--require-cdashid" ) - fi - spack -d buildcache download "${BUILDCACHE_DL_ARGS[@]}" - check_error $? "spack buildcache download" -fi - -# The next step is to relate this job to the jobs it depends on -if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then - if [ -f "${JOB_CDASH_ID_FILE}" ]; then - JOB_CDASH_BUILD_ID=$(<${JOB_CDASH_ID_FILE}) - - if [ "${JOB_CDASH_BUILD_ID}" == "NONE" ]; then - echo "ERROR: unable to read this jobs id from ${JOB_CDASH_ID_FILE}" - exit 1 - fi - - # Now get CDash ids for dependencies and "relate" each dependency build - # with this jobs build - for DEP_PKG_NAME in "${JOB_DEPS_PKG_NAMES[@]}"; do - echo "Getting cdash id for dependency --> ${DEP_PKG_NAME} <--" - DEP_SPEC_YAML_PATH="${SPEC_DIR}/${DEP_PKG_NAME}.yaml" - DEP_JOB_BUILDCACHE_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${DEP_SPEC_YAML_PATH}"` - - if [[ $? -eq 0 ]]; then - DEP_JOB_ID_FILE="${BUILD_CACHE_DIR}/${DEP_JOB_BUILDCACHE_NAME}.cdashid" - echo "DEP_JOB_ID_FILE path = ${DEP_JOB_ID_FILE}" - - if [ -f "${DEP_JOB_ID_FILE}" ]; then - DEP_JOB_CDASH_BUILD_ID=$(<${DEP_JOB_ID_FILE}) - echo "File ${DEP_JOB_ID_FILE} contained value ${DEP_JOB_CDASH_BUILD_ID}" - echo "Relating builds -> ${SPACK_CDASH_BUILD_NAME} (buildid=${JOB_CDASH_BUILD_ID}) depends on ${DEP_PKG_NAME} (buildid=${DEP_JOB_CDASH_BUILD_ID})" - relateBuildsPostBody="$(get_relate_builds_post_data "${SPACK_CDASH_PROJECT}" ${JOB_CDASH_BUILD_ID} ${DEP_JOB_CDASH_BUILD_ID})" - relateBuildsResult=`curl "${DEP_JOB_RELATEBUILDS_URL}" -H "Content-Type: application/json" -H "Accept: application/json" -d "${relateBuildsPostBody}"` - echo "Result of curl request: ${relateBuildsResult}" - else - echo "ERROR: Did not find expected .cdashid file for dependency: ${DEP_JOB_ID_FILE}" - exit 1 - fi - else - echo "ERROR: Unable to get buildcache entry name for ${DEP_SPEC_NAME}" - exit 1 - fi - done - else - echo "ERROR: Did not find expected .cdashid file ${JOB_CDASH_ID_FILE}" - exit 1 - fi -fi - -# Show the size of the buildcache and a list of what's in it, directly -# in the gitlab log output -( - restore_io - du -sh ${BUILD_CACHE_DIR} - find ${BUILD_CACHE_DIR} -maxdepth 3 -type d -ls -) - -echo "End of rebuild package script" |