diff options
Diffstat (limited to 'bin/rebuild-package.sh')
| -rwxr-xr-x | bin/rebuild-package.sh | 291 |
1 files changed, 181 insertions, 110 deletions
diff --git a/bin/rebuild-package.sh b/bin/rebuild-package.sh index 3813a92a1f..657e431052 100755 --- a/bin/rebuild-package.sh +++ b/bin/rebuild-package.sh @@ -12,29 +12,44 @@ ### not (i.e. the source code has changed in a way that caused a change in the ### full_hash of the spec), this script will build the package, create a ### binary cache for it, and then push all related files to the remote binary -### mirror. This script also communicates with a remote CDash instance to -### share status on the package build process. +### mirror. This script also optionally communicates with a remote CDash +### instance to share status on the package build process. ### -### The following environment variables are expected to be set in order for -### the various elements in this script to function properly. Listed first -### are two defaults we rely on from gitlab, then three we set up in the -### variables section of gitlab ourselves, and finally four variables -### written into the .gitlab-ci.yml file. +### The following environment variables are (possibly) used within this script +### in order for the various elements function properly. +### +### First are two defaults we rely on from gitlab: ### ### CI_PROJECT_DIR ### CI_JOB_NAME ### +### The following must be set up in the variables section of gitlab: +### ### AWS_ACCESS_KEY_ID ### AWS_SECRET_ACCESS_KEY ### SPACK_SIGNING_KEY ### -### CDASH_BASE_URL -### CDASH_PROJECT -### CDASH_PROJECT_ENC -### CDASH_BUILD_NAME -### ROOT_SPEC -### DEPENDENCIES -### MIRROR_URL +### SPACK_S3_UPLOAD_MIRROR_URL // only required in the short term for the cloud case +### +### The following variabes are defined by the ci generation process and are +### required: +### +### SPACK_ENABLE_CDASH +### SPACK_ROOT_SPEC +### SPACK_MIRROR_URL +### SPACK_JOB_SPEC_PKG_NAME +### SPACK_COMPILER_ACTION +### +### Finally, these variables are optionally defined by the ci generation +### process, and may or may not be present: +### +### SPACK_CDASH_BASE_URL +### SPACK_CDASH_PROJECT +### SPACK_CDASH_PROJECT_ENC +### SPACK_CDASH_BUILD_NAME +### SPACK_CDASH_SITE +### SPACK_RELATED_BUILDS +### SPACK_JOB_SPEC_BUILDGROUP ### shopt -s expand_aliases @@ -48,14 +63,19 @@ SPEC_DIR="${TEMP_DIR}/specs" LOCAL_MIRROR="${CI_PROJECT_DIR}/local_mirror" BUILD_CACHE_DIR="${LOCAL_MIRROR}/build_cache" SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin" -CDASH_UPLOAD_URL="${CDASH_BASE_URL}/submit.php?project=${CDASH_PROJECT_ENC}" -DEP_JOB_RELATEBUILDS_URL="${CDASH_BASE_URL}/api/v1/relateBuilds.php" -declare -a JOB_DEPS_PKG_NAMES + +if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then + CDASH_UPLOAD_URL="${SPACK_CDASH_BASE_URL}/submit.php?project=${SPACK_CDASH_PROJECT_ENC}" + DEP_JOB_RELATEBUILDS_URL="${SPACK_CDASH_BASE_URL}/api/v1/relateBuilds.php" + declare -a JOB_DEPS_PKG_NAMES +fi export SPACK_ROOT=${CI_PROJECT_DIR} -export PATH="${SPACK_BIN_DIR}:${PATH}" +# export PATH="${SPACK_BIN_DIR}:${PATH}" export GNUPGHOME="${CI_PROJECT_DIR}/opt/spack/gpg" +. "${CI_PROJECT_DIR}/share/spack/setup-env.sh" + mkdir -p ${JOB_LOG_DIR} mkdir -p ${SPEC_DIR} @@ -160,41 +180,89 @@ EOF } gen_full_specs_for_job_and_deps() { + SPEC_YAML_PATH="${SPEC_DIR}/${SPACK_JOB_SPEC_PKG_NAME}.yaml" + local spec_names_to_save="${SPACK_JOB_SPEC_PKG_NAME}" + + if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then + IFS=';' read -ra DEPS <<< "${SPACK_RELATED_BUILDS}" + for i in "${DEPS[@]}"; do + depPkgName="${i}" + spec_names_to_save="${spec_names_to_save} ${depPkgName}" + JOB_DEPS_PKG_NAMES+=("${depPkgName}") + done + fi - read -ra PARTSARRAY <<< "${CI_JOB_NAME}" - local pkgName="${PARTSARRAY[0]}" - local pkgVersion="${PARTSARRAY[1]}" - local compiler="${PARTSARRAY[2]}" - local osarch="${PARTSARRAY[3]}" - local buildGroup="${PARTSARRAY[@]:4}" # get everything after osarch - - JOB_GROUP="${buildGroup}" - JOB_PKG_NAME="${pkgName}" - SPEC_YAML_PATH="${SPEC_DIR}/${pkgName}.yaml" - local root_spec_name="${ROOT_SPEC}" - local spec_names_to_save="${pkgName}" - - IFS=';' read -ra DEPS <<< "${DEPENDENCIES}" - for i in "${DEPS[@]}"; do - read -ra PARTSARRAY <<< "${i}" - pkgName="${PARTSARRAY[0]}" - spec_names_to_save="${spec_names_to_save} ${pkgName}" - JOB_DEPS_PKG_NAMES+=("${pkgName}") - done - - spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec "${root_spec_name}" --yaml-dir "${SPEC_DIR}" + if [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then + # This corresponds to a bootstrapping phase where we need to + # rely on any available compiler to build the package (i.e. the + # compiler needed to be stripped from the spec), and thus we need + # to concretize the root spec again. + spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec "${SPACK_ROOT_SPEC}" --yaml-dir "${SPEC_DIR}" + else + # in this case, either we're relying on Spack to install missing compiler + # bootstrapped in a previous phase, or else we only had one phase (like a + # site which already knows what compilers are available on it's runners), + # so we don't want to concretize that root spec again. The reason we need + # this in the first case (bootstrapped compiler), is that we can't concretize + # a spec at this point if we're going to ask spack to "install_missing_compilers". + tmp_dir=$(mktemp -d) + TMP_YAML_PATH="${tmp_dir}/root.yaml" + ROOT_SPEC_YAML=$(spack python -c "import base64 ; import zlib ; print(str(zlib.decompress(base64.b64decode('${SPACK_ROOT_SPEC}')).decode('utf-8')))") + echo "${ROOT_SPEC_YAML}" > "${TMP_YAML_PATH}" + spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec-yaml "${TMP_YAML_PATH}" --yaml-dir "${SPEC_DIR}" + rm -rf ${tmp_dir} + fi } begin_logging -gen_full_specs_for_job_and_deps +echo "Running job for spec: ${CI_JOB_NAME}" + +# This should create the directory we referred to as GNUPGHOME earlier +spack gpg list -echo "Building package ${CDASH_BUILD_NAME}, ${HASH}, ${MIRROR_URL}" +# Importing the secret key using gpg2 directly should allow to +# sign and verify both +set +x +KEY_IMPORT_RESULT=`echo ${SPACK_SIGNING_KEY} | base64 --decode | gpg2 --import` +check_error $? "gpg2 --import" +set -x + +spack gpg list --trusted +spack gpg list --signing + +# To have spack install missing compilers, we need to add a custom +# configuration scope, then we pass that to the package installation +# command +CUSTOM_CONFIG_SCOPE_DIR="${TEMP_DIR}/config_scope" +mkdir -p "${CUSTOM_CONFIG_SCOPE_DIR}" +CUSTOM_CONFIG_SCOPE_ARG="" + +if [ "${SPACK_COMPILER_ACTION}" == "INSTALL_MISSING" ]; then + echo "Make sure bootstrapped compiler will be installed" + custom_config_file_path="${CUSTOM_CONFIG_SCOPE_DIR}/config.yaml" + cat <<CONFIG_STUFF > "${custom_config_file_path}" +config: + install_missing_compilers: true +CONFIG_STUFF + CUSTOM_CONFIG_SCOPE_ARG="-C ${CUSTOM_CONFIG_SCOPE_DIR}" + # Configure the binary mirror where, if needed, this jobs compiler + # was installed in binary pacakge form, then tell spack to + # install_missing_compilers. +elif [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then + echo "Just find any available compiler" + spack compiler find +else + echo "No compiler action to be taken" +fi # Finally, list the compilers spack knows about echo "Compiler Configurations:" spack config get compilers +# Write full-deps yamls for this job spec and its dependencies +gen_full_specs_for_job_and_deps + # Make the build_cache directory if it doesn't exist mkdir -p "${BUILD_CACHE_DIR}" @@ -204,41 +272,41 @@ mkdir -p "${BUILD_CACHE_DIR}" # to fail. JOB_BUILD_CACHE_ENTRY_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${SPEC_YAML_PATH}"` if [[ $? -ne 0 ]]; then - echo "ERROR, unable to get buildcache entry name for job ${CI_JOB_NAME} (spec: ${CDASH_BUILD_NAME})" + echo "ERROR, unable to get buildcache entry name for job ${CI_JOB_NAME}" exit 1 fi -# This should create the directory we referred to as GNUPGHOME earlier -spack gpg list - -# Importing the secret key using gpg2 directly should allow to -# sign and verify both -set +x -KEY_IMPORT_RESULT=`echo ${SPACK_SIGNING_KEY} | base64 --decode | gpg2 --import` -check_error $? "gpg2 --import" -set -x - -spack gpg list --trusted -spack gpg list --signing - -# Whether we have to build the spec or download it pre-built, we expect to find -# the cdash build id file sitting in this location afterwards. -JOB_CDASH_ID_FILE="${BUILD_CACHE_DIR}/${JOB_BUILD_CACHE_ENTRY_NAME}.cdashid" +if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then + # Whether we have to build the spec or download it pre-built, we expect to find + # the cdash build id file sitting in this location afterwards. + JOB_CDASH_ID_FILE="${BUILD_CACHE_DIR}/${JOB_BUILD_CACHE_ENTRY_NAME}.cdashid" +fi # Finally, we can check the spec we have been tasked with build against # the built binary on the remote mirror to see if it needs to be rebuilt -spack -d buildcache check --spec-yaml "${SPEC_YAML_PATH}" --mirror-url "${MIRROR_URL}" --rebuild-on-error +spack -d buildcache check --spec-yaml "${SPEC_YAML_PATH}" --mirror-url "${SPACK_MIRROR_URL}" --rebuild-on-error if [[ $? -ne 0 ]]; then # Configure mirror spack mirror add local_artifact_mirror "file://${LOCAL_MIRROR}" - JOB_CDASH_ID="NONE" + if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then + JOB_CDASH_ID="NONE" - # Install package, using the buildcache from the local mirror to - # satisfy dependencies. - BUILD_ID_LINE=`spack -d -k -v install --use-cache --keep-stage --cdash-upload-url "${CDASH_UPLOAD_URL}" --cdash-build "${CDASH_BUILD_NAME}" --cdash-site "Spack AWS Gitlab Instance" --cdash-track "${JOB_GROUP}" -f "${SPEC_YAML_PATH}" | grep "buildSummary\\.php"` - check_error $? "spack install" + # Install package, using the buildcache from the local mirror to + # satisfy dependencies. + BUILD_ID_LINE=`spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage --cdash-upload-url "${CDASH_UPLOAD_URL}" --cdash-build "${SPACK_CDASH_BUILD_NAME}" --cdash-site "${SPACK_CDASH_SITE}" --cdash-track "${SPACK_JOB_SPEC_BUILDGROUP}" -f "${SPEC_YAML_PATH}" | grep "buildSummary\\.php"` + check_error $? "spack install" + + # By parsing the output of the "spack install" command, we can get the + # buildid generated for us by CDash + JOB_CDASH_ID=$(extract_build_id "${BUILD_ID_LINE}") + + # Write the .cdashid file to the buildcache as well + echo "${JOB_CDASH_ID}" >> ${JOB_CDASH_ID_FILE} + else + spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage -f "${SPEC_YAML_PATH}" + fi # Copy some log files into an artifact location, once we have a way # to provide a spec.yaml file to more spack commands (e.g. "location") @@ -248,73 +316,76 @@ if [[ $? -ne 0 ]]; then # cp "${build_log_file}" "${JOB_LOG_DIR}/" # cp "${config_log_file}" "${JOB_LOG_DIR}/" - # By parsing the output of the "spack install" command, we can get the - # buildid generated for us by CDash - JOB_CDASH_ID=$(extract_build_id "${BUILD_ID_LINE}") - # Create buildcache entry for this package, reading the spec from the yaml # file. spack -d buildcache create --spec-yaml "${SPEC_YAML_PATH}" -a -f -d "${LOCAL_MIRROR}" --no-rebuild-index check_error $? "spack buildcache create" - # Write the .cdashid file to the buildcache as well - echo "${JOB_CDASH_ID}" >> ${JOB_CDASH_ID_FILE} - # TODO: The upload-s3 command should eventually be replaced with something # like: "spack buildcache put <mirror> <spec>", when that subcommand is # properly implemented. - spack -d upload-s3 spec --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" - check_error $? "spack upload-s3 spec" + if [ ! -z "${SPACK_S3_UPLOAD_MIRROR_URL}" ] ; then + spack -d upload-s3 spec --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --endpoint-url "${SPACK_S3_UPLOAD_MIRROR_URL}" + check_error $? "spack upload-s3 spec" + else + spack -d buildcache copy --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --destination-url "${SPACK_MIRROR_URL}" + fi else - echo "spec ${CDASH_BUILD_NAME} is already up to date on remote mirror, downloading it" + echo "spec ${CI_JOB_NAME} is already up to date on remote mirror, downloading it" # Configure remote mirror so we can download buildcache entry - spack mirror add remote_binary_mirror ${MIRROR_URL} + spack mirror add remote_binary_mirror ${SPACK_MIRROR_URL} # Now download it - spack -d buildcache download --spec-yaml "${SPEC_YAML_PATH}" --path "${BUILD_CACHE_DIR}/" --require-cdashid + BUILDCACHE_DL_ARGS=("--spec-yaml" "${SPEC_YAML_PATH}" "--path" "${BUILD_CACHE_DIR}/" ) + if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then + BUILDCACHE_DL_ARGS+=( "--require-cdashid" ) + fi + spack -d buildcache download "${BUILDCACHE_DL_ARGS[@]}" check_error $? "spack buildcache download" fi # The next step is to relate this job to the jobs it depends on -if [ -f "${JOB_CDASH_ID_FILE}" ]; then - JOB_CDASH_BUILD_ID=$(<${JOB_CDASH_ID_FILE}) +if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then + if [ -f "${JOB_CDASH_ID_FILE}" ]; then + JOB_CDASH_BUILD_ID=$(<${JOB_CDASH_ID_FILE}) - if [ "${JOB_CDASH_BUILD_ID}" == "NONE" ]; then - echo "ERROR: unable to read this jobs id from ${JOB_CDASH_ID_FILE}" - exit 1 - fi + if [ "${JOB_CDASH_BUILD_ID}" == "NONE" ]; then + echo "ERROR: unable to read this jobs id from ${JOB_CDASH_ID_FILE}" + exit 1 + fi - # Now get CDash ids for dependencies and "relate" each dependency build - # with this jobs build - for DEP_PKG_NAME in "${JOB_DEPS_PKG_NAMES[@]}"; do - echo "Getting cdash id for dependency --> ${DEP_PKG_NAME} <--" - DEP_SPEC_YAML_PATH="${SPEC_DIR}/${DEP_PKG_NAME}.yaml" - DEP_JOB_BUILDCACHE_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${DEP_SPEC_YAML_PATH}"` - - if [[ $? -eq 0 ]]; then - DEP_JOB_ID_FILE="${BUILD_CACHE_DIR}/${DEP_JOB_BUILDCACHE_NAME}.cdashid" - echo "DEP_JOB_ID_FILE path = ${DEP_JOB_ID_FILE}" - - if [ -f "${DEP_JOB_ID_FILE}" ]; then - DEP_JOB_CDASH_BUILD_ID=$(<${DEP_JOB_ID_FILE}) - echo "File ${DEP_JOB_ID_FILE} contained value ${DEP_JOB_CDASH_BUILD_ID}" - echo "Relating builds -> ${CDASH_BUILD_NAME} (buildid=${JOB_CDASH_BUILD_ID}) depends on ${DEP_PKG_NAME} (buildid=${DEP_JOB_CDASH_BUILD_ID})" - relateBuildsPostBody="$(get_relate_builds_post_data "${CDASH_PROJECT}" ${JOB_CDASH_BUILD_ID} ${DEP_JOB_CDASH_BUILD_ID})" - relateBuildsResult=`curl "${DEP_JOB_RELATEBUILDS_URL}" -H "Content-Type: application/json" -H "Accept: application/json" -d "${relateBuildsPostBody}"` - echo "Result of curl request: ${relateBuildsResult}" + # Now get CDash ids for dependencies and "relate" each dependency build + # with this jobs build + for DEP_PKG_NAME in "${JOB_DEPS_PKG_NAMES[@]}"; do + echo "Getting cdash id for dependency --> ${DEP_PKG_NAME} <--" + DEP_SPEC_YAML_PATH="${SPEC_DIR}/${DEP_PKG_NAME}.yaml" + DEP_JOB_BUILDCACHE_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${DEP_SPEC_YAML_PATH}"` + + if [[ $? -eq 0 ]]; then + DEP_JOB_ID_FILE="${BUILD_CACHE_DIR}/${DEP_JOB_BUILDCACHE_NAME}.cdashid" + echo "DEP_JOB_ID_FILE path = ${DEP_JOB_ID_FILE}" + + if [ -f "${DEP_JOB_ID_FILE}" ]; then + DEP_JOB_CDASH_BUILD_ID=$(<${DEP_JOB_ID_FILE}) + echo "File ${DEP_JOB_ID_FILE} contained value ${DEP_JOB_CDASH_BUILD_ID}" + echo "Relating builds -> ${SPACK_CDASH_BUILD_NAME} (buildid=${JOB_CDASH_BUILD_ID}) depends on ${DEP_PKG_NAME} (buildid=${DEP_JOB_CDASH_BUILD_ID})" + relateBuildsPostBody="$(get_relate_builds_post_data "${SPACK_CDASH_PROJECT}" ${JOB_CDASH_BUILD_ID} ${DEP_JOB_CDASH_BUILD_ID})" + relateBuildsResult=`curl "${DEP_JOB_RELATEBUILDS_URL}" -H "Content-Type: application/json" -H "Accept: application/json" -d "${relateBuildsPostBody}"` + echo "Result of curl request: ${relateBuildsResult}" + else + echo "ERROR: Did not find expected .cdashid file for dependency: ${DEP_JOB_ID_FILE}" + exit 1 + fi else - echo "ERROR: Did not find expected .cdashid file for dependency: ${DEP_JOB_ID_FILE}" + echo "ERROR: Unable to get buildcache entry name for ${DEP_SPEC_NAME}" exit 1 fi - else - echo "ERROR: Unable to get buildcache entry name for ${DEP_SPEC_NAME}" - exit 1 - fi - done -else - echo "ERROR: Did not find expected .cdashid file ${JOB_CDASH_ID_FILE}" - exit 1 + done + else + echo "ERROR: Did not find expected .cdashid file ${JOB_CDASH_ID_FILE}" + exit 1 + fi fi # Show the size of the buildcache and a list of what's in it, directly |