Implement an optional compiler bootstrapping phase

2 files changed, 193 insertions, 116 deletions

diff --git a/bin/generate-gitlab-ci-yml.sh b/bin/generate-gitlab-ci-yml.sh
index 1a021785d3..b414d6c404 100755
--- a/bin/generate-gitlab-ci-yml.sh
+++ b/bin/generate-gitlab-ci-yml.sh
@@ -19,8 +19,10 @@ if [ -z "${SPACK_RELEASE_ENVIRONMENT_PATH}" ] ; then
 fi
 
 if [ -z "${CDASH_AUTH_TOKEN}" ] ; then
-    echo "ERROR: missing variable: CDASH_AUTH_TOKEN" >&2
-    exit 1
+    echo "WARNING: missing variable: CDASH_AUTH_TOKEN" >&2
+else
+    token_file="${temp_dir}/cdash_auth_token"
+    echo ${CDASH_AUTH_TOKEN} > ${token_file}
 fi
 
 if [ -z "${SPACK_RELEASE_ENVIRONMENT_REPO}" ] ; then
@@ -51,11 +53,14 @@ fi
 
 cd $env_dir
 
-token_file="${temp_dir}/cdash_auth_token"
-echo ${CDASH_AUTH_TOKEN} > ${token_file}
+# The next commands generates the .gitlab-ci.yml (and optionally creates a
+# buildgroup in cdash)
+RELEASE_JOBS_ARGS=("--output-file" "${gen_ci_file}")
+if [ ! -z "${token_file}" ]; then
+    RELEASE_JOBS_ARGS+=("--cdash-credentials" "${token_file}")
+fi
 
-# This commands generates the .gitlab-ci.yml and creates buildgroup in cdash
-spack release-jobs --force --output-file ${gen_ci_file} --cdash-credentials ${token_file}
+spack release-jobs "${RELEASE_JOBS_ARGS[@]}"
 
 if [[ $? -ne 0 ]]; then
     echo "spack release-jobs command failed"
@@ -64,6 +69,7 @@ fi
 
 cp ${gen_ci_file} "${original_directory}/.gitlab-ci.yml"
 
+# Remove global from here, it's clobbering people git identity config
 git config --global user.email "robot@spack.io"
 git config --global user.name "Build Robot"
 
diff --git a/bin/rebuild-package.sh b/bin/rebuild-package.sh
index 3813a92a1f..657e431052 100755
--- a/bin/rebuild-package.sh
+++ b/bin/rebuild-package.sh
@@ -12,29 +12,44 @@
 ### not (i.e. the source code has changed in a way that caused a change in the
 ### full_hash of the spec), this script will build the package, create a
 ### binary cache for it, and then push all related files to the remote binary
-### mirror.  This script also communicates with a remote CDash instance to
-### share status on the package build process.
+### mirror.  This script also optionally communicates with a remote CDash
+### instance to share status on the package build process.
 ###
-### The following environment variables are expected to be set in order for
-### the various elements in this script to function properly.  Listed first
-### are two defaults we rely on from gitlab, then three we set up in the
-### variables section of gitlab ourselves, and finally four variables
-### written into the .gitlab-ci.yml file.
+### The following environment variables are (possibly) used within this script
+### in order for the various elements function properly.
+###
+### First are two defaults we rely on from gitlab:
 ###
 ### CI_PROJECT_DIR
 ### CI_JOB_NAME
 ###
+### The following must be set up in the variables section of gitlab:
+###
 ### AWS_ACCESS_KEY_ID
 ### AWS_SECRET_ACCESS_KEY
 ### SPACK_SIGNING_KEY
 ###
-### CDASH_BASE_URL
-### CDASH_PROJECT
-### CDASH_PROJECT_ENC
-### CDASH_BUILD_NAME
-### ROOT_SPEC
-### DEPENDENCIES
-### MIRROR_URL
+### SPACK_S3_UPLOAD_MIRROR_URL         // only required in the short term for the cloud case
+###
+### The following variabes are defined by the ci generation process and are
+### required:
+###
+### SPACK_ENABLE_CDASH
+### SPACK_ROOT_SPEC
+### SPACK_MIRROR_URL
+### SPACK_JOB_SPEC_PKG_NAME
+### SPACK_COMPILER_ACTION
+###
+### Finally, these variables are optionally defined by the ci generation
+### process, and may or may not be present:
+###
+### SPACK_CDASH_BASE_URL
+### SPACK_CDASH_PROJECT
+### SPACK_CDASH_PROJECT_ENC
+### SPACK_CDASH_BUILD_NAME
+### SPACK_CDASH_SITE
+### SPACK_RELATED_BUILDS
+### SPACK_JOB_SPEC_BUILDGROUP
 ###
 
 shopt -s expand_aliases
@@ -48,14 +63,19 @@ SPEC_DIR="${TEMP_DIR}/specs"
 LOCAL_MIRROR="${CI_PROJECT_DIR}/local_mirror"
 BUILD_CACHE_DIR="${LOCAL_MIRROR}/build_cache"
 SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin"
-CDASH_UPLOAD_URL="${CDASH_BASE_URL}/submit.php?project=${CDASH_PROJECT_ENC}"
-DEP_JOB_RELATEBUILDS_URL="${CDASH_BASE_URL}/api/v1/relateBuilds.php"
-declare -a JOB_DEPS_PKG_NAMES
+
+if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
+    CDASH_UPLOAD_URL="${SPACK_CDASH_BASE_URL}/submit.php?project=${SPACK_CDASH_PROJECT_ENC}"
+    DEP_JOB_RELATEBUILDS_URL="${SPACK_CDASH_BASE_URL}/api/v1/relateBuilds.php"
+    declare -a JOB_DEPS_PKG_NAMES
+fi
 
 export SPACK_ROOT=${CI_PROJECT_DIR}
-export PATH="${SPACK_BIN_DIR}:${PATH}"
+# export PATH="${SPACK_BIN_DIR}:${PATH}"
 export GNUPGHOME="${CI_PROJECT_DIR}/opt/spack/gpg"
 
+. "${CI_PROJECT_DIR}/share/spack/setup-env.sh"
+
 mkdir -p ${JOB_LOG_DIR}
 mkdir -p ${SPEC_DIR}
 
@@ -160,41 +180,89 @@ EOF
 }
 
 gen_full_specs_for_job_and_deps() {
+    SPEC_YAML_PATH="${SPEC_DIR}/${SPACK_JOB_SPEC_PKG_NAME}.yaml"
+    local spec_names_to_save="${SPACK_JOB_SPEC_PKG_NAME}"
+
+    if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
+        IFS=';' read -ra DEPS <<< "${SPACK_RELATED_BUILDS}"
+        for i in "${DEPS[@]}"; do
+            depPkgName="${i}"
+            spec_names_to_save="${spec_names_to_save} ${depPkgName}"
+            JOB_DEPS_PKG_NAMES+=("${depPkgName}")
+        done
+    fi
 
-    read -ra PARTSARRAY <<< "${CI_JOB_NAME}"
-    local pkgName="${PARTSARRAY[0]}"
-    local pkgVersion="${PARTSARRAY[1]}"
-    local compiler="${PARTSARRAY[2]}"
-    local osarch="${PARTSARRAY[3]}"
-    local buildGroup="${PARTSARRAY[@]:4}" # get everything after osarch
-
-    JOB_GROUP="${buildGroup}"
-    JOB_PKG_NAME="${pkgName}"
-    SPEC_YAML_PATH="${SPEC_DIR}/${pkgName}.yaml"
-    local root_spec_name="${ROOT_SPEC}"
-    local spec_names_to_save="${pkgName}"
-
-    IFS=';' read -ra DEPS <<< "${DEPENDENCIES}"
-    for i in "${DEPS[@]}"; do
-        read -ra PARTSARRAY <<< "${i}"
-        pkgName="${PARTSARRAY[0]}"
-        spec_names_to_save="${spec_names_to_save} ${pkgName}"
-        JOB_DEPS_PKG_NAMES+=("${pkgName}")
-    done
-
-    spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec "${root_spec_name}" --yaml-dir "${SPEC_DIR}"
+    if [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then
+        # This corresponds to a bootstrapping phase where we need to
+        # rely on any available compiler to build the package (i.e. the
+        # compiler needed to be stripped from the spec), and thus we need
+        # to concretize the root spec again.
+        spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec "${SPACK_ROOT_SPEC}" --yaml-dir "${SPEC_DIR}"
+    else
+        # in this case, either we're relying on Spack to install missing compiler
+        # bootstrapped in a previous phase, or else we only had one phase (like a
+        # site which already knows what compilers are available on it's runners),
+        # so we don't want to concretize that root spec again.  The reason we need
+        # this in the first case (bootstrapped compiler), is that we can't concretize
+        # a spec at this point if we're going to ask spack to "install_missing_compilers".
+        tmp_dir=$(mktemp -d)
+        TMP_YAML_PATH="${tmp_dir}/root.yaml"
+        ROOT_SPEC_YAML=$(spack python -c "import base64 ; import zlib ; print(str(zlib.decompress(base64.b64decode('${SPACK_ROOT_SPEC}')).decode('utf-8')))")
+        echo "${ROOT_SPEC_YAML}" > "${TMP_YAML_PATH}"
+        spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec-yaml "${TMP_YAML_PATH}" --yaml-dir "${SPEC_DIR}"
+        rm -rf ${tmp_dir}
+    fi
 }
 
 begin_logging
 
-gen_full_specs_for_job_and_deps
+echo "Running job for spec: ${CI_JOB_NAME}"
+
+# This should create the directory we referred to as GNUPGHOME earlier
+spack gpg list
 
-echo "Building package ${CDASH_BUILD_NAME}, ${HASH}, ${MIRROR_URL}"
+# Importing the secret key using gpg2 directly should allow to
+# sign and verify both
+set +x
+KEY_IMPORT_RESULT=`echo ${SPACK_SIGNING_KEY} | base64 --decode | gpg2 --import`
+check_error $? "gpg2 --import"
+set -x
+
+spack gpg list --trusted
+spack gpg list --signing
+
+# To have spack install missing compilers, we need to add a custom
+# configuration scope, then we pass that to the package installation
+# command
+CUSTOM_CONFIG_SCOPE_DIR="${TEMP_DIR}/config_scope"
+mkdir -p "${CUSTOM_CONFIG_SCOPE_DIR}"
+CUSTOM_CONFIG_SCOPE_ARG=""
+
+if [ "${SPACK_COMPILER_ACTION}" == "INSTALL_MISSING" ]; then
+    echo "Make sure bootstrapped compiler will be installed"
+    custom_config_file_path="${CUSTOM_CONFIG_SCOPE_DIR}/config.yaml"
+      cat <<CONFIG_STUFF > "${custom_config_file_path}"
+config:
+  install_missing_compilers: true
+CONFIG_STUFF
+    CUSTOM_CONFIG_SCOPE_ARG="-C ${CUSTOM_CONFIG_SCOPE_DIR}"
+    # Configure the binary mirror where, if needed, this jobs compiler
+    # was installed in binary pacakge form, then tell spack to
+    # install_missing_compilers.
+elif [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then
+    echo "Just find any available compiler"
+    spack compiler find
+else
+    echo "No compiler action to be taken"
+fi
 
 # Finally, list the compilers spack knows about
 echo "Compiler Configurations:"
 spack config get compilers
 
+# Write full-deps yamls for this job spec and its dependencies
+gen_full_specs_for_job_and_deps
+
 # Make the build_cache directory if it doesn't exist
 mkdir -p "${BUILD_CACHE_DIR}"
 
@@ -204,41 +272,41 @@ mkdir -p "${BUILD_CACHE_DIR}"
 # to fail.
 JOB_BUILD_CACHE_ENTRY_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${SPEC_YAML_PATH}"`
 if [[ $? -ne 0 ]]; then
-    echo "ERROR, unable to get buildcache entry name for job ${CI_JOB_NAME} (spec: ${CDASH_BUILD_NAME})"
+    echo "ERROR, unable to get buildcache entry name for job ${CI_JOB_NAME}"
     exit 1
 fi
 
-# This should create the directory we referred to as GNUPGHOME earlier
-spack gpg list
-
-# Importing the secret key using gpg2 directly should allow to
-# sign and verify both
-set +x
-KEY_IMPORT_RESULT=`echo ${SPACK_SIGNING_KEY} | base64 --decode | gpg2 --import`
-check_error $? "gpg2 --import"
-set -x
-
-spack gpg list --trusted
-spack gpg list --signing
-
-# Whether we have to build the spec or download it pre-built, we expect to find
-# the cdash build id file sitting in this location afterwards.
-JOB_CDASH_ID_FILE="${BUILD_CACHE_DIR}/${JOB_BUILD_CACHE_ENTRY_NAME}.cdashid"
+if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
+    # Whether we have to build the spec or download it pre-built, we expect to find
+    # the cdash build id file sitting in this location afterwards.
+    JOB_CDASH_ID_FILE="${BUILD_CACHE_DIR}/${JOB_BUILD_CACHE_ENTRY_NAME}.cdashid"
+fi
 
 # Finally, we can check the spec we have been tasked with build against
 # the built binary on the remote mirror to see if it needs to be rebuilt
-spack -d buildcache check --spec-yaml "${SPEC_YAML_PATH}" --mirror-url "${MIRROR_URL}" --rebuild-on-error
+spack -d buildcache check --spec-yaml "${SPEC_YAML_PATH}" --mirror-url "${SPACK_MIRROR_URL}" --rebuild-on-error
 
 if [[ $? -ne 0 ]]; then
     # Configure mirror
     spack mirror add local_artifact_mirror "file://${LOCAL_MIRROR}"
 
-    JOB_CDASH_ID="NONE"
+    if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
+        JOB_CDASH_ID="NONE"
 
-    # Install package, using the buildcache from the local mirror to
-    # satisfy dependencies.
-    BUILD_ID_LINE=`spack -d -k -v install --use-cache --keep-stage --cdash-upload-url "${CDASH_UPLOAD_URL}" --cdash-build "${CDASH_BUILD_NAME}" --cdash-site "Spack AWS Gitlab Instance" --cdash-track "${JOB_GROUP}" -f "${SPEC_YAML_PATH}" | grep "buildSummary\\.php"`
-    check_error $? "spack install"
+        # Install package, using the buildcache from the local mirror to
+        # satisfy dependencies.
+        BUILD_ID_LINE=`spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage --cdash-upload-url "${CDASH_UPLOAD_URL}" --cdash-build "${SPACK_CDASH_BUILD_NAME}" --cdash-site "${SPACK_CDASH_SITE}" --cdash-track "${SPACK_JOB_SPEC_BUILDGROUP}" -f "${SPEC_YAML_PATH}" | grep "buildSummary\\.php"`
+        check_error $? "spack install"
+
+        # By parsing the output of the "spack install" command, we can get the
+        # buildid generated for us by CDash
+        JOB_CDASH_ID=$(extract_build_id "${BUILD_ID_LINE}")
+
+        # Write the .cdashid file to the buildcache as well
+        echo "${JOB_CDASH_ID}" >> ${JOB_CDASH_ID_FILE}
+    else
+        spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage -f "${SPEC_YAML_PATH}"
+    fi
 
     # Copy some log files into an artifact location, once we have a way
     # to provide a spec.yaml file to more spack commands (e.g. "location")
@@ -248,73 +316,76 @@ if [[ $? -ne 0 ]]; then
     # cp "${build_log_file}" "${JOB_LOG_DIR}/"
     # cp "${config_log_file}" "${JOB_LOG_DIR}/"
 
-    # By parsing the output of the "spack install" command, we can get the
-    # buildid generated for us by CDash
-    JOB_CDASH_ID=$(extract_build_id "${BUILD_ID_LINE}")
-
     # Create buildcache entry for this package, reading the spec from the yaml
     # file.
     spack -d buildcache create --spec-yaml "${SPEC_YAML_PATH}" -a -f -d "${LOCAL_MIRROR}" --no-rebuild-index
     check_error $? "spack buildcache create"
 
-    # Write the .cdashid file to the buildcache as well
-    echo "${JOB_CDASH_ID}" >> ${JOB_CDASH_ID_FILE}
-
     # TODO: The upload-s3 command should eventually be replaced with something
     # like: "spack buildcache put <mirror> <spec>", when that subcommand is
     # properly implemented.
-    spack -d upload-s3 spec --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}"
-    check_error $? "spack upload-s3 spec"
+    if [ ! -z "${SPACK_S3_UPLOAD_MIRROR_URL}" ] ; then
+        spack -d upload-s3 spec --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --endpoint-url "${SPACK_S3_UPLOAD_MIRROR_URL}"
+        check_error $? "spack upload-s3 spec"
+    else
+        spack -d buildcache copy --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --destination-url "${SPACK_MIRROR_URL}"
+    fi
 else
-    echo "spec ${CDASH_BUILD_NAME} is already up to date on remote mirror, downloading it"
+    echo "spec ${CI_JOB_NAME} is already up to date on remote mirror, downloading it"
 
     # Configure remote mirror so we can download buildcache entry
-    spack mirror add remote_binary_mirror ${MIRROR_URL}
+    spack mirror add remote_binary_mirror ${SPACK_MIRROR_URL}
 
     # Now download it
-    spack -d buildcache download --spec-yaml "${SPEC_YAML_PATH}" --path "${BUILD_CACHE_DIR}/" --require-cdashid
+    BUILDCACHE_DL_ARGS=("--spec-yaml" "${SPEC_YAML_PATH}" "--path" "${BUILD_CACHE_DIR}/" )
+    if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
+        BUILDCACHE_DL_ARGS+=( "--require-cdashid" )
+    fi
+    spack -d buildcache download "${BUILDCACHE_DL_ARGS[@]}"
     check_error $? "spack buildcache download"
 fi
 
 # The next step is to relate this job to the jobs it depends on
-if [ -f "${JOB_CDASH_ID_FILE}" ]; then
-    JOB_CDASH_BUILD_ID=$(<${JOB_CDASH_ID_FILE})
+if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
+    if [ -f "${JOB_CDASH_ID_FILE}" ]; then
+        JOB_CDASH_BUILD_ID=$(<${JOB_CDASH_ID_FILE})
 
-    if [ "${JOB_CDASH_BUILD_ID}" == "NONE" ]; then
-        echo "ERROR: unable to read this jobs id from ${JOB_CDASH_ID_FILE}"
-        exit 1
-    fi
+        if [ "${JOB_CDASH_BUILD_ID}" == "NONE" ]; then
+            echo "ERROR: unable to read this jobs id from ${JOB_CDASH_ID_FILE}"
+            exit 1
+        fi
 
-    # Now get CDash ids for dependencies and "relate" each dependency build
-    # with this jobs build
-    for DEP_PKG_NAME in "${JOB_DEPS_PKG_NAMES[@]}"; do
-        echo "Getting cdash id for dependency --> ${DEP_PKG_NAME} <--"
-        DEP_SPEC_YAML_PATH="${SPEC_DIR}/${DEP_PKG_NAME}.yaml"
-        DEP_JOB_BUILDCACHE_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${DEP_SPEC_YAML_PATH}"`
-
-        if [[ $? -eq 0 ]]; then
-            DEP_JOB_ID_FILE="${BUILD_CACHE_DIR}/${DEP_JOB_BUILDCACHE_NAME}.cdashid"
-            echo "DEP_JOB_ID_FILE path = ${DEP_JOB_ID_FILE}"
-
-            if [ -f "${DEP_JOB_ID_FILE}" ]; then
-                DEP_JOB_CDASH_BUILD_ID=$(<${DEP_JOB_ID_FILE})
-                echo "File ${DEP_JOB_ID_FILE} contained value ${DEP_JOB_CDASH_BUILD_ID}"
-                echo "Relating builds -> ${CDASH_BUILD_NAME} (buildid=${JOB_CDASH_BUILD_ID}) depends on ${DEP_PKG_NAME} (buildid=${DEP_JOB_CDASH_BUILD_ID})"
-                relateBuildsPostBody="$(get_relate_builds_post_data "${CDASH_PROJECT}" ${JOB_CDASH_BUILD_ID} ${DEP_JOB_CDASH_BUILD_ID})"
-                relateBuildsResult=`curl "${DEP_JOB_RELATEBUILDS_URL}" -H "Content-Type: application/json" -H "Accept: application/json" -d "${relateBuildsPostBody}"`
-                echo "Result of curl request: ${relateBuildsResult}"
+        # Now get CDash ids for dependencies and "relate" each dependency build
+        # with this jobs build
+        for DEP_PKG_NAME in "${JOB_DEPS_PKG_NAMES[@]}"; do
+            echo "Getting cdash id for dependency --> ${DEP_PKG_NAME} <--"
+            DEP_SPEC_YAML_PATH="${SPEC_DIR}/${DEP_PKG_NAME}.yaml"
+            DEP_JOB_BUILDCACHE_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${DEP_SPEC_YAML_PATH}"`
+
+            if [[ $? -eq 0 ]]; then
+                DEP_JOB_ID_FILE="${BUILD_CACHE_DIR}/${DEP_JOB_BUILDCACHE_NAME}.cdashid"
+                echo "DEP_JOB_ID_FILE path = ${DEP_JOB_ID_FILE}"
+
+                if [ -f "${DEP_JOB_ID_FILE}" ]; then
+                    DEP_JOB_CDASH_BUILD_ID=$(<${DEP_JOB_ID_FILE})
+                    echo "File ${DEP_JOB_ID_FILE} contained value ${DEP_JOB_CDASH_BUILD_ID}"
+                    echo "Relating builds -> ${SPACK_CDASH_BUILD_NAME} (buildid=${JOB_CDASH_BUILD_ID}) depends on ${DEP_PKG_NAME} (buildid=${DEP_JOB_CDASH_BUILD_ID})"
+                    relateBuildsPostBody="$(get_relate_builds_post_data "${SPACK_CDASH_PROJECT}" ${JOB_CDASH_BUILD_ID} ${DEP_JOB_CDASH_BUILD_ID})"
+                    relateBuildsResult=`curl "${DEP_JOB_RELATEBUILDS_URL}" -H "Content-Type: application/json" -H "Accept: application/json" -d "${relateBuildsPostBody}"`
+                    echo "Result of curl request: ${relateBuildsResult}"
+                else
+                    echo "ERROR: Did not find expected .cdashid file for dependency: ${DEP_JOB_ID_FILE}"
+                    exit 1
+                fi
             else
-                echo "ERROR: Did not find expected .cdashid file for dependency: ${DEP_JOB_ID_FILE}"
+                echo "ERROR: Unable to get buildcache entry name for ${DEP_SPEC_NAME}"
                 exit 1
             fi
-        else
-            echo "ERROR: Unable to get buildcache entry name for ${DEP_SPEC_NAME}"
-            exit 1
-        fi
-    done
-else
-    echo "ERROR: Did not find expected .cdashid file ${JOB_CDASH_ID_FILE}"
-    exit 1
+        done
+    else
+        echo "ERROR: Did not find expected .cdashid file ${JOB_CDASH_ID_FILE}"
+        exit 1
+    fi
 fi
 
 # Show the size of the buildcache and a list of what's in it, directly