diff options
| -rwxr-xr-x | bin/rebuild-index.sh | 13 | ||||
| -rwxr-xr-x | bin/rebuild-package.sh | 316 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/upload_s3.py | 212 |
3 files changed, 541 insertions, 0 deletions
diff --git a/bin/rebuild-index.sh b/bin/rebuild-index.sh new file mode 100755 index 0000000000..09e14a9cee --- /dev/null +++ b/bin/rebuild-index.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +set -x + +SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin" +export PATH="${SPACK_BIN_DIR}:${PATH}" + +spack upload-s3 index diff --git a/bin/rebuild-package.sh b/bin/rebuild-package.sh new file mode 100755 index 0000000000..459303e95e --- /dev/null +++ b/bin/rebuild-package.sh @@ -0,0 +1,316 @@ +#!/bin/bash + +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +### +### This script represents a gitlab-ci job, corresponding to a single release +### spec. As such this script must first decide whether or not the spec it +### has been assigned is up to date on the remote binary mirror. If it is +### not (i.e. the source code has changed in a way that caused a change in the +### full_hash of the spec), this script will build the package, create a +### binary cache for it, and then push all related files to the remote binary +### mirror. This script also communicates with a remote CDash instance to +### share status on the package build process. +### +### The following environment variables are expected to be set in order for +### the various elements in this script to function properly. Listed first +### are two defaults we rely on from gitlab, then three we set up in the +### variables section of gitlab ourselves, and finally four variables +### written into the .gitlab-ci.yml file. +### +### CI_PROJECT_DIR +### CI_JOB_NAME +### +### AWS_ACCESS_KEY_ID +### AWS_SECRET_ACCESS_KEY +### SPACK_SIGNING_KEY +### +### CDASH_BASE_URL +### ROOT_SPEC +### DEPENDENCIES +### MIRROR_URL +### + +shopt -s expand_aliases + +export FORCE_UNSAFE_CONFIGURE=1 + +TEMP_DIR="${CI_PROJECT_DIR}/jobs_scratch_dir" + +JOB_LOG_DIR="${TEMP_DIR}/logs" +SPEC_DIR="${TEMP_DIR}/specs" +LOCAL_MIRROR="${CI_PROJECT_DIR}/local_mirror" +BUILD_CACHE_DIR="${LOCAL_MIRROR}/build_cache" +SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin" +CDASH_UPLOAD_URL="${CDASH_BASE_URL}/submit.php?project=Spack" +DEP_JOB_RELATEBUILDS_URL="${CDASH_BASE_URL}/api/v1/relateBuilds.php" +declare -a JOB_DEPS_PKG_NAMES + +export SPACK_ROOT=${CI_PROJECT_DIR} +export PATH="${SPACK_BIN_DIR}:${PATH}" +export GNUPGHOME="${CI_PROJECT_DIR}/opt/spack/gpg" + +mkdir -p ${JOB_LOG_DIR} +mkdir -p ${SPEC_DIR} + +cleanup() { + set +x + + if [ -z "$exit_code" ] ; then + + exit_code=$1 + if [ -z "$exit_code" ] ; then + exit_code=0 + fi + + restore_io + + if [ "$( type -t finalize )" '=' 'function' ] ; then + finalize "$JOB_LOG_DIR/cdash_log.txt" + fi + + # We can clean these out later on, once we have a good sense for + # how the logging infrastructure is working + # rm -rf "$JOB_LOG_DIR" + fi + + \exit $exit_code +} + +alias exit='cleanup' + +begin_logging() { + trap "cleanup 1; \\exit \$exit_code" INT TERM QUIT + trap "cleanup 0; \\exit \$exit_code" EXIT + + rm -rf "$JOB_LOG_DIR/cdash_log.txt" + + # NOTE: Here, some redirects are set up + exec 3>&1 # fd 3 is now a dup of stdout + exec 4>&2 # fd 4 is now a dup of stderr + + # stdout and stderr are joined and redirected to the log + exec &> "$JOB_LOG_DIR/cdash_log.txt" + + set -x +} + +restore_io() { + exec >&- + exec 2>&- + + exec >&3 + exec 2>&4 + + exec 3>&- + exec 4>&- +} + +finalize() { + # If you define a finalize function: + # - it will always be called at the very end of the script + # - the log file will be passed in as the first argument, and + # - the code in this function will not be logged. + echo "The full log file is located at $1" + # TODO: send this log data to cdash! +} + +check_error() +{ + local last_exit_code=$1 + local last_cmd=$2 + if [[ ${last_exit_code} -ne 0 ]]; then + echo "${last_cmd} exited with code ${last_exit_code}" + echo "TERMINATING JOB" + exit 1 + else + echo "${last_cmd} completed successfully" + fi +} + +extract_build_id() +{ + LINES_TO_SEARCH=$1 + regex="buildSummary\.php\?buildid=([[:digit:]]+)" + SINGLE_LINE_OUTPUT=$(echo ${LINES_TO_SEARCH} | tr -d '\n') + + if [[ ${SINGLE_LINE_OUTPUT} =~ ${regex} ]]; then + echo "${BASH_REMATCH[1]}" + else + echo "NONE" + fi +} + +get_relate_builds_post_data() +{ + cat <<EOF +{ + "project": "${1}", + "buildid": ${2}, + "relatedid": ${3}, + "relationship": "depends on" +} +EOF +} + +gen_full_specs_for_job_and_deps() { + + read -ra PARTSARRAY <<< "${CI_JOB_NAME}" + local pkgName="${PARTSARRAY[0]}" + local pkgVersion="${PARTSARRAY[1]}" + local compiler="${PARTSARRAY[2]}" + local osarch="${PARTSARRAY[3]}" + + JOB_SPEC_NAME="${pkgName}@${pkgVersion}%${compiler} arch=${osarch}" + JOB_PKG_NAME="${pkgName}" + SPEC_YAML_PATH="${SPEC_DIR}/${pkgName}.yaml" + local root_spec_name="${ROOT_SPEC}" + local spec_names_to_save="${pkgName}" + + IFS=';' read -ra DEPS <<< "${DEPENDENCIES}" + for i in "${DEPS[@]}"; do + read -ra PARTSARRAY <<< "${i}" + pkgName="${PARTSARRAY[0]}" + spec_names_to_save="${spec_names_to_save} ${pkgName}" + JOB_DEPS_PKG_NAMES+=("${pkgName}") + done + + spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec "${root_spec_name}" --yaml-dir "${SPEC_DIR}" +} + +begin_logging + +gen_full_specs_for_job_and_deps + +echo "Building package ${JOB_SPEC_NAME}, ${HASH}, ${MIRROR_URL}" + +# Finally, list the compilers spack knows about +echo "Compiler Configurations:" +spack config get compilers + +# Make the build_cache directory if it doesn't exist +mkdir -p "${BUILD_CACHE_DIR}" + +# Get buildcache name so we can write a CDash build id file in the right place. +# If we're unable to get the buildcache name, we may have encountered a problem +# concretizing the spec, or some other issue that will eventually cause the job +# to fail. +JOB_BUILD_CACHE_ENTRY_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${SPEC_YAML_PATH}"` +if [[ $? -ne 0 ]]; then + echo "ERROR, unable to get buildcache entry name for job ${CI_JOB_NAME} (spec: ${JOB_SPEC_NAME})" + exit 1 +fi + +# This should create the directory we referred to as GNUPGHOME earlier +spack gpg list + +# Importing the secret key using gpg2 directly should allow to +# sign and verify both +set +x +KEY_IMPORT_RESULT=`echo ${SPACK_SIGNING_KEY} | base64 --decode | gpg2 --import` +check_error $? "gpg2 --import" +set -x + +spack gpg list --trusted +spack gpg list --signing + +# Whether we have to build the spec or download it pre-built, we expect to find +# the cdash build id file sitting in this location afterwards. +JOB_CDASH_ID_FILE="${BUILD_CACHE_DIR}/${JOB_BUILD_CACHE_ENTRY_NAME}.cdashid" + +# Finally, we can check the spec we have been tasked with build against +# the built binary on the remote mirror to see if it needs to be rebuilt +spack -d buildcache check --spec-yaml "${SPEC_YAML_PATH}" --mirror-url "${MIRROR_URL}" --rebuild-on-error + +if [[ $? -ne 0 ]]; then + # Configure mirror + spack mirror add local_artifact_mirror "file://${LOCAL_MIRROR}" + + JOB_CDASH_ID="NONE" + + # Install package, using the buildcache from the local mirror to + # satisfy dependencies. + BUILD_ID_LINE=`spack -d -k -v install --use-cache --cdash-upload-url "${CDASH_UPLOAD_URL}" --cdash-build "${JOB_SPEC_NAME}" --cdash-site "Spack AWS Gitlab Instance" --cdash-track "Experimental" -f "${SPEC_YAML_PATH}" | grep "buildSummary\\.php"` + check_error $? "spack install" + + # By parsing the output of the "spack install" command, we can get the + # buildid generated for us by CDash + JOB_CDASH_ID=$(extract_build_id "${BUILD_ID_LINE}") + + # Create buildcache entry for this package, reading the spec from the yaml + # file. + spack -d buildcache create --spec-yaml "${SPEC_YAML_PATH}" -a -f -d "${LOCAL_MIRROR}" --no-rebuild-index + check_error $? "spack buildcache create" + + # Write the .cdashid file to the buildcache as well + echo "${JOB_CDASH_ID}" >> ${JOB_CDASH_ID_FILE} + + # TODO: The upload-s3 command should eventually be replaced with something + # like: "spack buildcache put <mirror> <spec>", when that subcommand is + # properly implemented. + spack -d upload-s3 spec --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" + check_error $? "spack upload-s3 spec" +else + echo "spec ${JOB_SPEC_NAME} is already up to date on remote mirror, downloading it" + + # Configure remote mirror so we can download buildcache entry + spack mirror add remote_binary_mirror ${MIRROR_URL} + + # Now download it + spack -d buildcache download --spec-yaml "${SPEC_YAML_PATH}" --path "${BUILD_CACHE_DIR}/" --require-cdashid + check_error $? "spack buildcache download" +fi + +# The next step is to relate this job to the jobs it depends on +if [ -f "${JOB_CDASH_ID_FILE}" ]; then + JOB_CDASH_BUILD_ID=$(<${JOB_CDASH_ID_FILE}) + + if [ "${JOB_CDASH_BUILD_ID}" == "NONE" ]; then + echo "ERROR: unable to read this jobs id from ${JOB_CDASH_ID_FILE}" + exit 1 + fi + + # Now get CDash ids for dependencies and "relate" each dependency build + # with this jobs build + for DEP_PKG_NAME in "${JOB_DEPS_PKG_NAMES[@]}"; do + echo "Getting cdash id for dependency --> ${DEP_PKG_NAME} <--" + DEP_SPEC_YAML_PATH="${SPEC_DIR}/${DEP_PKG_NAME}.yaml" + DEP_JOB_BUILDCACHE_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${DEP_SPEC_YAML_PATH}"` + + if [[ $? -eq 0 ]]; then + DEP_JOB_ID_FILE="${BUILD_CACHE_DIR}/${DEP_JOB_BUILDCACHE_NAME}.cdashid" + echo "DEP_JOB_ID_FILE path = ${DEP_JOB_ID_FILE}" + + if [ -f "${DEP_JOB_ID_FILE}" ]; then + DEP_JOB_CDASH_BUILD_ID=$(<${DEP_JOB_ID_FILE}) + echo "File ${DEP_JOB_ID_FILE} contained value ${DEP_JOB_CDASH_BUILD_ID}" + echo "Relating builds -> ${JOB_SPEC_NAME} (buildid=${JOB_CDASH_BUILD_ID}) depends on ${DEP_PKG_NAME} (buildid=${DEP_JOB_CDASH_BUILD_ID})" + relateBuildsPostBody="$(get_relate_builds_post_data "Spack" ${JOB_CDASH_BUILD_ID} ${DEP_JOB_CDASH_BUILD_ID})" + relateBuildsResult=`curl "${DEP_JOB_RELATEBUILDS_URL}" -H "Content-Type: application/json" -H "Accept: application/json" -d "${relateBuildsPostBody}"` + echo "Result of curl request: ${relateBuildsResult}" + else + echo "ERROR: Did not find expected .cdashid file for dependency: ${DEP_JOB_ID_FILE}" + exit 1 + fi + else + echo "ERROR: Unable to get buildcache entry name for ${DEP_SPEC_NAME}" + exit 1 + fi + done +else + echo "ERROR: Did not find expected .cdashid file ${JOB_CDASH_ID_FILE}" + exit 1 +fi + +# Show the size of the buildcache and a list of what's in it, directly +# in the gitlab log output +( + restore_io + du -sh ${BUILD_CACHE_DIR} + find ${BUILD_CACHE_DIR} -maxdepth 3 -type d -ls +) + +echo "End of rebuild package script" diff --git a/lib/spack/spack/cmd/upload_s3.py b/lib/spack/spack/cmd/upload_s3.py new file mode 100644 index 0000000000..ba3f0688ae --- /dev/null +++ b/lib/spack/spack/cmd/upload_s3.py @@ -0,0 +1,212 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# TODO: This will be merged into the buildcache command once +# everything is working. + +import os +import re +import sys + +try: + import boto3 + import botocore + have_boto3_support = True +except ImportError: + have_boto3_support = False + +import llnl.util.tty as tty + +from spack.error import SpackError +import spack.tengine as template_engine +from spack.spec import Spec + + +import spack.binary_distribution as bindist + + +description = "temporary command to upload buildcaches to 's3.spack.io'" +section = "packaging" +level = "long" + + +def setup_parser(subparser): + setup_parser.parser = subparser + subparsers = subparser.add_subparsers(help='upload-s3 sub-commands') + + # sub-command to upload a built spec to s3 + spec = subparsers.add_parser('spec', help=upload_spec.__doc__) + + spec.add_argument('-s', '--spec', default=None, + help='Spec to upload') + + spec.add_argument('-y', '--spec-yaml', default=None, + help='Path to spec yaml file containing spec to upload') + + spec.add_argument('-b', '--base-dir', default=None, + help='Path to root of buildcaches') + + spec.add_argument('-e', '--endpoint-url', + default='https://s3.spack.io', help='URL of mirror') + + spec.set_defaults(func=upload_spec) + + # sub-command to update the index of a buildcache on s3 + index = subparsers.add_parser('index', help=update_index.__doc__) + + index.add_argument('-e', '--endpoint-url', + default='https://s3.spack.io', help='URL of mirror') + + index.set_defaults(func=update_index) + + +def get_s3_session(endpoint_url): + if not have_boto3_support: + raise SpackError('boto3 module not available') + + session = boto3.Session() + s3 = session.resource('s3') + + bucket_names = [] + for bucket in s3.buckets.all(): + bucket_names.append(bucket.name) + + if len(bucket_names) > 1: + raise SpackError('More than one bucket associated with credentials') + + bucket_name = bucket_names[0] + + return s3, bucket_name + + +def update_index(args): + """Update the index of an s3 buildcache""" + s3, bucket_name = get_s3_session(args.endpoint_url) + + bucket = s3.Bucket(bucket_name) + exists = True + + try: + s3.meta.client.head_bucket(Bucket=bucket_name) + except botocore.exceptions.ClientError as e: + # If a client error is thrown, then check that it was a 404 error. + # If it was a 404 error, then the bucket does not exist. + error_code = e.response['Error']['Code'] + if error_code == '404': + exists = False + + if not exists: + tty.error('S3 bucket "{0}" does not exist'.format(bucket_name)) + sys.exit(1) + + build_cache_dir = os.path.join( + 'mirror', bindist.build_cache_relative_path()) + + spec_yaml_regex = re.compile('{0}/(.+\\.spec\\.yaml)$'.format( + build_cache_dir)) + spack_regex = re.compile('{0}/([^/]+)/.+\\.spack$'.format( + build_cache_dir)) + + top_level_keys = set() + + for key in bucket.objects.all(): + m = spec_yaml_regex.search(key.key) + if m: + top_level_keys.add(m.group(1)) + print(m.group(1)) + continue + + m = spack_regex.search(key.key) + if m: + top_level_keys.add(m.group(1)) + print(m.group(1)) + continue + + index_data = { + 'top_level_keys': top_level_keys, + } + + env = template_engine.make_environment() + template_dir = 'misc' + index_template = os.path.join(template_dir, 'buildcache_index.html') + t = env.get_template(index_template) + contents = t.render(index_data) + + index_key = os.path.join(build_cache_dir, 'index.html') + + tty.debug('Generated index:') + tty.debug(contents) + tty.debug('Pushing it to {0} -> {1}'.format(bucket_name, index_key)) + + s3_obj = s3.Object(bucket_name, index_key) + s3_obj.put(Body=contents, ACL='public-read') + + +def upload_spec(args): + """Upload a spec to s3 bucket""" + if not args.spec and not args.spec_yaml: + tty.error('Cannot upload spec without spec arg or path to spec yaml') + sys.exit(1) + + if not args.base_dir: + tty.error('No base directory for buildcache specified') + sys.exit(1) + + if args.spec: + try: + spec = Spec(args.spec) + spec.concretize() + except Exception: + tty.error('Unable to concrectize spec from string {0}'.format( + args.spec)) + sys.exit(1) + else: + try: + with open(args.spec_yaml, 'r') as fd: + spec = Spec.from_yaml(fd.read()) + except Exception: + tty.error('Unable to concrectize spec from yaml {0}'.format( + args.spec_yaml)) + sys.exit(1) + + s3, bucket_name = get_s3_session(args.endpoint_url) + + build_cache_dir = bindist.build_cache_relative_path() + + tarball_key = os.path.join( + build_cache_dir, bindist.tarball_path_name(spec, '.spack')) + tarball_path = os.path.join(args.base_dir, tarball_key) + + specfile_key = os.path.join( + build_cache_dir, bindist.tarball_name(spec, '.spec.yaml')) + specfile_path = os.path.join(args.base_dir, specfile_key) + + cdashidfile_key = os.path.join( + build_cache_dir, bindist.tarball_name(spec, '.cdashid')) + cdashidfile_path = os.path.join(args.base_dir, cdashidfile_key) + + tty.msg('Uploading {0}'.format(tarball_key)) + s3.meta.client.upload_file( + tarball_path, bucket_name, + os.path.join('mirror', tarball_key), + ExtraArgs={'ACL': 'public-read'}) + + tty.msg('Uploading {0}'.format(specfile_key)) + s3.meta.client.upload_file( + specfile_path, bucket_name, + os.path.join('mirror', specfile_key), + ExtraArgs={'ACL': 'public-read'}) + + if os.path.exists(cdashidfile_path): + tty.msg('Uploading {0}'.format(cdashidfile_key)) + s3.meta.client.upload_file( + cdashidfile_path, bucket_name, + os.path.join('mirror', cdashidfile_key), + ExtraArgs={'ACL': 'public-read'}) + + +def upload_s3(parser, args): + if args.func: + args.func(args) |