diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/update | 500 |
1 files changed, 500 insertions, 0 deletions
diff --git a/scripts/update b/scripts/update new file mode 100755 index 000000000..f1f02c238 --- /dev/null +++ b/scripts/update @@ -0,0 +1,500 @@ +#!/bin/sh -e + +HERE="$(dirname $(readlink -f "${0}"))"; +TEMP="$(mktemp -d)"; +cd "${HERE}"/..; # repository root (do not touch!) + +## +# README +# +# This script will determine which package(s) need to be +# built or rebuilt by evaluating the state of mirrormaster +# (or "next.adelielinux.org" in the case of a staging box) +# and comparing it to the state of an official builder. +# +# It is to be run only on an official builder by someone +# with access to and training to operate such a machine, +# which may include automation and robots. +# +# This script assumes it runs inside an autobuilder 'debug' +# environment. This is not an optional requirement. This is +# an internal tool not intended for public-facing usage. +# +# This script assumes "next" ('NEXT') and "ours" ('OURS') are +# synchronized. That is, binaries at the two locations are +# either the same, or point to the same location. This is to +# warn the operator that mirrormaster might be out of sync +# with builders, and is not useful for developers. + + +## +# TODO +# +# When checking 'DESCRIPTION' from APKINDEX, this is NOT +# a reliable way to see what is *actually* the last package +# built, and from what commit hash. This will cause issues! +# +# The computed list is too broad, specifically the 'comm -23' +# line on 'list.*.sort' files. +# +# Register trap handlers to clean up and restore backups +# appropriately; biggest risk is inconsistent state. Second +# thing to avoid is wasted CPU time or having to restart. +# +# Implement build logs as in normal autobuilder. +# +# Performance enhancements. +# +# Mathematically rigorous series of transformations to +# compute various bits here, not just a series of loops, +# lists, etc. +# + + +## +# Configuration (use defaults for production!). +# +NEXT=https://next.adelielinux.org; +OURS=/packages; +REPO="system user"; # public-facing repositories +ARCH="${AB_ARCH}"; # from autobuilder environment +BNCH=current; # probably do not change this + + +## +# Helpers. +# +message () +{ + printf " * %s...\n" "${1}"; +} + +fetch_local () +{ + cat "${1}"; +} +fetch_curly () +{ + curl -sL "${1}"; +} + +find_strategy () +{ + case "${1}" in + /*) down=fetch_local; ;; # local filesystem + *://*) down=fetch_curly; ;; # remote resource + *) printf "E: unsupported site type!\n" && exit 1; ;; + esac + + printf "%s\n" "${down}"; +} + +## +# Counts number of commits between two commits (A,B] +# +# LAST: latest commit available (e.g. origin/current) +# FROM: commit to compare +# +count_behind () +{ + f_last="${1}"; shift; + f_from="${1}"; shift; + + git log --format="%H" ${f_from}..${f_last} | wc -l; +} + +## +# Manually update (if greater than global) to work around +# a shell limitation in updating globals from subshells. +# +# NOTE: USES GLOBAL. +# +count_update () +{ + f_this="${1}"; shift; + + # store the newest maximum + if test "${OLD_this}" -lt "${f_this}"; then + OLD_this=${f_this}; + fi + + if test "${OLD_keep}" -lt "${f_this}"; then + OLD_keep=${f_this}; + fi +} + +## +# String comparison, but with more screaming. +# +check_match () +{ + f_next="${1}"; shift; + f_ours="${1}"; shift; + f_repo="${1}"; shift; + + # OK if ours is newer than next + hind=$(count_behind "${f_next}" "${f_ours}"); + test "${hind}" -eq 0 && return; + + printf "E: "next" ('%s') != "ours" ('%s') in repo '%s'!\n" "${f_next}" "${f_ours}" "${f_repo}"; + exit 1; +} + +## +# Produce a sorted list of files in a given ON-DISK +# binary repository. TODO: update requirements so that +# the "ours" repository is always required to be on-disk. +# +list_packages () +{ + f_repo="${1}"; shift; + + find "${OURS}"/${f_repo}/${ARCH} | sort; +} + + +## +# Sanity checks. +# +message "Verifying configuration and environment"; +test -z ${ARCH} && printf "E: 'ARCH' could not be determined!\n" && exit 1; +command -v curl >/dev/null 2>&1 || (printf "E: required utility 'curl' not found!\n" && exit 1); + + +## +# Internal. +# +message "Identifying transfer strategies"; +_NEXT=$(find_strategy "${NEXT}"); +_OURS=$(find_strategy "${OURS}"); + + +## +# Update source code, if needed. +# +message "Updating local source repository" +git checkout "${BNCH}"; +git pull --ff-only; + + +## +# Do it. +# +# Note: underscore-prefixed variables refer to friendly names. +# Sometimes these variables need to come before, or after, due +# to whether we use the friendly name to calculate commit, or +# the other way around. This is intentional and ugly. Sorry. +# +print_status () +{ + f_code="${1}"; shift; + f_kind="${1}"; shift; + f_repo="${1}"; shift; + f_hash="${1}"; shift; + f_many="${1}"; shift; + f_name="${1}"; shift; + + printf " - [%s] %s (%s)\tis at %s [-%4d] (%s)...\n" \ + "${f_code}" \ + "${f_kind}" \ + "${f_repo}" \ + "${f_hash}" \ + "${f_many}" \ + "${f_name}" \ + ; +} +message "Examining repositories"; +meta=$(git log --format="%H" . | head -n 1); +_meta=$(git describe ${meta}); +nmeta=$(count_behind "${meta}" "${meta}"); # redundant i know, but consistent :D +print_status REF .git meta "${meta}" "${nmeta}" "${_meta}"; +# source of truth +OLD_list=""; # starts empty, CURRENTLY UNUSED but probably useful! +OLD_keep=0; # global maximum +for repo in ${REPO}; do + OLD_this=0; # reset zero at each iteration + + # disk + disk=$(git log --format="%H" ${repo} | head -n 1); # last commit in repository on disk + _disk=$(git describe "${disk}"); + ndisk=$(count_behind "${meta}" "${disk}"); + count_update "${ndisk}"; + print_status SRC .git "${repo}" "${disk}" "${ndisk}" "${_disk}"; + + # next + ${_NEXT} ${NEXT}/${repo}/${ARCH}/APKINDEX.tar.gz | tar -C "${TEMP}" -xzf - DESCRIPTION; + _next=$(cut -d\ -f2 < "${TEMP}"/DESCRIPTION); + next=$(git log --format="%H" "${_next}" | head -n 1); + nnext=$(count_behind "${meta}" "${next}"); + count_update "${nnext}"; + + # ours + ${_OURS} ${OURS}/${repo}/${ARCH}/APKINDEX.tar.gz | tar -C "${TEMP}" -xzf - DESCRIPTION; + _ours=$(cut -d\ -f2 < "${TEMP}"/DESCRIPTION); + ours=$(git log --format="%H" "${_ours}" | head -n 1); + nours=$(count_behind "${meta}" "${ours}"); + count_update "${nours}"; + + # do they match (fatal if not)? + check_match "${_next}" "${_ours}" "${repo}"; # could also check commit or count but this is direct + + # cool, print current status + print_status BIN next "${repo}" "${next}" "${nnext}" "${_next}"; + print_status BIN ours "${repo}" "${ours}" "${nours}" "${_ours}"; + + # save per-repository freshness (DO NOT CHANGE STORAGE ORDER!) + OLD_list="${OLD_list} ${repo}:${OLD_this}"; +done + + +message "Generating local repository index" +./scripts/setup; + + +message "Calculating topological package build order" +temp=$(mktemp); +./scripts/deplist ${REPO} | ./scripts/depsort | cat -n > ${temp}; + + +message "Calculating initial build requirements" +LIST=$(git log --pretty=format:"%h" HEAD~${OLD_keep}..HEAD | while read commit; do + git diff-tree --no-commit-id --name-only -r ${commit}; +done | grep APKBUILD | cut -d/ -f-2 | sort | uniq | while read k; do + grep -E "${k}$" ${temp}; +done | sort -n | awk '{print $2}'); +rm ${temp}; + + +message "Going back in time by ${OLD_keep} commits"; +git checkout "${BNCH}"~"${OLD_keep}"; +for k in old new bin cmp; do + rm -fr "${TEMP}"/${k}; + mkdir "${TEMP}"/${k}; +done + +message "Tracking starting state" +for item in ${LIST}; do +( + test -f "${item}"/APKBUILD || continue; # could be new package + cd "${item}"; + . ./APKBUILD; + printf > "${TEMP}"/old/${item%/*}.${item#*/} "pkgname=%s\npkgver=%s\npkgrel=%s\n" "${pkgname}" "${pkgver}" "${pkgrel}"; +) +done + +message "Going forward in time to present"; +git checkout "${BNCH}"; +for item in ${LIST}; do +( + test -f "${item}"/APKBUILD || continue; # could be deleted package + cd "${item}"; + . ./APKBUILD; + printf > "${TEMP}"/new/${item%/*}.${item#*/} "pkgname=%s\npkgver=%s\npkgrel=%s\n" "${pkgname}" "${pkgver}" "${pkgrel}"; +) +done + +message "Verifying ALL pkgver/pkgrel are different"; +FAIL=0; +for item in ${LIST}; do +( + if ! test -f "${TEMP}"/old/${item%/*}.${item#*/} || ! test -f "${TEMP}"/new/${item%/*}.${item#*/}; then + continue; # not in both + fi + + if ! cmp --silent "${TEMP}"/old/${item%/*}.${item#*/} "${TEMP}"/old/${item%/*}.${item#*/}; then + printf "E: package '%s' needs a relbump!\n" "${item}"; + FAIL=1; + continue; + fi + +if false; then + . "${TEMP}"/old/${item%/*}.${item#*/}; + old_pkgver=$pkgver; + old_pkgrel=$pkgrel; + + . "${TEMP}"/new/${item%/*}.${item#*/}; + new_pkgver=$pkgver; + new_pkgrel=$pkgrel; + + # TODO: actually compare versions?? or not necessary?? +fi +) +done +# THIS PROPERTY IS VERY IMPORTANT!! DO NOT PROCEED IF ANY PACKAGES WOULD BE CLOBBERED! +test "${FAIL}" -eq 1 && printf "E: not all package versions are good (would clobber)!\n" && exit 1; + + +message "Generating backup list of local package binaries" +for repo in ${REPO}; do + list_packages ${repo} > "${TEMP}"/manifest.${repo}.a; +done + + +message "Backing up APKINDEX files"; +for repo in ${REPO}; do + cp -p "${OURS}"/${repo}/${ARCH}/APKINDEX.tar.gz "${TEMP}"/APKINDEX.tar.gz.${repo}.bak; +done + + +message "Building packages for TESTING (STILL DANGEROUS!)" +for item in ${LIST}; do +( + cd "${item}"; + + # save state of binary package repository before build + list_packages ${item%/*} > "${TEMP}"/build.a; + + # do the build and hope nothing gets clobbered! + # TODO: can this be replaced by 'BOOTSTRAP=1 abuild -r'? do we need full control? + # the current command does not clean up any files (like '-K') + abuild sanitycheck clean deps unpack prepare mkusers build rootpkg index undeps; + + # save state of binary package repository after build + list_packages ${item%/*} > "${TEMP}"/build.b; + + # compute delta and store to a file for later use + comm -13 "${TEMP}"/build.a "${TEMP}"/build.b > "${TEMP}"/bin/${item%/*}.${item#*/}; +) +done + + +message "Moving newly-built packages to a less scary directory" +for item in ${LIST}; do +( + cd "${item}"; + + # get those toxic critters outta' here! + while read k; do + mv "${k}" "${TEMP}"/cmp/; + done < "${TEMP}"/bin/${item%/*}.${item#*/}; +) +done + + +message "Clearing any stale cached files" +for k in a b; do + rm -f "${TEMP}"/list.${k}; +done + + +## +# NOTE: we CANNOT assume that there aren't multiple older versions +# of a given package in "${OURS}", while we only care about the +# "current" (latest) version that existed prior to the above build. +# +message "Scanning packages for shared library changes" +for item in ${LIST}; do +( + cd "${item}"; + + # get sorted list of .so files in all old .apk files for this package + test ! -f "${TEMP}"/old/${item%/*}.${item#*/} && continue; # skip because it's new! + ( + . "${TEMP}"/old/${item%/*}.${item#*/}; + + find "${OURS}"/${item%/*}/${ARCH} -type f -name "${pkgname}-*${pkgver}-r${pkgrel}.apk" | while read k; do + tar -tzf "${k}"; + done | grep \\.so | sort >> "${TEMP}"/list.a; + ) + + # get sorted list of .so files in all new .apk files for this package + # FIXME: what happens if a package is deleted? still shows up in 'LIST' or not? + ( + . "${TEMP}"/new/${item%/*}.${item#*/}; + + find "${TEMP}"/cmp -type f -name "${pkgname}-*${pkgver}-r${pkgrel}.apk" | while read k; do + tar -tzf "${k}"; + done | grep \\.so | sort >> "${TEMP}"/list.b; + ) +) +done + + +## +# FIXME: this needs to be done anyway, but this step is ALSO +# a shortcut/kludge to avoid manually updating the index files +# after moving the newly-built binaries out of "${OURS}". +# After this step, there should be no visible before/after. +# +# NOTE: this is a full swap so that we also have a copy of +# the "new" APKINDEX for later analysis. +# +message "Restoring original APKINDEX files"; +for repo in ${REPO}; do + cp -p "${OURS}"/${repo}/${ARCH}/APKINDEX.tar.gz "${TEMP}"/APKINDEX.tar.gz.${repo}.tmp; + mv "${TEMP}"/APKINDEX.tar.gz.${repo}.bak "${OURS}"/${repo}/${ARCH}/APKINDEX.tar.gz; + mv "${TEMP}"/APKINDEX.tar.gz.${repo}.tmp "${TEMP}"/APKINDEX.tar.gz.${repo}; +done + + +message "Sorting list of changed shared libraries" +for k in a b; do + sort "${TEMP}"/list.${k} | uniq > "${TEMP}"/list.${k}.sort; +done + + +## +# NOTE: the '-q' to 'apk' gives us package names without +# version information. It may be possible for multiple +# versions to exist, and only one be affected, and that only +# the latest one could be affected, and that we already store +# the "old" version information, ... but it doesn't matter +# anyway since that information is derived from a binary repo +# and we only care about the source repo from here forward. +# +# The '-q' also removes the "blah is required by..." line. +# +# TODO: don't actually call 'apk' here (slow); parse the +# APKINDEX data and go from there. +# +message "Tracing dependencies for each changed shared library" +comm -23 "${TEMP}"/list.a.sort "${TEMP}"/list.b.sort | while read file; do + apk search -rq so:${file##*/}; +done > "${TEMP}"/deps + + +message "Finding parents of each affected package" +sort "${TEMP}"/deps | while read name; do + grep ${name} scripts/.index; # generated by 'setup' +done | awk '{print $1}' | sort | uniq > "${TEMP}"/parents; + + +## +# TODO: since we cannot know which repository(ies) provide the +# possibly-affected package(s), we must search all repositories +# for all occurrences. +# +message "Finding all APKBUILD files for each parent" +NEED=""; +while read name; do + for repo in ${REPO}; do + if test -f ${repo}/${name}/APKBUILD; then + NEED="${NEED} ${repo}/${name}"; + fi + done +done < "${TEMP}"/parents; + + + +message "Removing any duplicate entries" +printf "%s\n" "${LIST}" | sort > "${TEMP}"/LIST; +printf "%s\n" "${NEED}" | tr ' ' '\n' | sort > "${TEMP}"/NEED; +comm -13 "${TEMP}"/LIST "${TEMP}"/NEED; + +echo $TEMP; + + +message "Generating change list of local package binaries" +for repo in ${REPO}; do + list_packages ${repo} > "${TEMP}"/manifest.${repo}.b; +done + + +message "Comparing backup and change lists (sanity check)" +for repo in ${REPO}; do + diff -ur "${TEMP}"/manifest.${repo}.a "${TEMP}"/manifest.${repo}.b; +done + + +message "Cleaning up" +#rm -fr "${TEMP}"; |