path: root/scripts/update

#!/bin/sh -e

HERE="$(dirname $(readlink -f "${0}"))";
TEMP="$(mktemp -d)";
cd "${HERE}"/..; # repository root (do not touch!)

##
# README
#
# This script will determine which package(s) need to be
# built or rebuilt by evaluating the state of mirrormaster
# (or "next.adelielinux.org" in the case of a staging box)
# and comparing it to the state of an official builder.
#
# It is to be run only on an official builder by someone
# with access to and training to operate such a machine,
# which may include automation and robots.
#
# This script assumes it runs inside an autobuilder 'debug'
# environment. This is not an optional requirement. This is
# an internal tool not intended for public-facing usage.
#
# This script assumes "next" ('NEXT') and "ours" ('OURS') are
# synchronized. That is, binaries at the two locations are
# either the same, or point to the same location. This is to
# warn the operator that mirrormaster might be out of sync
# with builders, and is not useful for developers.


##
# TODO
#
# When checking 'DESCRIPTION' from APKINDEX, this is NOT
# a reliable way to see what is *actually* the last package
# built, and from what commit hash. This will cause issues!
#
# The computed list is too broad, specifically the 'comm -23'
# line on 'list.*.sort' files.
#
# Register trap handlers to clean up and restore backups
# appropriately; biggest risk is inconsistent state. Second
# thing to avoid is wasted CPU time or having to restart.
#
# Implement build logs as in normal autobuilder.
#
# Performance enhancements.
#
# Mathematically rigorous series of transformations to
# compute various bits here, not just a series of loops,
# lists, etc.
#


##
# Configuration (use defaults for production!).
#
NEXT=https://next.adelielinux.org;
OURS=/packages;
REPO="system user"; # public-facing repositories
ARCH="${AB_ARCH}"; # from autobuilder environment
BNCH=current; # probably do not change this


##
# Helpers.
#
message ()
{
    printf "  * %s...\n" "${1}";
}

fetch_local ()
{
    cat "${1}";
}
fetch_curly ()
{
    curl -sL "${1}";
}

find_strategy ()
{
    case "${1}" in
        /*)    down=fetch_local; ;; # local filesystem
        *://*) down=fetch_curly; ;; # remote resource
        *)     printf "E: unsupported site type!\n" && exit 1; ;;
    esac

    printf "%s\n" "${down}";
}

##
# Counts number of commits between two commits (A,B]
#
# LAST: latest commit available (e.g. origin/current)
# FROM: commit to compare
#
count_behind ()
{
    f_last="${1}"; shift;
    f_from="${1}"; shift;

    git log --format="%H" ${f_from}..${f_last} | wc -l;
}

##
# Manually update (if greater than global) to work around
# a shell limitation in updating globals from subshells.
#
# NOTE: USES GLOBAL.
#
count_update ()
{
    f_this="${1}"; shift;

    # store the newest maximum
    if test "${OLD_this}" -lt "${f_this}"; then
        OLD_this=${f_this};
    fi

    if test "${OLD_keep}" -lt "${f_this}"; then
        OLD_keep=${f_this};
    fi
}

##
# String comparison, but with more screaming.
#
check_match ()
{
    f_next="${1}"; shift;
    f_ours="${1}"; shift;
    f_repo="${1}"; shift;

    # OK if ours is newer than next
    hind=$(count_behind "${f_next}" "${f_ours}");
    test "${hind}" -eq 0 && return;

    printf "E: "next" ('%s') != "ours" ('%s') in repo '%s'!\n" "${f_next}" "${f_ours}" "${f_repo}";
    exit 1;
}

##
# Produce a sorted list of files in a given ON-DISK
# binary repository. TODO: update requirements so that
# the "ours" repository is always required to be on-disk.
#
list_packages ()
{
    f_repo="${1}"; shift;

    find "${OURS}"/${f_repo}/${ARCH} | sort;
}


##
# Sanity checks.
#
message "Verifying configuration and environment";
test -z ${ARCH} && printf "E: 'ARCH' could not be determined!\n" && exit 1;
command -v curl >/dev/null 2>&1 || (printf "E: required utility 'curl' not found!\n" && exit 1);


##
# Internal.
#
message "Identifying transfer strategies";
_NEXT=$(find_strategy "${NEXT}");
_OURS=$(find_strategy "${OURS}");


##
# Update source code, if needed.
#
message "Updating local source repository"
git checkout "${BNCH}";
git pull --ff-only;


##
# Do it.
#
# Note: underscore-prefixed variables refer to friendly names.
# Sometimes these variables need to come before, or after, due
# to whether we use the friendly name to calculate commit, or
# the other way around. This is intentional and ugly. Sorry.
#
print_status ()
{
    f_code="${1}"; shift;
    f_kind="${1}"; shift;
    f_repo="${1}"; shift;
    f_hash="${1}"; shift;
    f_many="${1}"; shift;
    f_name="${1}"; shift;

    printf "      - [%s] %s (%s)\tis at %s [-%4d] (%s)...\n" \
        "${f_code}" \
        "${f_kind}" \
        "${f_repo}" \
        "${f_hash}" \
        "${f_many}" \
        "${f_name}" \
        ;
}
message "Examining repositories";
meta=$(git log --format="%H" . | head -n 1);
_meta=$(git describe ${meta});
nmeta=$(count_behind "${meta}" "${meta}"); # redundant i know, but consistent :D
print_status REF .git meta "${meta}" "${nmeta}" "${_meta}";
# source of truth
OLD_list=""; # starts empty, CURRENTLY UNUSED but probably useful!
OLD_keep=0; # global maximum
for repo in ${REPO}; do
    OLD_this=0; # reset zero at each iteration

    # disk
    disk=$(git log --format="%H" ${repo} | head -n 1); # last commit in repository on disk
    _disk=$(git describe "${disk}");
    ndisk=$(count_behind "${meta}" "${disk}");
    count_update "${ndisk}";
    print_status SRC .git "${repo}" "${disk}" "${ndisk}" "${_disk}";

    # next
    ${_NEXT} ${NEXT}/${repo}/${ARCH}/APKINDEX.tar.gz | tar -C "${TEMP}" -xzf - DESCRIPTION;
    _next=$(cut -d\  -f2 < "${TEMP}"/DESCRIPTION);
    next=$(git log --format="%H" "${_next}" | head -n 1);
    nnext=$(count_behind "${meta}" "${next}");
    count_update "${nnext}";

    # ours
    ${_OURS} ${OURS}/${repo}/${ARCH}/APKINDEX.tar.gz | tar -C "${TEMP}" -xzf - DESCRIPTION;
    _ours=$(cut -d\  -f2 < "${TEMP}"/DESCRIPTION);
    ours=$(git log --format="%H" "${_ours}" | head -n 1);
    nours=$(count_behind "${meta}" "${ours}");
    count_update "${nours}";

    # do they match (fatal if not)?
    check_match "${_next}" "${_ours}" "${repo}"; # could also check commit or count but this is direct

    # cool, print current status
    print_status BIN next "${repo}" "${next}" "${nnext}" "${_next}";
    print_status BIN ours "${repo}" "${ours}" "${nours}" "${_ours}";

    # save per-repository freshness (DO NOT CHANGE STORAGE ORDER!)
    OLD_list="${OLD_list} ${repo}:${OLD_this}";
done


message "Generating local repository index"
./scripts/setup;


message "Calculating topological package build order"
temp=$(mktemp);
./scripts/deplist ${REPO} | ./scripts/depsort | cat -n > ${temp};


message "Calculating initial build requirements"
LIST=$(git log --pretty=format:"%h" HEAD~${OLD_keep}..HEAD | while read commit; do
    git diff-tree --no-commit-id --name-only -r ${commit};
done | grep APKBUILD | cut -d/ -f-2 | sort | uniq | while read k; do
    grep -E "${k}$" ${temp};
done | sort -n | awk '{print $2}');
rm ${temp};


message "Going back in time by ${OLD_keep} commits";
git checkout "${BNCH}"~"${OLD_keep}";
for k in old new bin cmp; do
    rm -fr "${TEMP}"/${k};
    mkdir  "${TEMP}"/${k};
done

message "Tracking starting state"
for item in ${LIST}; do
(
    test -f "${item}"/APKBUILD || continue; # could be new package
    cd "${item}";
    . ./APKBUILD;
    printf > "${TEMP}"/old/${item%/*}.${item#*/} "pkgname=%s\npkgver=%s\npkgrel=%s\n" "${pkgname}" "${pkgver}" "${pkgrel}";
)
done

message "Going forward in time to present";
git checkout "${BNCH}";
for item in ${LIST}; do
(
    test -f "${item}"/APKBUILD || continue; # could be deleted package
    cd "${item}";
    . ./APKBUILD;
    printf > "${TEMP}"/new/${item%/*}.${item#*/} "pkgname=%s\npkgver=%s\npkgrel=%s\n" "${pkgname}" "${pkgver}" "${pkgrel}";
)
done

message "Verifying ALL pkgver/pkgrel are different";
FAIL=0;
for item in ${LIST}; do
(
    if ! test -f "${TEMP}"/old/${item%/*}.${item#*/} || ! test -f "${TEMP}"/new/${item%/*}.${item#*/}; then
        continue; # not in both
    fi

    if ! cmp --silent "${TEMP}"/old/${item%/*}.${item#*/} "${TEMP}"/old/${item%/*}.${item#*/}; then
        printf "E: package '%s' needs a relbump!\n" "${item}";
        FAIL=1;
        continue;
    fi

if false; then
    . "${TEMP}"/old/${item%/*}.${item#*/};
    old_pkgver=$pkgver;
    old_pkgrel=$pkgrel;

    . "${TEMP}"/new/${item%/*}.${item#*/};
    new_pkgver=$pkgver;
    new_pkgrel=$pkgrel;

    # TODO: actually compare versions?? or not necessary??
fi
)
done
# THIS PROPERTY IS VERY IMPORTANT!! DO NOT PROCEED IF ANY PACKAGES WOULD BE CLOBBERED!
test "${FAIL}" -eq 1 && printf "E: not all package versions are good (would clobber)!\n" && exit 1;


message "Generating backup list of local package binaries"
for repo in ${REPO}; do
    list_packages ${repo} > "${TEMP}"/manifest.${repo}.a;
done


message "Backing up APKINDEX files";
for repo in ${REPO}; do
    cp -p "${OURS}"/${repo}/${ARCH}/APKINDEX.tar.gz "${TEMP}"/APKINDEX.tar.gz.${repo}.bak;
done


message "Building packages for TESTING (STILL DANGEROUS!)"
for item in ${LIST}; do
(
    cd "${item}";

    # save state of binary package repository before build
    list_packages ${item%/*} > "${TEMP}"/build.a;

    # do the build and hope nothing gets clobbered!
    # TODO: can this be replaced by 'BOOTSTRAP=1 abuild -r'? do we need full control?
    # the current command does not clean up any files (like '-K')
    abuild sanitycheck clean deps unpack prepare mkusers build rootpkg index undeps;

    # save state of binary package repository  after build
    list_packages ${item%/*} > "${TEMP}"/build.b;

    # compute delta and store to a file for later use
    comm -13 "${TEMP}"/build.a "${TEMP}"/build.b > "${TEMP}"/bin/${item%/*}.${item#*/};
)
done


message "Moving newly-built packages to a less scary directory"
for item in ${LIST}; do
(
    cd "${item}";

    # get those toxic critters outta' here!
    while read k; do
        mv "${k}" "${TEMP}"/cmp/;
    done < "${TEMP}"/bin/${item%/*}.${item#*/};
)
done


message "Clearing any stale cached files"
for k in a b; do
    rm -f "${TEMP}"/list.${k};
done


##
# NOTE: we CANNOT assume that there aren't multiple older versions
# of a given package in "${OURS}", while we only care about the
# "current" (latest) version that existed prior to the above build.
#
message "Scanning packages for shared library changes"
for item in ${LIST}; do
(
    cd "${item}";

    # get sorted list of .so files in all old .apk files for this package
    test ! -f "${TEMP}"/old/${item%/*}.${item#*/} && continue; # skip because it's new!
    (
        . "${TEMP}"/old/${item%/*}.${item#*/};

        find "${OURS}"/${item%/*}/${ARCH} -type f -name "${pkgname}-*${pkgver}-r${pkgrel}.apk" | while read k; do
            tar -tzf "${k}";
        done | grep \\.so | sort >> "${TEMP}"/list.a;
    )

    # get sorted list of .so files in all new .apk files for this package
    # FIXME: what happens if a package is deleted? still shows up in 'LIST' or not?
    (
        . "${TEMP}"/new/${item%/*}.${item#*/};

        find "${TEMP}"/cmp -type f -name "${pkgname}-*${pkgver}-r${pkgrel}.apk" | while read k; do
            tar -tzf "${k}";
        done | grep \\.so | sort >> "${TEMP}"/list.b;
    )
)
done


##
# FIXME: this needs to be done anyway, but this step is ALSO
# a shortcut/kludge to avoid manually updating the index files
# after moving the newly-built binaries out of "${OURS}".
# After this step, there should be no visible before/after.
#
# NOTE: this is a full swap so that we also have a copy of
# the "new" APKINDEX for later analysis.
#
message "Restoring original APKINDEX files";
for repo in ${REPO}; do
    cp -p "${OURS}"/${repo}/${ARCH}/APKINDEX.tar.gz "${TEMP}"/APKINDEX.tar.gz.${repo}.tmp;
    mv "${TEMP}"/APKINDEX.tar.gz.${repo}.bak "${OURS}"/${repo}/${ARCH}/APKINDEX.tar.gz;
    mv "${TEMP}"/APKINDEX.tar.gz.${repo}.tmp "${TEMP}"/APKINDEX.tar.gz.${repo};
done


message "Sorting list of changed shared libraries"
for k in a b; do
    sort "${TEMP}"/list.${k} | uniq > "${TEMP}"/list.${k}.sort;
done


##
# NOTE: the '-q' to 'apk' gives us package names without
# version information. It may be possible for multiple
# versions to exist, and only one be affected, and that only
# the latest one could be affected, and that we already store
# the "old" version information, ... but it doesn't matter
# anyway since that information is derived from a binary repo
# and we only care about the source repo from here forward.
#
# The '-q' also removes the "blah is required by..." line.
#
# TODO: don't actually call 'apk' here (slow); parse the
# APKINDEX data and go from there.
#
message "Tracing dependencies for each changed shared library"
comm -23 "${TEMP}"/list.a.sort "${TEMP}"/list.b.sort | while read file; do
    apk search -rq so:${file##*/};
done > "${TEMP}"/deps


message "Finding parents of each affected package"
sort "${TEMP}"/deps | while read name; do
    grep ${name} scripts/.index; # generated by 'setup'
done | awk '{print $1}' | sort | uniq > "${TEMP}"/parents;


##
# TODO: since we cannot know which repository(ies) provide the
# possibly-affected package(s), we must search all repositories
# for all occurrences.
#
message "Finding all APKBUILD files for each parent"
NEED="";
while read name; do
    for repo in ${REPO}; do
        if test -f ${repo}/${name}/APKBUILD; then
            NEED="${NEED} ${repo}/${name}";
        fi
    done    
done < "${TEMP}"/parents;



message "Removing any duplicate entries"
printf "%s\n" "${LIST}" | sort > "${TEMP}"/LIST;
printf "%s\n" "${NEED}" | tr ' ' '\n' | sort > "${TEMP}"/NEED;
comm -13 "${TEMP}"/LIST "${TEMP}"/NEED;

echo $TEMP;


message "Generating change list of local package binaries"
for repo in ${REPO}; do
    list_packages ${repo} > "${TEMP}"/manifest.${repo}.b;
done


message "Comparing backup and change lists (sanity check)"
for repo in ${REPO}; do
    diff -ur "${TEMP}"/manifest.${repo}.a "${TEMP}"/manifest.${repo}.b;
done


message "Cleaning up"
#rm -fr "${TEMP}";