path: root/scripts/bootstrap.sh

#!/bin/sh -e

#===============================================================
# README
#===============================================================
#
# overview
# --------
#
# Given a basic development environment ("Baseline System") that
# contains standard system utilities, this script bootstraps the
# Adélie Linux distribution for any suported target architecture
# (assuming that musl, gcc, etc. have been ported to it) without
# requiring 'root' privileges. The procedure is outlined below:
#
#     - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#                        (unstable, everything provided by user)
#
#     +-----------------+       User-provided tools. Dependency
#     | Baseline System |       of 'mcmtools', which will verify
#     +-----------------+       that these tools are available.
#              |
#        +----------+           Script to build pinned versions
#        | mcmtools |           of common system utilities, a
#        +----------+           host-arch host-libc toolchain, a
#              |                host-arch musl-libc toolchain...
#              |
#     - - - - -|- - - - - - - - - - - - - - - - - - - - - - - -
#              |                (stable versions, unstable libc)
#              |
#       +-------------+         ...and a 'chroot'-able rootfs. A
#       | seed rootfs |         sane, but not clean, environment
#       +-------------+         in which we begin the bootstrap.
#              |
#        +-----------+          Script to build 'PRoot' and its
#        | emulators |          dependencies, as well as static
#        +-----------+          QEMU user binaries. Add to seed.
#              |
#      +---------------+        Static musl libc cross toolchain
#      | musl cross tc |        targeting a given architecture.
#      +---------------+        Output binaries will run via the
#              |                'binfmt_misc' mechanism + QEMU.
#              |
#       +-------------+         Script to build Alpine Package
#       | build tools |         Keeper (APK) and dependencies.
#       +-------------+         All binaries are cross-compiled!
#              |
#              |
#       +-------------+         Script to build the minimum set
#       | seed rootfs |         of tools to start building the
#       +-------------+         "system/" package repository.
#
# mcmtools; a separate script
# that 
# mcmtools provides almost everything needed to build 'abuild',
# and it is a hard dependency for our bootstrap process now.
#
#     https://git.zv.io/toolchains/bootstrap
#
# The output of this step is a host-native toolchain with fixed
# versions of development utilities, and a host-architecture hosted toolchain that cross-compiles to both musl and/or
# a foreign CPU architecture. (Table above prevents this).
#

HERE="$(dirname $(readlink -f ${0}))";
TEMP="$(mktemp -d)";            # do not change if not expert!

#---------------------------------------------------------------
# initialization

usage ()
{
    printf "Usage: %s ARCH\n\n" "${0}";
    cat <<EOF
    ARCH is { aarch64, armv7, ppc64, ppc, x86_64, pmmx }

Optional environment variables:

    MCMTOOLS=/path/to/existing/mcmtools/ **MAY CAUSE DATA LOSS**
EOF
    exit 0;
}

##
# argv[1]: ARCH
#
# ARCH is the Adélie Linux target. This is the first step in the
# porting process, so e.g. mips64, sparc64, riscv64 will need to
# be added to this table when the time comes to port to them.
#
# ARCH is translated to canonical GCC and QEMU machine types.
#
case "${1}" in
#   adelie        gcc         qemu
#   ------        ---         ----
    aarch64)    m=aarch64;  q=aarch64;  ;;
    armv7)      m=armv7l ;  q=arm    ;  ;;
    ppc)        m=ppc    ;  q=ppc    ;  ;;
    ppc64)      m=ppc64  ;  q=ppc64  ;  ;;
    x86_64)     m=x86_64 ;  q=x86_64 ;  ;;
    pmmx)       m=i686   ;  q=i386   ;  ;;
    *) usage ;;
esac
shift;

test ! -n "${m}" && printf "Invoking '%s TARGET' where 'TARGET=%s' is not valid.\n" "${0}" "${m}" && exit 1;

##
# Internal variables. Do not modify this section directly.
#
CHAINS=https://git.zv.io/toolchains;
printf "CHAINS=%s\n" "${CHAINS}";

SYSTEM="-linux-musl";           # we only target musl on Linux
printf "SYSTEM=%s\n" "${SYSTEM}";

NATIVE=$(cc -dumpmachine);      # host arch, host libc
printf "NATIVE=%s\n" "${NATIVE}";

BUILDS="${NATIVE%%-*}${SYSTEM}"; # host arch, musl libc
printf "BUILDS=%s\n" "${BUILDS}";

TARGET="${m}${SYSTEM}";         # ultimate Adélie Linux target
printf "TARGET=%s\n" "${TARGET}";

MTOOLS=${MCMTOOLS:-"${TEMP}/mcmtools"}; # CAREFUL! MAY CAUSE DATA LOSS!
printf "MTOOLS=%s\n" "${MTOOLS}";


#---------------------------------------------------------------
# mcmtools

##
# Allow the user to supply an existing mcmtools installation. It
# is not ideal, but since mcmtools cannot be (easily) relocated,
# allow the user to save some CPU cycles at the cost of adding a
# bunch of tools to their existing installation. A temporary dir
# is used if this environment variable is omitted. Another case
# for providing a custom value is if '/tmp' is mounted weird.
#
if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code
    cd "${TEMP}";

    test -d bootstrap                                          \
        || git clone ${CHAINS}/bootstrap.git;
    cd bootstrap;
    git checkout 773363265da441f472c207fb59c1acecc8ba2eb4;

    ## seed rootfs
    #
    # This will build absolutely everything that is needed to be
    # self-reliant, except for some build deps for QEMU.
    #
    test -d "${MTOOLS}" ||                                     \
    DEST="${MTOOLS}"                                           \
    ARCH=${BUILDS}                                             \
        ./bootstrap                                            \
        ;
    test -f "${MTOOLS}"/config.mak ||                          \
        cp "${MTOOLS}"/tmp/musl-cross-make/config.mak          \
        "${MTOOLS}"/config.mak                                 \
        ;

    ##
    # emulators
    #
    # Dependencies are built with the mcmtools host toolchain; a
    # reason to not force musl here is in the event that these
    # cannot be built statically and the host libc is different.
    # Our priority is to obtain a functioning 'PRoot' above all.
    #
    # QEMU itself is built inside an Alpine Linux rootfs; we do
    # this because we still need Python 3 to build it. You can
    # manually provide your own static QEMU user binaries and be
    # on your way without Alpine, but it is a good 'PRoot' test.
    #
    test -d "${MTOOLS}/emus/bin" ||                            \
    PATH="${MTOOLS}/host/bin:${MTOOLS}/sys/bin"                \
    DEST="${MTOOLS}"                                           \
        ./prootemu                                             \
        ;
fi


##
# Now we have a musl-targeting toolchain that runs on the host.
# We need to build the same toolchain, but static.
#
# This is required for it to run inside 'PRoot' if it differs in
# architecture.
#
if ! test -d "${MTOOLS}/sys/tc/cross"; then # FIXME: no hard code
    cd "${TEMP}";

    test -d musl-cross-make                                    \
        || git clone ${CHAINS}/musl-cross-make.git;
    cd musl-cross-make;
    git pull; # always use the latest

    cp "${MTOOLS}"/config.mak config.mak;

    ## musl cross tc
    #
    # Build this toolchain statically using the musl toolchain
    # from the seed rootfs so that it is known to work correctly
    # (the original musl toolchain itself may itself be linked
    # with glibc or be unsafe to use in some contexts).
    #
    PATH="${MTOOLS}/musl/bin:${MTOOLS}/sys/bin" \
        ./scripts/buildcross ${TARGET} \
        ;
    cd output;
    mkdir -p "${MTOOLS}"/sys/tc/cross;
    tar -C "${MTOOLS}"/sys/tc/cross \
        --strip-components=1 \
        -xzf ${TARGET}-cross.tgz \
        ;
fi


##
# Finish setting up the rootfs to support what we're doing.
#
# FIXME: what of this is actually needed?
#
(
    cd "${MTOOLS}"/sys;
    mkdir -p dev;
    mkdir -p proc;
    mkdir -p sys;
    rm -fr usr;
    ln -s . usr;
)

##
# Build 'abuild', its dependencies, and other utilities.
# Once finished, add them to PATH.
#

PROOT_NO_SECCOMP=1 \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tc/cross/bin \
CROSS_COMPILE=${TARGET}- \
SHELL=/bin/sh \
DEST=/usr/local \
CURL_CA_BUNDLE=/cacert.pem \
${MTOOLS}/sys/emus/bin/proot \
    -S "${MTOOLS}"/sys \
    -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \
    -b "${HERE}" \
    "${HERE}"/setup-abuild \
    ;

echo ok
exit 0;

export PATH="${MTOOLS}/abuild/bin:${PATH}";
bash

##
# Additional configuration.
#
export PACKAGER="Zach van Rijn <me@zv.io>";
export CBUILD=${TARGET_ARCH};
export CBUILDROOT="${MCMTOOLS}/abuild/${TARGET_ARCH}";
export ABUILD_USERDIR="${CBUILDROOT}.conf";
export SRCDEST="${MCMTOOLS}/abuild/src";
export REPODEST="${MCMTOOLS}/abuild/apk";
export ABUILD_APK_INDEX_OPTS="--allow-untrusted"; # FIXME
#export BUILD_ROOT="${CBUILDROOT}";

SUDO_APK=abuild-apk

# get abuild configurables
[ -e "${MCMTOOLS}/abuild/share/abuild/functions.sh" ] || (echo "abuild not found" ; exit 1)
CBUILDROOT="$(CTARGET=$TARGET_ARCH . ${MCMTOOLS}/abuild/share/abuild/functions.sh ; echo $CBUILDROOT)"
. "${MCMTOOLS}/abuild/share/abuild/functions.sh"
[ -z "$CBUILD_ARCH" ] && die "abuild is too old (use 2.29.0 or later)"
[ -z "$CBUILDROOT" ] && die "CBUILDROOT not set for $TARGET_ARCH"

# deduce aports directory
[ -z "$APORTS" ] && APORTS=$(realpath $(dirname $0)/../)
[ -e "$APORTS/system/build-tools" ] || die "Unable to deduce packages directory"

apkbuildname() {
	local repo="${1%%/*}"
	local pkg="${1##*/}"
	[ "$repo" = "$1" ] && repo="system"
	echo $APORTS/$repo/$pkg/APKBUILD
}

msg() {
	[ -n "$quiet" ] && return 0
	local prompt="$GREEN>>>${NORMAL}"
	local name="${BLUE}bootstrap-${TARGET_ARCH}${NORMAL}"
        printf "${prompt} ${name}: %s\n" "$1" >&2
}

if [ -z "$TARGET_ARCH" ]; then
	program=$(basename $0)
	cat <<EOF
usage: $program TARGET_ARCH

This script creates a local cross-compiler, and uses it to
cross-compile an Adélie Linux base system for new architecture.

Steps for introducing new architecture include:
- adding the compiler tripler and arch type to abuild
- adding the arch type detection to apk-tools
- adjusting build rules for packages that are arch aware:
  gcc, musl, binutils, easy-kernel
- create new kernel config for easy-kernel

After these steps the initial cross-build can be completed
by running this with the target arch as parameter, e.g.:
	./$program aarch64

EOF
	return 1
fi


##
# Package signing keys. Public and Private keys are stored in a
# different location; variables for which are installed to arch-
# specific 'abuild.conf' file.
#
if [ ! -d "$CBUILDROOT/etc/apk/keys" ] || [ -n "$(find $CBUILDROOT -type f -name '*.rsa')" ]; then
	msg "Creating sysroot in $CBUILDROOT"
	mkdir -p "$CBUILDROOT/etc/apk/keys"
    abuild-keygen -an;
    p=$(find "${ABUILD_USERDIR}" -type f -name "*.rsa.pub");
    mv "${p}" "$CBUILDROOT/etc/apk/keys";
    grep 1>/dev/null PACKAGER_PUBKEY= "${ABUILD_USERDIR}/abuild.conf" || printf >> "${ABUILD_USERDIR}/abuild.conf" "PACKAGER_PUBKEY=\"%s\"\n" "$CBUILDROOT/etc/apk/keys/${p##*/}";
fi


##
# APK database.
#
if [ ! -f "${CBUILDROOT}/._database-${TARGET_ARCH}" ]; then
    mkdir -p "${CBUILDROOT}/var/log"; # why not created by default?
    ${SUDO_APK} add --quiet --initdb --arch $TARGET_ARCH --root $CBUILDROOT
    touch "${CBUILDROOT}/._database-${TARGET_ARCH}";
fi

msg "Building cross-compiler"

# Build and install cross binutils (--with-sysroot)
CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname binutils) abuild -r
exit
if ! CHOST=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname musl) abuild up2date 2>/dev/null; then
	# C-library headers for target
	CHOST=$TARGET_ARCH BOOTSTRAP=nocc APKBUILD=$(apkbuildname musl) abuild -r

	# Minimal cross GCC
	EXTRADEPENDS_HOST="musl-dev" \
	CTARGET=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname gcc) abuild -r

	# Cross build bootstrap C-library for the target
	EXTRADEPENDS_BUILD="gcc-pass2-$TARGET_ARCH" \
	CHOST=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname musl) abuild -r
fi

# Full cross GCC
EXTRADEPENDS_TARGET="musl musl-dev" \
CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname gcc) abuild -r

# Cross build-base
CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname build-base) abuild -r

msg "Cross building base system"
msg "Change your abuild.conf NOW to avoid build errors!!"
read

# add implicit target prerequisite packages
apk info --quiet --installed --root "$CBUILDROOT" libgcc libstdc++ musl-dev || \
	${SUDO_APK} --root "$CBUILDROOT" add --repository "$REPODEST/system" libgcc libstdc++ musl-dev

# ordered cross-build
for PKG in musl pkgconf zlib \
	   gettext-tiny ncurses bash binutils make bison flex m4 \
	   openssl apk-tools linux-pam shadow \
	   gmp mpfr3 mpc1 isl gcc ca-certificates \
	   openrc libcap-ng coreutils sed gzip bzip2 diffutils \
	   attr libcap patch sudo acl fakeroot libarchive mawk \
	   pax-utils abuild grep findutils patch lzip unzip autoconf automake libtool \
	   ncurses util-linux lvm2 popt xz lddtree libssh2 curl build-tools pcre \
	   debianutils file shimmy procps zsh sharutils net-tools check kbd sysklogd vim db groff libpipeline man-db psmisc less \
	   adelie-base \
	    ; do

	CHOST=$TARGET_ARCH BOOTSTRAP=bootimage APKBUILD=$(apkbuildname $PKG) abuild -r

	case "$PKG" in
	fortify-headers | libc-dev | build-base)
		# headers packages which are implicit but mandatory dependency
		apk info --quiet --installed --root "$CBUILDROOT" $PKG || \
			${SUDO_APK} --update --root "$CBUILDROOT" --repository "$REPODEST/system" add $PKG
		;;
	musl | gcc)
		# target libraries rebuilt, force upgrade
		[ "$(apk upgrade --root "$CBUILDROOT" --repository "$REPODEST/main" --available --simulate | wc -l)" -gt 1 ] &&
			${SUDO_APK} upgrade --root "$CBUILDROOT" --repository "$REPODEST/main" --available
		;;
	esac
done