Initial draft of new bootstrap tooling.

5 files changed, 337 insertions, 11 deletions

diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh
index 87d7b1d48..78f8ac7ff 100755
--- a/scripts/bootstrap.sh
+++ b/scripts/bootstrap.sh
@@ -1,14 +1,56 @@
-#!/bin/sh
+#!/bin/sh -e
 
-set -e
+set -x
 
+HERE="$(dirname $(readlink -f ${0}))";
+
+##
+# mcmtools provides almost everything needed to build 'abuild',
+# and it is a hard dependency for our bootstrap process now.
+#
+#     https://git.zv.io/toolchains/bootstrap
+#
+# It is possible to bootstrap from a non- Alpine/Adélie system.
+# We will build 'abuild' and other utilities momentarily.
+#
+MCMTOOLS=${MCMTOOLS:-"$HOME/mcmtools"}
+test ! -d "${MCMTOOLS}/sys/bin" && printf "Environment 'MCMTOOLS=%s' is not valid.\n" "${MCMTOOLS}" && exit 1;
+
+
+##
+# TARGET_ARCH (argv[1]) is used during cross-compilation.
+#
 TARGET_ARCH="$1"
+test ! -n "${TARGET_ARCH}" && printf "Invoking '%s TARGET_ARCH' where 'TARGET_ARCH=%s' is not valid.\n" "${0}" "${TARGET_ARCH}" && exit 1;
+
+
+##
+# Build 'abuild', its dependencies, and other utilities.
+# Once finished, add them to PATH.
+#
+"${HERE}/setup-abuild" ${TARGET_ARCH};
+export PATH="${MCMTOOLS}/abuild/bin:${PATH}";
+#bash;
+#exit;
+
+##
+# Additional configuration.
+#
+export PACKAGER="Zach van Rijn <me@zv.io>";
+export CBUILD=${TARGET_ARCH};
+export CBUILDROOT="${MCMTOOLS}/abuild/${TARGET_ARCH}";
+export ABUILD_USERDIR="${CBUILDROOT}.conf";
+export SRCDEST="${MCMTOOLS}/abuild/src";
+export REPODEST="${MCMTOOLS}/abuild/apk";
+export ABUILD_APK_INDEX_OPTS="--allow-untrusted"; # FIXME
+#export BUILD_ROOT="${CBUILDROOT}";
+
 SUDO_APK=abuild-apk
 
 # get abuild configurables
-[ -e /usr/share/abuild/functions.sh ] || (echo "abuild not found" ; exit 1)
-CBUILDROOT="$(CTARGET=$TARGET_ARCH . /usr/share/abuild/functions.sh ; echo $CBUILDROOT)"
-. /usr/share/abuild/functions.sh
+[ -e "${MCMTOOLS}/abuild/share/abuild/functions.sh" ] || (echo "abuild not found" ; exit 1)
+CBUILDROOT="$(CTARGET=$TARGET_ARCH . ${MCMTOOLS}/abuild/share/abuild/functions.sh ; echo $CBUILDROOT)"
+. "${MCMTOOLS}/abuild/share/abuild/functions.sh"
 [ -z "$CBUILD_ARCH" ] && die "abuild is too old (use 2.29.0 or later)"
 [ -z "$CBUILDROOT" ] && die "CBUILDROOT not set for $TARGET_ARCH"
 
@@ -53,19 +95,37 @@ EOF
 	return 1
 fi
 
-if [ ! -d "$CBUILDROOT" ]; then
+
+##
+# Package signing keys. Public and Private keys are stored in a
+# different location; variables for which are installed to arch-
+# specific 'abuild.conf' file.
+#
+if [ ! -d "$CBUILDROOT/etc/apk/keys" ] || [ -n "$(find $CBUILDROOT -type f -name '*.rsa')" ]; then
 	msg "Creating sysroot in $CBUILDROOT"
 	mkdir -p "$CBUILDROOT/etc/apk/keys"
-	cp -a /etc/apk/keys/* "$CBUILDROOT/etc/apk/keys"
-	${SUDO_APK} add --quiet --initdb --arch $TARGET_ARCH --root $CBUILDROOT
+    abuild-keygen -an;
+    p=$(find "${ABUILD_USERDIR}" -type f -name "*.rsa.pub");
+    mv "${p}" "$CBUILDROOT/etc/apk/keys";
+    grep 1>/dev/null PACKAGER_PUBKEY= "${ABUILD_USERDIR}/abuild.conf" || printf >> "${ABUILD_USERDIR}/abuild.conf" "PACKAGER_PUBKEY=\"%s\"\n" "$CBUILDROOT/etc/apk/keys/${p##*/}";
+fi
+
+
+##
+# APK database.
+#
+if [ ! -f "${CBUILDROOT}/._database-${TARGET_ARCH}" ]; then
+    mkdir -p "${CBUILDROOT}/var/log"; # why not created by default?
+    ${SUDO_APK} add --quiet --initdb --arch $TARGET_ARCH --root $CBUILDROOT
+    touch "${CBUILDROOT}/._database-${TARGET_ARCH}";
 fi
 
 msg "Building cross-compiler"
 
 # Build and install cross binutils (--with-sysroot)
 CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname binutils) abuild -r
-
-#if ! CHOST=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname musl) abuild up2date 2>/dev/null; then
+exit
+if ! CHOST=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname musl) abuild up2date 2>/dev/null; then
 	# C-library headers for target
 	CHOST=$TARGET_ARCH BOOTSTRAP=nocc APKBUILD=$(apkbuildname musl) abuild -r
 
@@ -76,7 +136,7 @@ CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname binutils) abuild -
 	# Cross build bootstrap C-library for the target
 	EXTRADEPENDS_BUILD="gcc-pass2-$TARGET_ARCH" \
 	CHOST=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname musl) abuild -r
-#fi
+fi
 
 # Full cross GCC
 EXTRADEPENDS_TARGET="musl musl-dev" \
diff --git a/scripts/patches/0001-allow-untrusted.diff b/scripts/patches/0001-allow-untrusted.diff
new file mode 100644
index 000000000..868d25a69
--- /dev/null
+++ b/scripts/patches/0001-allow-untrusted.diff
@@ -0,0 +1,10 @@
+--- a/abuild-sudo.c	2021-10-09 16:55:36.705593308 -0500
++++ b/abuild-sudo.c	2021-10-09 16:55:48.037927253 -0500
+@@ -33,8 +33,6 @@
+ };
+ 
+ static const char* invalid_opts[] = {
+-	"--allow-untrusted",
+-	"--keys-dir",
+ 	NULL,
+ };
diff --git a/scripts/patches/0001-etc-apk-keys.diff b/scripts/patches/0001-etc-apk-keys.diff
new file mode 100644
index 000000000..e2bc4d9de
--- /dev/null
+++ b/scripts/patches/0001-etc-apk-keys.diff
@@ -0,0 +1,48 @@
+diff -ur a/abuild.in b/abuild.in
+--- a/abuild.in	2021-10-09 17:36:01.878022231 -0500
++++ b/abuild.in	2021-10-09 17:37:38.584487176 -0500
+@@ -2392,7 +2392,7 @@
+ 	cp /etc/abuild.conf /etc/group /etc/passwd "$BUILD_ROOT/etc"
+ 
+ 	local dir
+-	for dir in /usr/share/apk/keys/$CBUILD_ARCH /etc/apk/keys; do
++	for dir in /usr/share/apk/keys/$CBUILD_ARCH ${CBUILDROOT}/etc/apk/keys; do
+ 		cp $dir/* "$BUILD_ROOT/etc/apk/keys"
+ 	done
+ 
+diff -ur a/abuild-keygen.in b/abuild-keygen.in
+--- a/abuild-keygen.in	2021-10-09 17:36:01.878022231 -0500
++++ b/abuild-keygen.in	2021-10-09 17:39:18.403044294 -0500
+@@ -59,9 +59,9 @@
+ 	openssl rsa -in "$privkey" -pubout -out "$pubkey"
+ 
+ 	if [ -n "$install_pubkey" ]; then
+-		msg "Installing $pubkey to /etc/apk/keys..."
+-		$SUDO mkdir -p "${abuild_keygen_install_root}"/etc/apk/keys
+-		$SUDO cp ${interactive:+-i} "$pubkey" "${abuild_keygen_install_root}"/etc/apk/keys/
++		msg "Installing $pubkey to ${CBUILDROOT}/etc/apk/keys..."
++		$SUDO mkdir -p "${CBUILDROOT}"/etc/apk/keys
++		$SUDO cp ${interactive:+-i} "$pubkey" "${CBUILDROOT}"/etc/apk/keys/
+ 	else
+ 
+ 		msg ""
+@@ -98,7 +98,7 @@
+ 		  -a, --append         Set PACKAGER_PRIVKEY=<generated key> in
+ 		                       $ABUILD_USERCONF
+ 
+-		  -i, --install        Install public key into /etc/apk/keys using sudo
++		  -i, --install        Install public key into ${CBUILDROOT}/etc/apk/keys using sudo
+ 		  -n                   Non-interactive. Use defaults
+ 		  -b, --numbits [BITS] The size of the private key to generate in bits.
+ 		  -q, --quiet
+diff -ur a/abuild-sign.in b/abuild-sign.in
+--- a/abuild-sign.in	2021-10-09 17:36:01.878022231 -0500
++++ b/abuild-sign.in	2021-10-09 17:37:38.588487278 -0500
+@@ -60,7 +60,7 @@
+ 		  -e, --installed    Check only of there exist a private key for signing
+ 		  -k, --private KEY  The private key to use for signing
+ 		  -p, --public KEY   The name of public key. apk add will look for
+-		                     /etc/apk/keys/KEY
++		                     ${CBUILDROOT}/etc/apk/keys/KEY
+ 		  -q, --quiet
+ 		  -h, --help         Show this help
diff --git a/scripts/patches/0001-extra-lib-paths.diff b/scripts/patches/0001-extra-lib-paths.diff
new file mode 100644
index 000000000..2dfb49e4e
--- /dev/null
+++ b/scripts/patches/0001-extra-lib-paths.diff
@@ -0,0 +1,20 @@
+--- a	2021-10-09 20:59:28.977394444 -0500
++++ b	2021-10-09 21:00:09.470414858 -0500
+@@ -1363,7 +1363,7 @@
+ 	local rpaths=$(cat "$1")
+ 	shift
+ 	while [ $# -gt 0 ]; do
+-		real_so_path "$1" /usr/lib /lib $rpaths || return 1
++		real_so_path "$1" /usr/lib /lib $(ldd /tmp/m32/host/bin/gcc | grep -oE '/.*\ ' | xargs -I {} dirname {} | sort | uniq) $rpaths || return 1
+ 		shift
+ 	done
+ 	return 0
+@@ -1396,7 +1396,7 @@
+ 	local apkroot=
+ 
+ 	case "$parch" in
+-	$CBUILD_ARCH) ;;
++	$CBUILD_ARCH) apkroot="--root $CBUILDROOT" ;;
+ 	$CARCH | $CTARGET_ARCH) apkroot="--root $CBUILDROOT --arch $CTARGET_ARCH" ;;
+ 	esac
+ 
diff --git a/scripts/setup-abuild b/scripts/setup-abuild
new file mode 100755
index 000000000..cc8e238be
--- /dev/null
+++ b/scripts/setup-abuild
@@ -0,0 +1,188 @@
+#!/bin/sh -e
+
+##
+# This script is to be called from 'bootstrap.sh', not sourced.
+# PATH is fully contained. We install to 'MCMTOOLS/abuild/'.
+#
+export PATH="${MCMTOOLS}/sys/bin:${MCMTOOLS}/host/bin";
+
+HERE="$(dirname $(readlink -f ${0}))";
+DEST="${MCMTOOLS}/abuild";
+
+mkdir -p "${DEST}";
+cd "${DEST}";
+
+
+##
+# OpenSSL
+#
+nssl=openssl;
+vssl=1.1.1l;
+test ! -f ._${nssl}-${vssl} &&                                 \
+(
+    test ! -d ${nssl}-${vssl}                                  \
+        && curl -s https://www.openssl.org/source/${nssl}-${vssl}.tar.gz \
+        | tar -xzf -                                           \
+        ;
+    cd ${nssl}-${vssl};
+    rm -fr x; mkdir x; cd x;
+    ../Configure cc                                            \
+        --prefix="${DEST}"                                     \
+        --openssldir="${DEST}"                                 \
+        no-shared                                              \
+        ;
+    make -j$(nproc);
+    make install_sw install_ssldirs;
+)
+touch ._${nssl}-${vssl};
+rm -fr  ${nssl}-${vssl};
+
+
+##
+# abuild
+#
+nbld=abuild;
+vbld=ff913b49072352604ce081378f015af1714e1bd2;
+test ! -f ._${nbld}-${vbld} &&                                 \
+(
+    test ! -d ${nbld}-${vbld}                                  \
+        && git clone https://git.alpinelinux.org/${nbld} ${nbld}-${vbld} \
+        ;
+    cd abuild-${vbld};
+    patch -p1 --forward < "${HERE}/patches/0001-allow-untrusted.diff" || true; # FIXME
+    patch -p1 --forward < "${HERE}/patches/0001-etc-apk-keys.diff" || true; # FIXME
+    patch -p1 --forward < "${HERE}/patches/0001-extra-lib-paths.diff" || true; # FIXME
+    rm -fr x; mkdir x; cd x;
+    export SSL_CFLAGS="-I${DEST}/include -I${MCMTOOLS}/sys/include";
+    export SSL_LDFLAGS="-L${DEST}/lib -L${MCMTOOLS}/sys/lib";
+    export SSL_LIBS="-lssl -lcrypto"; # not in mcmtools
+    export ZLIB_LIBS="-lz"; # from mcmtools
+    export CFLAGS="-DABUILD_GROUP=\\\"$(id -gn)\\\""; # default 'abuild' if undefined
+    sed -i "${DEST}/abuild-${vbld}/abuild-sudo.c"              \
+        -e "s@/sbin/apk@${DEST}/bin/apk@"                      \
+        ; # hardcoded
+    make -j$(nproc) -C .. install                              \
+        prefix="${DEST}"                                       \
+        sysconfdir="${DEST}"                                   \
+        ;
+    sed -i "${DEST}/bin/abuild"                                \
+        -e 's@/bin/ash -e@/usr/bin/env bash@'                  \
+        ; # hardcoded
+)
+touch ._${nbld}-${vbld};
+#rm -fr  ${nbld}-${vbld};
+
+
+##
+# util-linux (for 'getopt' used by 'abuild-keygen')
+#
+nutl=util-linux;
+vutl=2.37.2;
+test ! -f ._${nutl}-${vutl} &&                                 \
+(
+    test ! -d ${nutl}-${vutl}                                  \
+        && mkdir ${nutl}-${vutl}                               \
+        && curl -sL https://github.com/karelzak/${nutl}/archive/refs/tags/v${vutl}.tar.gz \
+        | tar -C ${nutl}-${vutl} --strip-components=1 -xzf -   \
+        ;
+    cd ${nutl}-${vutl};
+    test -f configure || ./autogen.sh;
+    rm -fr x; mkdir x; cd x;
+    ../configure                                               \
+        --prefix="${DEST}"                                     \
+        --enable-static                                        \
+        --disable-shared                                       \
+        ;
+    sed -i Makefile                                            \
+        -e 's/chgrp/-chgrp/g'                                  \
+        -e 's/chmod/-chmod/g'                                  \
+        -e 's/chown/-chown/g'                                  \
+        ; # allow non-root installation
+    make -j$(nproc) install;
+)
+touch ._${nutl}-${vutl};
+rm -fr  ${nutl}-${vutl};
+
+
+##
+# apk-tools
+#
+natl=apk-tools;
+vatl=9f07a3447ea1e8fb67cdbd5c30b2ea144e826490;
+test ! -f ._${natl}-${vatl} &&                                 \
+(
+    test ! -d ${natl}-${vatl}                                  \
+        && git clone https://git.alpinelinux.org/${natl} ${natl}-${vatl} \
+        ;
+    cd ${natl}-${vatl};
+    sed -i Make.rules                                          \
+        -e '/targets += $(__shlibs) $(shobjs)/d'               \
+        ; # disable shared libs
+    sed -i src/Makefile                                        \
+        -e 's/$(install-libapk_so)//g' -e 's/$(libapk_so)//g'  \
+        ; # disable shared libs
+    sed -i src/context.c                                       \
+        -e "s@var/log@${DEST}/${1}/var/log@"                   \
+        ; # hardcoded
+    export LUA=no; # documentation requires lua
+    make clean;
+    make                                                       \
+        INSTALLDIR="${DEST}"                                   \
+        CFLAGS="-I${DEST}/include -I${MCMTOOLS}/sys/include"   \
+        LDFLAGS="-L${DEST}/lib -L${MCMTOOLS}/sys/lib -L${DEST}/${natl}-${vatl}/libfetch" \
+        LIBS="-lapk -lfetch -lssl -lcrypto -lz"                \
+        ;
+    cp src/apk "${DEST}/bin";
+)
+touch ._${natl}-${vatl};
+rm -fr  ${natl}-${vatl};
+
+
+##
+# pax-utils
+#
+npax=pax-utils;
+vpax=1.3.3;
+test ! -f ._${npax}-${vpax} &&                                 \
+(
+    test ! -d ${npax}-${vpax}                                  \
+        && curl -s https://gitweb.gentoo.org/proj/${npax}.git/snapshot/${npax}-${vpax}.tar.gz \
+        | tar -xzf -                                           \
+        ;
+    cd ${npax}-${vpax};
+    sed -i Makefile                                            \
+        -e '/$(MPAGES)/d'                                      \
+        ;
+    rm -fr x; mkdir x; cd x;
+    make -j$(nproc) -C .. install                              \
+        PREFIX="${DEST}"                                       \
+        MANDIR="${DEST}/man"                                   \
+        USE_PYTHON=no                                          \
+        ;
+)
+touch ._${npax}-${vpax};
+rm -fr  ${npax}-${vpax};
+
+
+##
+# fakeroot
+#
+nfrt=fakeroot;
+vfrt=20210907T092512Z:1.26; # look on snapshot.d.o for this
+test ! -f ._${nfrt}-${vfrt#*:} &&                              \
+(
+    test ! -d ${nfrt}-${vfrt#*:}                               \
+        && curl -s https://snapshot.debian.org/archive/debian/${vfrt%:*}/pool/main/f/${nfrt}/${nfrt}_${vfrt#*:}.orig.tar.gz \
+        | tar -xzf -                                           \
+        ;
+    cd ${nfrt}-${vfrt#*:};
+    f=$(mktemp);
+    echo > ${f} "#include <linux/capability.h>";
+    cat libfakeroot.c >> ${f};
+    mv ${f} libfakeroot.c;
+    rm -fr x; mkdir x; cd x;
+    ../configure --prefix="${DEST}";
+    make -j$(nproc) install;
+)
+touch ._${nfrt}-${vfrt#*:};
+rm -fr  ${nfrt}-${vfrt#*:};