more...

2 files changed, 274 insertions, 25 deletions

diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh
index db165140b..d5334b11b 100755
--- a/scripts/bootstrap.sh
+++ b/scripts/bootstrap.sh
@@ -1,37 +1,260 @@
 #!/bin/sh -e
 
-set -x
+#===============================================================
+# README
+#===============================================================
+#
+# overview
+# --------
+#
+# Given a basic development environment ("Baseline System") that
+# contains standard system utilities, this script bootstraps the
+# Adélie Linux distribution for any suported target architecture
+# (assuming that musl, gcc, etc. have been ported to it) without
+# requiring 'root' privileges. The procedure is outlined below:
+#
+#     - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+#                        (unstable, everything provided by user)
+#
+#     +-----------------+       User-provided tools. Dependency
+#     | Baseline System |       of 'mcmtools', which will verify
+#     +-----------------+       that these tools are available.
+#              |
+#        +----------+           Script to build pinned versions
+#        | mcmtools |           of common system utilities, a
+#        +----------+           host-arch host-libc toolchain, a
+#              |                host-arch musl-libc toolchain...
+#              |
+#     - - - - -|- - - - - - - - - - - - - - - - - - - - - - - -
+#              |                (stable versions, unstable libc)
+#              |
+#       +-------------+         ...and a 'chroot'-able rootfs. A
+#       | seed rootfs |         sane, but not clean, environment
+#       +-------------+         in which we begin the bootstrap.
+#              |
+#        +-----------+          Script to build 'PRoot' and its
+#        | emulators |          dependencies, as well as static
+#        +-----------+          QEMU user binaries. Add to seed.
+#              |
+#      +---------------+        Static musl libc cross toolchain
+#      | musl cross tc |        targeting a given architecture.
+#      +---------------+        Output binaries will run via the
+#              |                'binfmt_misc' mechanism + QEMU.
+#              |
+#       +-------------+         Script to build Alpine Package
+#       | build tools |         Keeper (APK) and dependencies.
+#       +-------------+         All binaries are cross-compiled!
+#              |
+#              |
+#       +-------------+         Script to build the minimum set
+#       | seed rootfs |         of tools to start building the
+#       +-------------+         "system/" package repository.
+#
+# mcmtools; a separate script
+# that 
+# mcmtools provides almost everything needed to build 'abuild',
+# and it is a hard dependency for our bootstrap process now.
+#
+#     https://git.zv.io/toolchains/bootstrap
+#
+# The output of this step is a host-native toolchain with fixed
+# versions of development utilities, and a host-architecture hosted toolchain that cross-compiles to both musl and/or
+# a foreign CPU architecture. (Table above prevents this).
+#
 
 HERE="$(dirname $(readlink -f ${0}))";
+TEMP="$(mktemp -d)";            # do not change if not expert!
+
+#---------------------------------------------------------------
+# initialization
+
+usage ()
+{
+    printf "Usage: %s ARCH\n\n" "${0}";
+    cat <<EOF
+    ARCH is { aarch64, armv7, ppc64, ppc, x86_64, pmmx }
+
+Optional environment variables:
+
+    MCMTOOLS=/path/to/existing/mcmtools/ **MAY CAUSE DATA LOSS**
+EOF
+    exit 0;
+}
 
 ##
-# mcmtools provides almost everything needed to build 'abuild',
-# and it is a hard dependency for our bootstrap process now.
+# argv[1]: ARCH
 #
-#     https://git.zv.io/toolchains/bootstrap
+# ARCH is the Adélie Linux target. This is the first step in the
+# porting process, so e.g. mips64, sparc64, riscv64 will need to
+# be added to this table when the time comes to port to them.
 #
-# It is possible to bootstrap from a non- Alpine/Adélie system.
-# We will build 'abuild' and other utilities momentarily.
+# ARCH is translated to canonical GCC and QEMU machine types.
 #
-MCMTOOLS=${MCMTOOLS:-"$HOME/mcmtools"}
-test ! -d "${MCMTOOLS}/sys/bin" && printf "Environment 'MCMTOOLS=%s' is not valid.\n" "${MCMTOOLS}" && exit 1;
+case "${1}" in
+#   adelie        gcc         qemu
+#   ------        ---         ----
+    aarch64)    m=aarch64;  q=aarch64;  ;;
+    armv7)      m=armv7l ;  q=arm    ;  ;;
+    ppc)        m=ppc    ;  q=ppc    ;  ;;
+    ppc64)      m=ppc64  ;  q=ppc64  ;  ;;
+    x86_64)     m=x86_64 ;  q=x86_64 ;  ;;
+    pmmx)       m=i686   ;  q=i386   ;  ;;
+    *) usage ;;
+esac
+shift;
+
+test ! -n "${m}" && printf "Invoking '%s TARGET' where 'TARGET=%s' is not valid.\n" "${0}" "${m}" && exit 1;
+
+##
+# Internal variables. Do not modify this section directly.
+#
+CHAINS=https://git.zv.io/toolchains;
+printf "CHAINS=%s\n" "${CHAINS}";
+
+SYSTEM="-linux-musl";           # we only target musl on Linux
+printf "SYSTEM=%s\n" "${SYSTEM}";
+
+NATIVE=$(cc -dumpmachine);      # host arch, host libc
+printf "NATIVE=%s\n" "${NATIVE}";
+
+BUILDS="${NATIVE%%-*}${SYSTEM}"; # host arch, musl libc
+printf "BUILDS=%s\n" "${BUILDS}";
+
+TARGET="${m}${SYSTEM}";         # ultimate Adélie Linux target
+printf "TARGET=%s\n" "${TARGET}";
+
+MTOOLS=${MCMTOOLS:-"${TEMP}/mcmtools"}; # CAREFUL! MAY CAUSE DATA LOSS!
+printf "MTOOLS=%s\n" "${MTOOLS}";
+
+
+#---------------------------------------------------------------
+# mcmtools
+
+##
+# Allow the user to supply an existing mcmtools installation. It
+# is not ideal, but since mcmtools cannot be (easily) relocated,
+# allow the user to save some CPU cycles at the cost of adding a
+# bunch of tools to their existing installation. A temporary dir
+# is used if this environment variable is omitted. Another case
+# for providing a custom value is if '/tmp' is mounted weird.
+#
+if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code
+    cd "${TEMP}";
+
+    test -d bootstrap                                          \
+        || git clone ${CHAINS}/bootstrap.git;
+    cd bootstrap;
+    git checkout 773363265da441f472c207fb59c1acecc8ba2eb4;
+
+    ## seed rootfs
+    #
+    # This will build absolutely everything that is needed to be
+    # self-reliant, except for some build deps for QEMU.
+    #
+    test -d "${MTOOLS}" ||                                     \
+    DEST="${MTOOLS}"                                           \
+    ARCH=${BUILDS}                                             \
+        ./bootstrap                                            \
+        ;
+    test -f "${MTOOLS}"/config.mak ||                          \
+        cp "${MTOOLS}"/tmp/musl-cross-make/config.mak          \
+        "${MTOOLS}"/config.mak                                 \
+        ;
+
+    ##
+    # emulators
+    #
+    # Dependencies are built with the mcmtools host toolchain; a
+    # reason to not force musl here is in the event that these
+    # cannot be built statically and the host libc is different.
+    # Our priority is to obtain a functioning 'PRoot' above all.
+    #
+    # QEMU itself is built inside an Alpine Linux rootfs; we do
+    # this because we still need Python 3 to build it. You can
+    # manually provide your own static QEMU user binaries and be
+    # on your way without Alpine, but it is a good 'PRoot' test.
+    #
+    test -d "${MTOOLS}/emus/bin" ||                            \
+    PATH="${MTOOLS}/host/bin:${MTOOLS}/sys/bin"                \
+    DEST="${MTOOLS}"                                           \
+        ./prootemu                                             \
+        ;
+fi
 
 
 ##
-# TARGET_ARCH (argv[1]) is used during cross-compilation.
+# Now we have a musl-targeting toolchain that runs on the host.
+# We need to build the same toolchain, but static.
 #
-TARGET_ARCH="$1"
-test ! -n "${TARGET_ARCH}" && printf "Invoking '%s TARGET_ARCH' where 'TARGET_ARCH=%s' is not valid.\n" "${0}" "${TARGET_ARCH}" && exit 1;
+# This is required for it to run inside 'PRoot' if it differs in
+# architecture.
+#
+if ! test -d "${MTOOLS}/sys/tc/cross"; then # FIXME: no hard code
+    cd "${TEMP}";
+
+    test -d musl-cross-make                                    \
+        || git clone ${CHAINS}/musl-cross-make.git;
+    cd musl-cross-make;
+    git pull; # always use the latest
+
+    cp "${MTOOLS}"/config.mak config.mak;
+
+    ## musl cross tc
+    #
+    # Build this toolchain statically using the musl toolchain
+    # from the seed rootfs so that it is known to work correctly
+    # (the original musl toolchain itself may itself be linked
+    # with glibc or be unsafe to use in some contexts).
+    #
+    PATH="${MTOOLS}/musl/bin:${MTOOLS}/sys/bin" \
+        ./scripts/buildcross ${TARGET} \
+        ;
+    cd output;
+    mkdir -p "${MTOOLS}"/sys/tc/cross;
+    tar -C "${MTOOLS}"/sys/tc/cross \
+        --strip-components=1 \
+        -xzf ${TARGET}-cross.tgz \
+        ;
+fi
 
 
 ##
+# Finish setting up the rootfs to support what we're doing.
+#
+# FIXME: what of this is actually needed?
+#
+(
+    cd "${MTOOLS}"/sys;
+    mkdir -p dev;
+    mkdir -p proc;
+    mkdir -p sys;
+    rm -fr usr;
+    ln -s . usr;
+)
+
+##
 # Build 'abuild', its dependencies, and other utilities.
 # Once finished, add them to PATH.
 #
-"${HERE}/setup-abuild" ${TARGET_ARCH};
-export PATH="${MCMTOOLS}/abuild/bin:${PATH}";
-bash;
-exit;
+
+PROOT_NO_SECCOMP=1 \
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tc/cross/bin \
+CROSS_COMPILE=${TARGET}- \
+SHELL=/bin/sh \
+DEST=/usr/local \
+CURL_CA_BUNDLE=/cacert.pem \
+${MTOOLS}/sys/emus/bin/proot \
+    -S "${MTOOLS}"/sys \
+    -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \
+    -b "${HERE}" \
+    "${HERE}"/setup-abuild \
+    ;
+
+echo ok
+exit 0;
+
+export PATH="${MTOOLS}/abuild/bin:${PATH}";
+bash
 
 ##
 # Additional configuration.
diff --git a/scripts/setup-abuild b/scripts/setup-abuild
index 95e56d787..a0f8fb9d0 100755
--- a/scripts/setup-abuild
+++ b/scripts/setup-abuild
@@ -2,21 +2,46 @@
 
 ##
 # This script is to be called from 'bootstrap.sh', not sourced.
-# PATH is fully contained. We install to 'MCMTOOLS/abuild/'.
+# PATH is fully contained. We install to 'MCMTOOLS/sys', which
+# will add (and may overwrite) in some cases. This process is
+# destructive. We do this so that we may easily 'chroot' later.
 #
 HERE="$(dirname $(readlink -f ${0}))";
-DEST="${MCMTOOLS}/abuild";
-export PATH="${MCMTOOLS}/sys/bin:${MCMTOOLS}/host/bin:${DEST}/bin";
+DEST=/usr/local;
+
+git config --global http.sslCAInfo "${CURL_CA_BUNDLE}";
 
 mkdir -p "${DEST}";
-cd "${DEST}";
+cd "${DEST}"; # this directory will already exist if correct
+
+if false; then # provided by latest 'bootstrap'
+##
+# Perl
+#
+nprl=perl;
+vprl=5.36.0;
+test ! -f ._${vprl}-${vprl} &&                                 \
+(
+    test ! -d ${nprl}-${vprl}                                  \
+        && curl -s https://www.cpan.org/src/${vprl%%.*}.0/perl-${vprl}.tar.gz \
+        | tar -xzf -                                           \
+        ;
+    cd ${nprl}-${vprl};
+    rm -fr x; mkdir x; cd x;
+    ../Configure -des;
+    make -j$(nproc);
+    make install;
+)
+touch ._${nprl}-${vprl};
+rm -fr  ${nprl}-${vprl};
+fi
 
 
 ##
 # OpenSSL
 #
 nssl=openssl;
-vssl=1.1.1m;
+vssl=1.1.1s;
 test ! -f ._${nssl}-${vssl} &&                                 \
 (
     test ! -d ${nssl}-${vssl}                                  \
@@ -41,16 +66,17 @@ rm -fr  ${nssl}-${vssl};
 # abuild
 #
 nbld=abuild;
-vbld=681ef9dfcf6e57bdda767efed7e50ce8e9de0563
+vbld=ee13f777d56e8f3e8848fec9a422430a66292cc1;
 test ! -f ._${nbld}-${vbld} &&                                 \
 (
     test ! -d ${nbld}-${vbld}                                  \
         && git clone https://git.alpinelinux.org/${nbld} ${nbld}-${vbld} \
         ;
     cd abuild-${vbld};
-    patch -p1 --forward < "${HERE}/patches/0001-allow-untrusted.diff" || true; # FIXME
-    patch -p1 --forward < "${HERE}/patches/0001-etc-apk-keys.diff" || true; # FIXME
-    patch -p1 --forward < "${HERE}/patches/0001-extra-lib-paths.diff" || true; # FIXME
+    git checkout ${vbld};
+#    patch -p1 --forward < "${HERE}/patches/0001-allow-untrusted.diff" || true; # FIXME
+#    patch -p1 --forward < "${HERE}/patches/0001-etc-apk-keys.diff" || true; # FIXME
+#    patch -p1 --forward < "${HERE}/patches/0001-extra-lib-paths.diff" || true; # FIXME
     rm -fr x; mkdir x; cd x;
     export SSL_CFLAGS="-I${DEST}/include -I${MCMTOOLS}/sys/include";
     export SSL_LDFLAGS="-L${DEST}/lib -L${MCMTOOLS}/sys/lib";
@@ -145,7 +171,7 @@ test ! -f ._${nsam}-${vsam} &&
         PREFIX=""                                              \
         DESTDIR="${DEST}"                                      \
         ;
-    
+
 )
 touch ._${nsam}-${vsam};
 rm -fr  ${nsam}-${vsam};