summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZach van Rijn <me@zv.io>2022-12-06 18:11:11 -0600
committerZach van Rijn <me@zv.io>2024-08-10 06:09:07 +0000
commit6deb53548e8e8fcc952882ec601e9251885e3646 (patch)
tree989840029d13d118a3a2f7e009fc753dcfc683dd
parentb2fe4cd61080ecd5bd51c730794cb6d03ab1151b (diff)
downloadpackages-6deb53548e8e8fcc952882ec601e9251885e3646.tar.gz
packages-6deb53548e8e8fcc952882ec601e9251885e3646.tar.bz2
packages-6deb53548e8e8fcc952882ec601e9251885e3646.tar.xz
packages-6deb53548e8e8fcc952882ec601e9251885e3646.zip
Even more.
-rwxr-xr-xscripts/bootstrap-abuild351
-rwxr-xr-xscripts/bootstrap-host4
-rwxr-xr-xscripts/bootstrap.sh546
-rw-r--r--[-rwxr-xr-x]scripts/setup-abuild2
4 files changed, 674 insertions, 229 deletions
diff --git a/scripts/bootstrap-abuild b/scripts/bootstrap-abuild
new file mode 100755
index 000000000..4333113a8
--- /dev/null
+++ b/scripts/bootstrap-abuild
@@ -0,0 +1,351 @@
+#!/bin/sh -e
+
+##
+# This script is to be called from 'bootstrap.sh', not sourced.
+# PATH is fully contained. We install to 'MCMTOOLS/sys', which
+# will add (and may overwrite) in some cases. This process is
+# destructive. We do this so that we may easily 'chroot' later.
+#
+HERE="$(dirname $(readlink -f ${0}))";
+DEST=/usr/local;
+
+git config --global http.sslCAInfo "${CURL_CA_BUNDLE}";
+
+mkdir -p "${DEST}";
+cd "${DEST}"; # this directory will already exist if correct
+
+if false; then # provided by latest 'bootstrap'
+##
+# Perl
+#
+nprl=perl;
+vprl=5.36.0;
+test ! -f ._${vprl}-${vprl} && \
+(
+ test ! -d ${nprl}-${vprl} \
+ && curl -s https://www.cpan.org/src/${vprl%%.*}.0/perl-${vprl}.tar.gz \
+ | tar -xzf - \
+ ;
+ cd ${nprl}-${vprl};
+ rm -fr x; mkdir x; cd x;
+ ../Configure -des;
+ make -j$(nproc);
+ make install;
+)
+touch ._${nprl}-${vprl};
+rm -fr ${nprl}-${vprl};
+fi
+
+##
+# musl
+#
+# This provides the dynamic loader for the foreign (target) arch
+# so that we do not have to force building static musl binaries.
+#
+nmus=musl;
+vmus=1.2.3;
+test ! -f ._${nmus}-${vmus} && \
+(
+ test ! -d ${nmus}-${vmus} \
+ && curl -s https://musl.libc.org/releases/${nmus}-${vmus}.tar.gz \
+ | tar -xzf - \
+ ;
+ cd ${nmus}-${vmus};
+ rm -fr x; mkdir x; cd x;
+ ../configure \
+ --prefix="${DEST}" \
+ --enable-static \
+ --enable-shared \
+ ;
+ make -j$(nproc);
+ make install;
+)
+touch ._${nmus}-${vmus};
+rm -fr ${nmus}-${vmus};
+
+
+##
+# OpenSSL
+#
+nssl=openssl;
+vssl=1.1.1s;
+test ! -f ._${nssl}-${vssl} && \
+(
+ test ! -d ${nssl}-${vssl} \
+ && curl -s https://www.openssl.org/source/${nssl}-${vssl}.tar.gz \
+ | tar -xzf - \
+ ;
+ cd ${nssl}-${vssl};
+ rm -fr x; mkdir x; cd x;
+ ../Configure cc \
+ --prefix="${DEST}" \
+ --openssldir="${DEST}" \
+ no-shared \
+ ;
+ make -j$(nproc);
+ make install_sw install_ssldirs;
+)
+touch ._${nssl}-${vssl};
+rm -fr ${nssl}-${vssl};
+
+
+##
+# zlib
+#
+nzlb=zlib;
+vzlb=1.2.13;
+test ! -f ._${nzlb}-${vzlb} && \
+(
+ test ! -d ${nzlb}-${vzlb} \
+ && curl -s https://www.zlib.net/${nzlb}-${vzlb}.tar.gz \
+ | tar -xzf - \
+ ;
+ cd ${nzlb}-${vzlb};
+ rm -fr x; mkdir x; cd x;
+ ../configure \
+ --prefix="${DEST}" \
+ --static \
+ ;
+ make -j$(nproc);
+ make install;
+)
+touch ._${nzlb}-${vzlb};
+rm -fr ${nzlb}-${vzlb};
+
+
+##
+# abuild
+#
+nbld=abuild;
+vbld=ee13f777d56e8f3e8848fec9a422430a66292cc1;
+test ! -f ._${nbld}-${vbld} && \
+(
+ test ! -d ${nbld}-${vbld} \
+ && git clone https://git.alpinelinux.org/${nbld} ${nbld}-${vbld} \
+ ;
+ cd abuild-${vbld};
+ git checkout ${vbld};
+# patch -p1 --forward < "${HERE}/patches/0001-allow-untrusted.diff" || true; # FIXME
+# patch -p1 --forward < "${HERE}/patches/0001-etc-apk-keys.diff" || true; # FIXME
+# patch -p1 --forward < "${HERE}/patches/0001-extra-lib-paths.diff" || true; # FIXME
+ rm -fr x; mkdir x; cd x;
+ export SSL_CFLAGS="-I${DEST}/include";
+ export SSL_LDFLAGS="-L${DEST}/lib";
+ export SSL_LIBS="-lssl -lcrypto"; # not in mcmtools
+ export ZLIB_LIBS="-lz"; # do not use from mcmtools
+ export LDFLAGS="-L${DEST}/lib -lssl -lcrypto";
+ export CFLAGS="-DABUILD_GROUP=\\\"$(id -gn)\\\" -static -I${DEST}/include"; # default 'abuild' if undefined
+ sed -i "${DEST}/abuild-${vbld}/abuild-sudo.c" \
+ -e "s@/sbin/apk@${DEST}/bin/apk@" \
+ ; # hardcoded
+ make -j$(nproc) -C .. install \
+ prefix="${DEST}" \
+ sysconfdir="${DEST}" \
+ SCDOC=true \
+ ;
+ sed -i "${DEST}/bin/abuild" \
+ -e 's@/bin/ash -e@/usr/bin/env bash@' \
+ ; # hardcoded
+)
+touch ._${nbld}-${vbld};
+rm -fr ${nbld}-${vbld};
+
+
+##
+# util-linux (for 'getopt' used by 'abuild-keygen')
+#
+nutl=util-linux;
+vutl=08431acdf5b3accd0887ab550bfa4efabed751d6;
+test ! -f ._${nutl}-${vutl} && \
+(
+ test ! -d ${nutl}-${vutl} \
+ && mkdir ${nutl}-${vutl} \
+ && git clone https://github.com/karelzak/${nutl}.git ${nutl}-${vutl} \
+ ;
+ cd ${nutl}-${vutl};
+ git checkout ${vutl};
+ test -f configure || ./autogen.sh;
+ rm -fr x; mkdir x; cd x;
+ ../configure \
+ --prefix="${DEST}" \
+ --host="$(${CC} -dumpmachine)" \
+ --enable-static \
+ --disable-shared \
+ ;
+ sed -i Makefile \
+ -e 's/chgrp/-chgrp/g' \
+ -e 's/chmod/-chmod/g' \
+ -e 's/chown/-chown/g' \
+ ; # allow non-root installation
+ make -j$(nproc) install;
+)
+touch ._${nutl}-${vutl};
+rm -fr ${nutl}-${vutl};
+
+
+##
+# pkgconf (pkg-config replacement)
+#
+npkg=pkgconf;
+vpkg=623b8f7851648a5c476de904a8ffed7b7b679aab; # until autoconf 2.71
+test ! -f ._${npkg}-${vpkg} &&
+(
+ test ! -d ${npkg}-${vpkg} \
+ && git clone https://github.com/${npkg}/${npkg}.git ${npkg}-${vpkg} \
+ ;
+ cd ${npkg}-${vpkg};
+ git checkout ${vpkg};
+ ./autogen.sh;
+ ./configure \
+ --prefix="${DEST}" \
+ --host="$(${CC} -dumpmachine)" \
+ --enable-static \
+ --disable-shared \
+ --with-system-libdir=/lib:/usr/lib \
+ --with-system-includedir=/usr/include \
+ ;
+ make -j$(nproc) install;
+)
+touch ._${npkg}-${vpkg};
+rm -fr ${npkg}-${vpkg};
+
+
+##
+# samurai (ninja replacement)
+#
+nsam=samurai;
+vsam=4cc8f4a3654b72c977e0e62367d1ab6d5b873def;
+test ! -f ._${nsam}-${vsam} &&
+(
+ test ! -d ${nsam}-${vsam} \
+ && git clone https://github.com/michaelforney/${nsam}.git ${nsam}-${vsam} \
+ ;
+ cd ${nsam}-${vsam};
+ make -j$(nproc) install \
+ PREFIX="" \
+ DESTDIR="${DEST}" \
+ ;
+
+)
+touch ._${nsam}-${vsam};
+rm -fr ${nsam}-${vsam};
+
+
+##
+# muon (meson replacement)
+#
+nmun=muon;
+vmun=62ce4561b4444e5020dc39aad0381655afeda0d6;
+test ! -f ._${nmun}-${vmun} &&
+(
+ test ! -d ${nmun}-${vmun} \
+ && git clone https://git.sr.ht/~lattis/${nmun} ${nmun}-${vmun} \
+ ;
+ cd ${nmun}-${vmun};
+ sed -i bootstrap.sh \
+ -e 's/if.*then/if false; then/g' \
+ ;
+ ./bootstrap.sh \
+ bootstrap \
+ ;
+ bootstrap/muon setup \
+ -Dstatic=true \
+ build \
+ ;
+ sed -i build/build.ninja \
+ -e "s@\bar\b@${AR}@g" \
+ ;
+ samu -C build;
+ cp build/muon "${DEST}/bin";
+)
+touch ._${nmun}-${vmun};
+rm -fr ${nmun}-${vmun};
+
+
+##
+# apk-tools
+#
+# Ariadne says use meson, and 'muon' doesn't work so... kludges!
+# Also, '-j' will break the build.
+#
+natl=apk-tools;
+vatl=be4ce40797af9056c79be4dc74ff978f1f4957e4;
+test ! -f ._${natl}-${vatl} && \
+(
+ test ! -d ${natl}-${vatl} \
+ && git clone https://git.alpinelinux.org/${natl} ${natl}-${vatl} \
+ ;
+ cd ${natl}-${vatl};
+ sed -i Make.rules \
+ -e '/targets += $(__shlibs) $(shobjs)/d' \
+ ; # disable shared libs
+ sed -i src/Makefile \
+ -e 's/$(install-libapk_so)//g' -e 's/$(libapk_so)//g' \
+ -e 's/ version.o/ version.o strlcpy.o/' \
+ ; # disable shared libs, hack the hack
+ sed -i src/context.c \
+ -e "s@var/log@${DEST}/${1}/var/log@" \
+ ; # hardcoded
+ ln -sf ../portability/strlcpy.c src/strlcpy.c;
+ export LUA=no; # documentation requires lua
+ make clean;
+ make \
+ INSTALLDIR="${DEST}" \
+ CFLAGS="-I${DEST}/include -DNEED_STRLCPY -Wno-error" \
+ LDFLAGS="-L${DEST}/lib -L${DEST}/${natl}-${vatl}/libfetch" \
+ LIBS="-lapk -lfetch -lssl -lcrypto -lz" \
+ ;
+ cp src/apk "${DEST}/bin";
+)
+touch ._${natl}-${vatl};
+rm -fr ${natl}-${vatl};
+
+
+##
+# pax-utils
+#
+npax=pax-utils;
+vpax=974b9359c2f89d57e69598572aafcd8f920d79e2;
+test ! -f ._${npax}-${vpax} && \
+(
+ test ! -d ${npax}-${vpax} \
+ && git clone https://anongit.gentoo.org/git/proj/${npax}.git ${npax}-${vpax} \
+ ;
+ cd ${npax}-${vpax};
+ git checkout ${vpax};
+ muon setup build;
+)
+touch ._${npax}-${vpax};
+rm -fr ${npax}-${vpax};
+
+
+##
+# fakeroot
+#
+nfrt=fakeroot;
+vfrt=8c0260009e85264fd1ea282fbb22063fc694c552; # until autoconf 2.71
+test ! -f ._${nfrt}-${vfrt#*:} && \
+(
+ test ! -d ${nfrt}-${vfrt} \
+ && git clone https://salsa.debian.org/clint/${nfrt}.git ${nfrt}-${vfrt} \
+ ;
+ cd ${nfrt}-${vfrt};
+ git checkout ${vfrt};
+ ./bootstrap;
+ f=$(mktemp); # needed due to "error: unknown type name 'cap_user_header_t'"
+ echo > ${f} "#include <linux/capability.h>";
+ cat libfakeroot.c >> ${f};
+ mv ${f} libfakeroot.c;
+ rm -fr x; mkdir x; cd x;
+ # -D_ID_T is for "error: conflicting types for 'id_t'; have 'int'"
+ CFLAGS="-D_ID_T" \
+ ../configure \
+ --prefix="${DEST}" \
+ ;
+ sed -i Makefile \
+ -e '/^SUBDIRS =/ s/doc//g' \
+ ;
+ make -j$(nproc) install;
+)
+touch ._${nfrt}-${vfrt#*:};
+rm -fr ${nfrt}-${vfrt#*:};
diff --git a/scripts/bootstrap-host b/scripts/bootstrap-host
new file mode 100755
index 000000000..7619f68a9
--- /dev/null
+++ b/scripts/bootstrap-host
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+cd "${BASE}"/bootstrap;
+./bootstrap;
diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh
index d5334b11b..21156c5ad 100755
--- a/scripts/bootstrap.sh
+++ b/scripts/bootstrap.sh
@@ -1,6 +1,14 @@
#!/bin/sh -e
#===============================================================
+# Filename : scripts/bootstrap.sh (part of Adélie package repo).
+# Purpose : Bootstraps Adélie from source for any architecture.
+# Authors : Zach van Rijn <me@zv.io>
+# License : MIT
+# Revision : 20221206
+#===============================================================
+
+#===============================================================
# README
#===============================================================
#
@@ -11,10 +19,109 @@
# contains standard system utilities, this script bootstraps the
# Adélie Linux distribution for any suported target architecture
# (assuming that musl, gcc, etc. have been ported to it) without
-# requiring 'root' privileges. The procedure is outlined below:
+# requiring 'root' privileges. This process is slow* and costly,
+# but demonstrates that bootstrapping from source is possible.
+#
+# Effort has been focused on correctness instead of performance.
+# This means, some files can be deleted sooner and some of the
+# binaries can run natively on the build machine. We know and do
+# appreciate there are different opinions on how to approach the
+# bootstrap process. This meets our needs and hopefully offers a
+# different perspective.
+#
+# This process takes up a lot of TIME and SPACE, and this cannot
+# be improved by throwing hardware at it. This is primarily due
+# to the tradeoff of not requiring root privileges at any point,
+# and the decision to emulate a native environment instead of to
+# force explicit cross-compilation at each step in the process.
+#
+# (*) See the "requirements" section for mitigations/discussion.
+#
+#
+# features
+# --------
+#
+# * One-click bootstrap. Just start the script and walk away.
+#
+# $ ./scripts/bootstrap.sh ppc64 /path/to/new/scratch/dir
+#
+# * Can be done without root privileges: no 'chroot(8)'.
+#
+# * This is the real deal. No "seed" binaries are required to
+# go from start to finish. The final output is what we use
+# to set up our official build servers totally* from source.
+#
+# * Can be done on a wide range of platforms, almost certainly
+# those supported by mcmtools (want to contribute???).
+#
+# * Minimal dependencies. Nothing fancy. Shell scripts only.
+#
+# * Can be adapted to assist with porting Adélie to new target
+# architectures. TODO: write a tool to automatically do it.
+#
+# (*) We don't count binaries in your starting environment, and
+# recommend https://www.gnu.org/software/mes/ for the brave.
+#
+#
+# requirements
+# ------------
+#
+# TL;DR: You must be able to run the mcmtools bootstrap script.
+#
+# mcmtools is a hard dependency for our bootstrap process now:
+#
+# https://git.zv.io/toolchains/bootstrap
+#
+# It is a simple analog to Buildroot or Yocto. Those tools could
+# be used, too, and would provide the "host rootfs" environment.
+#
+# Internet access is required, but (as an exercise left to the
+# reader) it is possible to pre-download all required sources if
+# you provide your own rootfs for the static QEMU build process.
+#
+# If you are in a position to use native hardware, then you are
+# able to get away with only the final stages of bootstrapping.
+# To do this, you'd essentially comment out the first stages or
+# copy the results of the first stages elsewhere and continue.
+#
+# Other requirements that you should be aware of, estimated:
+#
+# * As many CPU cores as you can throw at it;
+#
+# * ~15 GB for each rootfs ("seed" and "host"), so ~30GB;
+#
+# * ~ 3 GB for toolchains;
+#
+# * ~ 2 GB for the "system/" package repository, when built;
+#
+# * Please refer to the README in 'bootstrap', linked above,
+# for more information about performance. Most of this can
+# be gained back if you adapt this script to use 'chroot(8)'
+# instead of 'PRoot', at the expense of requiring privilege,
+# with the correct registration of QEMU with 'binfmt_misc'.
+#
+# In brief, there is an approximate factor of 13 slowdown on
+# practical workloads when using QEMU user and 'PRoot' to do
+# dynamic binary translation and emulate a foreign machine.
+#
+#
+# process
+# -------
+#
+# The illustration below outlines the complete bootstrap process
+# and roughly corresponds to the script layout/organization. Our
+# terminology is not perfectly consistent; please excuse this.
+#
+# The term "build" is shorthand for "build-native" CPU, which
+# is the machine on which you are performing the bootstrap.
+#
+# The term "native" is shorthand for "foreign-native" CPU, which
+# is the machine to which you are targeting the bootstrap, and
+# relates to the "host" CPU on which the code will run, but that
+# with the help of dynamic binary translation, runs "natively".
#
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# (unstable, everything provided by user)
+# build (unstable, everything provided by user)
#
# +-----------------+ User-provided tools. Dependency
# | Baseline System | of 'mcmtools', which will verify
@@ -26,7 +133,7 @@
# | host-arch musl-libc toolchain...
# |
# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - -
-# | (stable versions, unstable libc)
+# mixed | (stable versions, unstable libc)
# |
# +-------------+ ...and a 'chroot'-able rootfs. A
# | seed rootfs | sane, but not clean, environment
@@ -36,51 +143,101 @@
# | emulators | dependencies, as well as static
# +-----------+ QEMU user binaries. Add to seed.
# |
-# +---------------+ Static musl libc cross toolchain
-# | musl cross tc | targeting a given architecture.
-# +---------------+ Output binaries will run via the
+# +-----------------+ Static musl toolchains targeting
+# | musl toolchains | a given architecture: cross from
+# +-----------------+ the host CPU & foreign "native".
+# | Output binaries will run via the
# | 'binfmt_misc' mechanism + QEMU.
# |
# +-------------+ Script to build Alpine Package
# | build tools | Keeper (APK) and dependencies.
# +-------------+ All binaries are cross-compiled!
+# | This step is a sanity check.
+# |
+# +----------+ Script to build pinned versions
+# | mcmtools | of common system utilities. This
+# +----------+ time, all utilities are target-
+# | native. Build more toolchains...
+# |
+# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - -
+# host | (stable versions, stable libc)
# |
+# +-------------+ ...and a 'chroot'-able rootfs. A
+# | host rootfs | sane, clean, foreign "native"
+# +-------------+ rootfs that requires QEMU, or is
+# | able to run on native hardware.
+# |
+# +-------------+ Script to build Alpine Package
+# | build tools | Keeper (APK) and dependencies.
+# +-------------+ These binaries are native built!
+# | This step is required!
+# |
+# +-------------+ Script to build the Adélie Linux
+# | system repo | "system/" package repository. It
+# +-------------+ is used to build core packages.
+# |
+# +---------------+ Script to install packages into
+# | image creator | a clean rootfs. Carryover from
+# +---------------+ the mcmtools process is removed.
+# |
+# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - -
+# verify | (optional verification)
+# |
+# ... The "host" stage can be repeated
+# | zero or more times to ensure the
+# | final image is not contaminated.
+# |
+# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - -
+# output | (Welcome to Adélie Linux)
# |
-# +-------------+ Script to build the minimum set
-# | seed rootfs | of tools to start building the
-# +-------------+ "system/" package repository.
+# +---------------+ This is a minimal Adélie rootfs.
+# | adelie rootfs | Copy it to native hardware, then
+# +---------------+ use it as a builder 'chroot'. It
+# is used to seed official Adélie
+# build boxen (e.g. autobuilder).
#
-# mcmtools; a separate script
-# that
-# mcmtools provides almost everything needed to build 'abuild',
-# and it is a hard dependency for our bootstrap process now.
+# limitations
+# -----------
#
-# https://git.zv.io/toolchains/bootstrap
+# The builds are not hermetically sealed. That is not the point.
+# The build environment should not *need* anything from outside,
+# but it is not *prevented* from accessing anything. You should
+# run this on a clean, trusted machine.
+#
+# There is no guarantee of byte-for-byte reproducible builds at
+# this time. This is, in part, due to timestamps and tar headers
+# but may involve a lack of determinism in parallel builds.
#
-# The output of this step is a host-native toolchain with fixed
-# versions of development utilities, and a host-architecture hosted toolchain that cross-compiles to both musl and/or
-# a foreign CPU architecture. (Table above prevents this).
+# QEMU user emulation may cause subtle incompatibilites with the
+# target CPU hardware. For example, CPU feature (mis)detection.
#
+# It is not currently possible to safely reuse any of the output
+# if the target architecture is changed. You will need to start
+# from scratch if you wish to change the target. This is a TODO.
+#
HERE="$(dirname $(readlink -f ${0}))";
-TEMP="$(mktemp -d)"; # do not change if not expert!
+
#---------------------------------------------------------------
# initialization
+##
+# Haaaalp!
+#
usage ()
{
- printf "Usage: %s ARCH\n\n" "${0}";
+ printf "Usage: %s ARCH BASE\n\n" "${0}";
cat <<EOF
- ARCH is { aarch64, armv7, ppc64, ppc, x86_64, pmmx }
+ ARCH { aarch64, armv7, ppc64, ppc, x86_64, pmmx }
-Optional environment variables:
-
- MCMTOOLS=/path/to/existing/mcmtools/ **MAY CAUSE DATA LOSS**
+ BASE an absolute path to bootstrap out of.
+ ** MAY CAUSE DATA LOSS IF SET INCORRECTLY! **
EOF
exit 0;
}
+
##
# argv[1]: ARCH
#
@@ -91,19 +248,35 @@ EOF
# ARCH is translated to canonical GCC and QEMU machine types.
#
case "${1}" in
-# adelie gcc qemu
-# ------ --- ----
- aarch64) m=aarch64; q=aarch64; ;;
- armv7) m=armv7l ; q=arm ; ;;
- ppc) m=ppc ; q=ppc ; ;;
- ppc64) m=ppc64 ; q=ppc64 ; ;;
- x86_64) m=x86_64 ; q=x86_64 ; ;;
- pmmx) m=i686 ; q=i386 ; ;;
+# adelie gcc qemu
+# ------ --- ----
+ aarch64) m=aarch64 ; q=aarch64 ; ;;
+ armv7) m=armv7l ; q=arm ; ;;
+ ppc) m=powerpc ; q=ppc ; ;;
+ ppc64) m=powerpc64 ; q=ppc64 ; ;;
+ x86_64) m=x86_64 ; q=x86_64 ; ;;
+ pmmx) m=i586 ; q=i386 ; ;;
*) usage ;;
esac
shift;
-test ! -n "${m}" && printf "Invoking '%s TARGET' where 'TARGET=%s' is not valid.\n" "${0}" "${m}" && exit 1;
+
+##
+# argv[2]: BASE
+#
+# BASE is a semi-permanent scratch directory. It is where all of
+# the magic happens, and probably cannot be relocated easily. Be
+# careful to not set it incorrectly or to a place you'd regret
+# being overwritten, corrupted, or deleted.
+#
+case "${1}" in
+ /*) BASE="${1}";
+ ;;
+ *) printf "BASE not set, or not an absolute path!\n";
+ exit 1;
+ ;;
+esac
+
##
# Internal variables. Do not modify this section directly.
@@ -123,12 +296,17 @@ printf "BUILDS=%s\n" "${BUILDS}";
TARGET="${m}${SYSTEM}"; # ultimate Adélie Linux target
printf "TARGET=%s\n" "${TARGET}";
-MTOOLS=${MCMTOOLS:-"${TEMP}/mcmtools"}; # CAREFUL! MAY CAUSE DATA LOSS!
+MTOOLS=${MCMTOOLS:-"${BASE}/mcmtools"}; # CAREFUL! MAY CAUSE DATA LOSS!
printf "MTOOLS=%s\n" "${MTOOLS}";
+##
+# Default 'PATH' for use inside various rootfs environments.
+#
+DEF_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
+
#---------------------------------------------------------------
-# mcmtools
+# mcmtools (seed)
##
# Allow the user to supply an existing mcmtools installation. It
@@ -138,19 +316,22 @@ printf "MTOOLS=%s\n" "${MTOOLS}";
# is used if this environment variable is omitted. Another case
# for providing a custom value is if '/tmp' is mounted weird.
#
-if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code
- cd "${TEMP}";
+if ! test -d "${MTOOLS}/sys/emus/bin"; then # FIXME: no hard code
+ cd "${BASE}";
test -d bootstrap \
|| git clone ${CHAINS}/bootstrap.git;
cd bootstrap;
- git checkout 773363265da441f472c207fb59c1acecc8ba2eb4;
+ git checkout d148555321a391fd6f23a289fa51c2dfa26f4b10;
## seed rootfs
#
# This will build absolutely everything that is needed to be
# self-reliant, except for some build deps for QEMU.
#
+ # We copy 'config.mak' from mcmtools bootstrap to the rootfs
+ # so that when we build "real" toolchain they are the same.
+ #
test -d "${MTOOLS}" || \
DEST="${MTOOLS}" \
ARCH=${BUILDS} \
@@ -160,9 +341,19 @@ if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code
cp "${MTOOLS}"/tmp/musl-cross-make/config.mak \
"${MTOOLS}"/config.mak \
;
-
- ##
- # emulators
+ #rm -fr "${MTOOLS}"/tmp; # save 10 GB (FIXME: make safe)
+
+ # is any of this actually needed?
+ (
+ cd "${MTOOLS}"/sys;
+ mkdir -p dev;
+ mkdir -p proc;
+ mkdir -p sys;
+ rm -fr usr;
+ ln -s . usr;
+ )
+
+ ## emulators
#
# Dependencies are built with the mcmtools host toolchain; a
# reason to not force musl here is in the event that these
@@ -174,7 +365,7 @@ if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code
# manually provide your own static QEMU user binaries and be
# on your way without Alpine, but it is a good 'PRoot' test.
#
- test -d "${MTOOLS}/emus/bin" || \
+ test -d "${MTOOLS}/sys/emus/bin" || \
PATH="${MTOOLS}/host/bin:${MTOOLS}/sys/bin" \
DEST="${MTOOLS}" \
./prootemu \
@@ -182,225 +373,124 @@ if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code
fi
+#---------------------------------------------------------------
+# musl toolchains
+
##
-# Now we have a musl-targeting toolchain that runs on the host.
-# We need to build the same toolchain, but static.
+# We have a musl-targeting toolchain that runs on the host, but
+# it is linked to the host libc and may not run in this chroot.
+#
+# We need to build the same toolchain, but static. There are two
+# possible directions to go: (1) a cross toolchain that runs at
+# full speed on the build machine, at the cost of having to tell
+# downstream build scripts how to cross compile, or (2) a native
+# toolchain for the foreign (target) architecture that runs slow
+# in QEMU and still requires the cross toolchain to build it.
#
-# This is required for it to run inside 'PRoot' if it differs in
-# architecture.
+# Build both. We will eventually need both toolchains, anyway.
#
-if ! test -d "${MTOOLS}/sys/tc/cross"; then # FIXME: no hard code
- cd "${TEMP}";
+if ! test -d "${MTOOLS}/sys/tc/native"; then # FIXME: no hard code
+ cd "${BASE}";
test -d musl-cross-make \
|| git clone ${CHAINS}/musl-cross-make.git;
cd musl-cross-make;
git pull; # always use the latest
+ ##
+ # Ensure consistent 'config.mak' for all toolchain builds.
+ #
cp "${MTOOLS}"/config.mak config.mak;
- ## musl cross tc
+ ## musl toolchains
#
- # Build this toolchain statically using the musl toolchain
+ # Build these toolchains statically using the musl toolchain
# from the seed rootfs so that it is known to work correctly
# (the original musl toolchain itself may itself be linked
# with glibc or be unsafe to use in some contexts).
#
- PATH="${MTOOLS}/musl/bin:${MTOOLS}/sys/bin" \
- ./scripts/buildcross ${TARGET} \
- ;
- cd output;
- mkdir -p "${MTOOLS}"/sys/tc/cross;
- tar -C "${MTOOLS}"/sys/tc/cross \
- --strip-components=1 \
- -xzf ${TARGET}-cross.tgz \
+ # Note: "native" is for the foreign target CPU architecture.
+ #
+ PATH="${MTOOLS}/musl/bin:${MTOOLS}/sys/bin" \
+ ./scripts/build ${TARGET} \
;
+ for k in cross native; do
+ rm -fr "${MTOOLS}"/sys/tc/${k};
+ mkdir "${MTOOLS}"/sys/tc/${k};
+ tar -C "${MTOOLS}"/sys/tc/${k} \
+ --strip-components=1 \
+ -xzf output/${TARGET}-${k}.tgz \
+ ;
+ done
fi
-##
-# Finish setting up the rootfs to support what we're doing.
-#
-# FIXME: what of this is actually needed?
-#
-(
- cd "${MTOOLS}"/sys;
- mkdir -p dev;
- mkdir -p proc;
- mkdir -p sys;
- rm -fr usr;
- ln -s . usr;
-)
+#---------------------------------------------------------------
+# build tools (cross)
##
# Build 'abuild', its dependencies, and other utilities.
-# Once finished, add them to PATH.
-#
-
-PROOT_NO_SECCOMP=1 \
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tc/cross/bin \
-CROSS_COMPILE=${TARGET}- \
-SHELL=/bin/sh \
-DEST=/usr/local \
-CURL_CA_BUNDLE=/cacert.pem \
-${MTOOLS}/sys/emus/bin/proot \
- -S "${MTOOLS}"/sys \
- -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \
- -b "${HERE}" \
- "${HERE}"/setup-abuild \
+#
+PROOT_NO_SECCOMP=1 \
+PATH="/tc/cross/bin:${DEF_PATH}" \
+SHELL=/bin/sh \
+DEST=/usr/local \
+CURL_CA_BUNDLE=/cacert.pem \
+CROSS_COMPILE=${TARGET}- \
+CC=${TARGET}-gcc \
+CXX=${TARGET}-g++ \
+AR=${TARGET}-ar \
+LD=${TARGET}-ld \
+CPP=${TARGET}-cpp \
+${MTOOLS}/sys/emus/bin/proot \
+ -S "${MTOOLS}"/sys \
+ -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \
+ -b "${HERE}" \
+ "${HERE}"/bootstrap-abuild \
;
-echo ok
-exit 0;
-
-export PATH="${MTOOLS}/abuild/bin:${PATH}";
-bash
-
-##
-# Additional configuration.
-#
-export PACKAGER="Zach van Rijn <me@zv.io>";
-export CBUILD=${TARGET_ARCH};
-export CBUILDROOT="${MCMTOOLS}/abuild/${TARGET_ARCH}";
-export ABUILD_USERDIR="${CBUILDROOT}.conf";
-export SRCDEST="${MCMTOOLS}/abuild/src";
-export REPODEST="${MCMTOOLS}/abuild/apk";
-export ABUILD_APK_INDEX_OPTS="--allow-untrusted"; # FIXME
-#export BUILD_ROOT="${CBUILDROOT}";
-
-SUDO_APK=abuild-apk
-
-# get abuild configurables
-[ -e "${MCMTOOLS}/abuild/share/abuild/functions.sh" ] || (echo "abuild not found" ; exit 1)
-CBUILDROOT="$(CTARGET=$TARGET_ARCH . ${MCMTOOLS}/abuild/share/abuild/functions.sh ; echo $CBUILDROOT)"
-. "${MCMTOOLS}/abuild/share/abuild/functions.sh"
-[ -z "$CBUILD_ARCH" ] && die "abuild is too old (use 2.29.0 or later)"
-[ -z "$CBUILDROOT" ] && die "CBUILDROOT not set for $TARGET_ARCH"
-
-# deduce aports directory
-[ -z "$APORTS" ] && APORTS=$(realpath $(dirname $0)/../)
-[ -e "$APORTS/system/build-tools" ] || die "Unable to deduce packages directory"
-
-apkbuildname() {
- local repo="${1%%/*}"
- local pkg="${1##*/}"
- [ "$repo" = "$1" ] && repo="system"
- echo $APORTS/$repo/$pkg/APKBUILD
-}
-
-msg() {
- [ -n "$quiet" ] && return 0
- local prompt="$GREEN>>>${NORMAL}"
- local name="${BLUE}bootstrap-${TARGET_ARCH}${NORMAL}"
- printf "${prompt} ${name}: %s\n" "$1" >&2
-}
-
-if [ -z "$TARGET_ARCH" ]; then
- program=$(basename $0)
- cat <<EOF
-usage: $program TARGET_ARCH
-
-This script creates a local cross-compiler, and uses it to
-cross-compile an Adélie Linux base system for new architecture.
-
-Steps for introducing new architecture include:
-- adding the compiler tripler and arch type to abuild
-- adding the arch type detection to apk-tools
-- adjusting build rules for packages that are arch aware:
- gcc, musl, binutils, easy-kernel
-- create new kernel config for easy-kernel
-
-After these steps the initial cross-build can be completed
-by running this with the target arch as parameter, e.g.:
- ./$program aarch64
-
-EOF
- return 1
-fi
+#---------------------------------------------------------------
+# mcmtools (host)
##
-# Package signing keys. Public and Private keys are stored in a
-# different location; variables for which are installed to arch-
-# specific 'abuild.conf' file.
-#
-if [ ! -d "$CBUILDROOT/etc/apk/keys" ] || [ -n "$(find $CBUILDROOT -type f -name '*.rsa')" ]; then
- msg "Creating sysroot in $CBUILDROOT"
- mkdir -p "$CBUILDROOT/etc/apk/keys"
- abuild-keygen -an;
- p=$(find "${ABUILD_USERDIR}" -type f -name "*.rsa.pub");
- mv "${p}" "$CBUILDROOT/etc/apk/keys";
- grep 1>/dev/null PACKAGER_PUBKEY= "${ABUILD_USERDIR}/abuild.conf" || printf >> "${ABUILD_USERDIR}/abuild.conf" "PACKAGER_PUBKEY=\"%s\"\n" "$CBUILDROOT/etc/apk/keys/${p##*/}";
-fi
+# Build the 'system/' repository.
+#
+# NOTE: The 'PATH' order is *really* important. Foreign "native"
+# toolchains must be the first ones found; e.g. '/usr/bin/gcc'
+# is a symlink to 'ccache' and isn't a functional compiler.
+#
+# Once 'coreutils' is built, 'uname' will return correctly, then
+# future software should(tm) build as if it is built natively.
+#
+# FIXME: is Linux 3.2.0 really appropriate here?
+#
+PROOT_NO_SECCOMP=1 \
+PATH="/tc/native/bin:${DEF_PATH}" \
+SHELL=/bin/sh \
+BASE="${BASE}" \
+DEST="${BASE}"/mcmtools-${TARGET} \
+ARCH=${TARGET} \
+CURL_CA_BUNDLE=/cacert.pem \
+${MTOOLS}/sys/emus/bin/proot \
+ -S "${MTOOLS}"/sys \
+ -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \
+ -b "${HERE}" \
+ -k "3.2.0" \
+ "${HERE}"/bootstrap-host \
+ ;
-##
-# APK database.
-#
-if [ ! -f "${CBUILDROOT}/._database-${TARGET_ARCH}" ]; then
- mkdir -p "${CBUILDROOT}/var/log"; # why not created by default?
- ${SUDO_APK} add --quiet --initdb --arch $TARGET_ARCH --root $CBUILDROOT
- touch "${CBUILDROOT}/._database-${TARGET_ARCH}";
-fi
+#---------------------------------------------------------------
+# system repository
-msg "Building cross-compiler"
+# TODO
-# Build and install cross binutils (--with-sysroot)
-CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname binutils) abuild -r
-exit
-if ! CHOST=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname musl) abuild up2date 2>/dev/null; then
- # C-library headers for target
- CHOST=$TARGET_ARCH BOOTSTRAP=nocc APKBUILD=$(apkbuildname musl) abuild -r
- # Minimal cross GCC
- EXTRADEPENDS_HOST="musl-dev" \
- CTARGET=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname gcc) abuild -r
+#---------------------------------------------------------------
+# image creator
- # Cross build bootstrap C-library for the target
- EXTRADEPENDS_BUILD="gcc-pass2-$TARGET_ARCH" \
- CHOST=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname musl) abuild -r
-fi
+# TODO
-# Full cross GCC
-EXTRADEPENDS_TARGET="musl musl-dev" \
-CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname gcc) abuild -r
-
-# Cross build-base
-CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname build-base) abuild -r
-
-msg "Cross building base system"
-msg "Change your abuild.conf NOW to avoid build errors!!"
-read
-
-# add implicit target prerequisite packages
-apk info --quiet --installed --root "$CBUILDROOT" libgcc libstdc++ musl-dev || \
- ${SUDO_APK} --root "$CBUILDROOT" add --repository "$REPODEST/system" libgcc libstdc++ musl-dev
-
-# ordered cross-build
-for PKG in musl pkgconf zlib \
- gettext-tiny ncurses bash binutils make bison flex m4 \
- openssl apk-tools linux-pam shadow \
- gmp mpfr3 mpc1 isl gcc ca-certificates \
- openrc libcap-ng coreutils sed gzip bzip2 diffutils \
- attr libcap patch sudo acl fakeroot libarchive mawk \
- pax-utils abuild grep findutils patch lzip unzip autoconf automake libtool \
- ncurses util-linux lvm2 popt xz lddtree libssh2 curl build-tools pcre \
- debianutils file shimmy procps zsh sharutils net-tools check kbd sysklogd vim db groff libpipeline man-db psmisc less \
- adelie-base \
- ; do
-
- CHOST=$TARGET_ARCH BOOTSTRAP=bootimage APKBUILD=$(apkbuildname $PKG) abuild -r
-
- case "$PKG" in
- fortify-headers | libc-dev | build-base)
- # headers packages which are implicit but mandatory dependency
- apk info --quiet --installed --root "$CBUILDROOT" $PKG || \
- ${SUDO_APK} --update --root "$CBUILDROOT" --repository "$REPODEST/system" add $PKG
- ;;
- musl | gcc)
- # target libraries rebuilt, force upgrade
- [ "$(apk upgrade --root "$CBUILDROOT" --repository "$REPODEST/main" --available --simulate | wc -l)" -gt 1 ] &&
- ${SUDO_APK} upgrade --root "$CBUILDROOT" --repository "$REPODEST/main" --available
- ;;
- esac
-done
+# apk --root /foo --arch armv7 --initdb add
+# apk --root /foo add adelie-core dash-binsh build-tools
diff --git a/scripts/setup-abuild b/scripts/setup-abuild
index 4333113a8..f35ddd114 100755..100644
--- a/scripts/setup-abuild
+++ b/scripts/setup-abuild
@@ -53,7 +53,7 @@ test ! -f ._${nmus}-${vmus} && \
cd ${nmus}-${vmus};
rm -fr x; mkdir x; cd x;
../configure \
- --prefix="${DEST}" \
+ --prefix=/usr \
--enable-static \
--enable-shared \
;