#!/bin/sh -e #=============================================================== # Filename : bootstrap # Purpose : Bootstraps Adélie from source for any architecture. # Authors : Zach van Rijn # License : MIT # Revision : 20230919 #=============================================================== #=============================================================== # README #=============================================================== # # overview # -------- # # Given a basic development environment ("Baseline System") that # contains standard system utilities, this script bootstraps the # Adélie Linux distribution for any suported target architecture # (assuming that musl, gcc, etc. have been ported to it) WITHOUT # requiring 'root' privileges. This process is slow* and costly, # but demonstrates that bootstrapping from source is possible. # # Effort has been focused on correctness instead of performance. # This means, some files can be deleted sooner and some of the # binaries can run natively on the build machine. We know and do # appreciate there are different opinions on how to approach the # bootstrap process. This meets our needs and hopefully offers a # different perspective on how to simplify cross compilation. # # (*) See the "requirements" section for mitigations/discussion. # # # features # -------- # # * One-click bootstrap. Just start the script and walk away. # # $ ./bootstrap ppc64 /path/to/new/scratch/dir # # * Can be done without root privileges: no 'chroot(8)'. # # * This is the real deal. No "seed" binaries are required to # go from start to finish. The final output is what we use # to set up our official build servers totally* from source. # # * Can be done on a wide range of platforms, almost certainly # those supported by mcmtools (want to contribute???). # # * Minimal dependencies. Nothing fancy. Shell scripts only. # # * Can be adapted to assist with porting Adélie to new target # architectures. TODO: write a tool to automatically do so. # # (*) We don't count binaries in your starting environment, and # recommend https://www.gnu.org/software/mes/ for the brave. # # # design # ------ # # The goal of this script is to support as wide a range of uses # as possible. For example, a full cross-platform bootstrap that # needs to be done without target hardware or virtual machines. # # There are a few different strategies, depending on your needs # and what type of privileges/hardware you have access to. # # In this diagram, indentation refers to an output product, and # moving down vertically refers to an input step. CPU A is the # build architecture, and CPU B is the target architecture. Your # mileage may vary, and you can mix/match other approaches.. # # * CPU A --> CPU B (slowest: no privilege, no CPU B hardware) # --> Bootstrap 1 (Cross Libc + Partial Cross CPU B) # --> CPU A + CPU B Mixed Rootfs # --> Partial Emulation (PRoot) # --> Bootstrap 2 (Full Cross CPU B) # --> CPU B Host Rootfs # --> Full Emulation (PRoot) # --> Adélie Bootstrap # --> Adélie Rootfs for CPU B # # * CPU A --> CPU B (slower: privilege, no CPU B hardware) # --> Bootstrap 1 (Cross Libc + Partial Cross CPU B) # --> CPU A + CPU B Mixed Rootfs # --> Partial Emulation (registered binfmt_misc) # --> Bootstrap 2 (Full Cross CPU B) # --> CPU B Host Rootfs # --> Full Emulation (registered binfmt_misc) # --> Adélie Bootstrap # --> Adélie Rootfs for CPU B # # * CPU A --> CPU B (faster; no privilege, CPU B hardware) # --> Bootstrap 1 (Cross Libc + Partial Cross CPU B) # --> CPU A + CPU B Mixed Rootfs # --> Partial Emulation (PRoot) # --> Bootstrap 2 (Full Cross CPU B) # --> CPU B Host Rootfs # --> Copy to Native CPU B Hardware # --> Adélie Bootstrap # --> Adélie Rootfs for CPU B # # * CPU B --> CPU B (fastest; QEMU-system or native hardware) # --> Bootstrap 1 (Cross Libc) # --> CPU B Mixed Rootfs # --> Bootstrap 2 (Remove Contamination) # --> CPU B Host Rootfs # --> Adélie Bootstrap # --> Adélie Rootfs for CPU B # # The resulting Adélie Rootfs is capable of building the entire # Adélie Linux distribution from source. We use it to seed our # official build servers. This removes the dependency on using # previous binary releases of our distribution, which had been # manually bootstrapped way back from Gentoo and/or on a G5. # # # requirements # ------------ # # TL;DR: You must be able to run the mcmtools bootstrap script. # # Cross builds take up a lot of TIME and SPACE, and this cannot # be improved by throwing hardware at it. This is primarily due # to the tradeoff of not requiring root privileges at any point, # and the decision to emulate a native environment instead of to # force explicit cross-compilation at each step in the process. # # Targeting an architecture of opposite endianness will be even # slower; this is a QEMU limitation. # # If you wish to bootstrap to the same target CPU architecture, # or a compatible mode (e.g. i686 on x86_64), it will be faster. # # mcmtools is a hard dependency for our bootstrap process now: # # https://git.zv.io/toolchains/bootstrap # # It is a simple analog to Buildroot or Yocto. Those tools could # be used, too, and would provide the "host rootfs" environment. # # Internet access is required, but (as an exercise left to the # reader) it is possible to pre-download all required sources if # you provide your own rootfs for the static QEMU build process. # # If you are in a position to use native hardware, then you are # able to get away with only the final stages of bootstrapping. # To do this, you'd essentially comment out the first stages or # copy the results of the first stages elsewhere and continue. # # Other requirements that you should be aware of, estimated: # # * As many CPU cores as you can throw at it; # # * ~15 GB for each mcmtools rootfs ("seed" and "host"); # # * ~ 3 GB for toolchains; # # * ~ 2 GB for each intermediate rootfs; # # * ~ 2 GB for the "system/" package repository, when built; # # * Please refer to the README in 'bootstrap', linked above, # for more information about performance. Most of this can # be gained back if you adapt this script to use 'chroot(8)' # instead of 'PRoot', at the expense of requiring privilege, # with the correct registration of QEMU with 'binfmt_misc'. # # In brief, there is an approximate factor of 13 slowdown on # practical workloads when using QEMU user and 'PRoot' to do # dynamic binary translation and emulate a foreign machine. # # # process # ------- # # The illustration below outlines the complete bootstrap process # and roughly corresponds to the script layout/organization. Our # terminology is not perfectly consistent; please excuse this. # # The term "build" is shorthand for "build-native" CPU, which # is the machine on which you are performing the bootstrap. # # The term "native" is shorthand for "foreign-native" CPU, which # is the machine to which you are targeting the bootstrap, and # relates to the "host" CPU on which the code will run, but that # with the help of dynamic binary translation, runs "natively". # # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # build (unstable, everything provided by user) # # +-----------------+ User-provided tools. Dependency # | Baseline System | of 'mcmtools', which will verify # +-----------------+ that these tools are available. # | # +----------+ Script to build pinned versions # | mcmtools | of common system utilities, a # +----------+ host-arch host-libc toolchain, a # | host-arch musl-libc toolchain... # | # - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - # mixed | (stable versions, unstable libc) # | # +-------------+ ...and a 'chroot'-able rootfs. A # | seed rootfs | sane, but not clean, environment # +-------------+ in which we begin the bootstrap. # | # +-----------+ Script to build 'PRoot' and its # | emulators | dependencies, as well as static # +-----------+ QEMU user binaries. Add to seed. # | # +-----------------+ Static musl toolchains targeting # | musl toolchains | a given architecture: cross from # +-----------------+ the host CPU & foreign "native". # | Output binaries will run via the # | 'binfmt_misc' mechanism + QEMU. # | # +-------------+ Script to build Alpine Package # | build tools | Keeper (APK) and dependencies. # +-------------+ All binaries are cross-compiled! # | This step is a sanity check. # | # +----------+ Script to build pinned versions # | mcmtools | of common system utilities. This # +----------+ time, all utilities are target- # | native. Build more toolchains... # | # - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - # host | (stable versions, stable libc) # | # +-------------+ ...and a 'chroot'-able rootfs. A # | host rootfs | sane, clean, foreign "native" # +-------------+ rootfs that requires QEMU, or is # | able to run on native hardware. # | # +-------------+ Script to build Alpine Package # | build tools | Keeper (APK) and dependencies. # +-------------+ These binaries are native built! # | This step is required! # | # +-------------+ Script to build the Adélie Linux # | system repo | "system/" package repository. It # +-------------+ is used to build core packages. # | # +---------------+ Script to install packages into # | image creator | a clean rootfs. Carryover from # +---------------+ the mcmtools process is removed. # | # - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - # verify | (optional verification) # | # ... The "host" stage can be repeated # | zero or more times to ensure the # | final image is not contaminated. # | # - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - # output | (Welcome to Adélie Linux) # | # +---------------+ This is a minimal Adélie rootfs. # | adelie rootfs | Copy it to native hardware, then # +---------------+ use it as a builder 'chroot'. It # is used to seed official Adélie # build boxen (e.g. autobuilder). # # limitations # ----------- # # The builds are not hermetically sealed. That is not the point. # The build environment should not *need* anything from outside, # but it is not *prevented* from accessing anything. You should # run this on a clean, trusted machine. # # There is no guarantee of byte-for-byte reproducible builds at # this time. This is, in part, due to timestamps and tar headers # but may involve a lack of determinism in parallel builds. # # QEMU user emulation may cause subtle incompatibilites with the # target CPU hardware. For example, CPU feature (mis)detection. # # It is not currently possible to safely reuse any of the output # if the target architecture is changed. You will need to start # from scratch if you wish to change the target. This is a TODO. # # # todo # ---- # # * Do not inherit the build environment; avoid contamination. # # * Finish the "system/" build; the "bootstrap.sh" script will # require some work. Current status: preimage is finished. # HERE="$(dirname $(readlink -f ${0}))"; #--------------------------------------------------------------- # initialization ## # Haaaalp! # usage () { printf "Usage: %s ARCH BASE\n\n" "${0}"; cat < '/' instead # of the empty string) to avoid gotchas. Assumes '//' is the # same as '/'. I can't think of a counterexample. # x="${BASE}"/mcmtools-${TARGET}/sys; grep -rl "${x}" ${t} | while read k; do ./binsub ${k} "${x}" "/"; done tar -C "${t}" \ -pczf rootfs-${TARGET}-patched.tgz \ . \ ; rm -fr "${t}"; fi #--------------------------------------------------------------- # preimage ## # The preimage has some modifications to support using the real # "bootstrap.sh" script. Note that we need to bind-mount and use # a different working directory because PRoot has a few defaults # that would otherwise shadow paths we're trying to create. # cd "${BASE}"; if ! test -f rootfs-${TARGET}-preimage.tgz; then # FIXME: no hard code rm -fr rootfs-${TARGET}-preimage; mkdir rootfs-${TARGET}-preimage; tar -C rootfs-${TARGET}-preimage \ -xzf rootfs-${TARGET}-patched.tgz \ || true; # we're somehow messing up some bzip binaries? PROOT_NO_SECCOMP=1 \ PATH="${DEF_PATH}" \ SHELL=/bin/sh \ CURL_CA_BUNDLE=/cacert.pem \ ${MTOOLS}/sys/emus/bin/proot \ -S "${MTOOLS}"/sys \ -b "${BASE}"/rootfs-${TARGET}-preimage \ -w "${BASE}"/rootfs-${TARGET}-preimage \ -b "${HERE}" \ "${HERE}"/setup-preimage \ ; tar -C rootfs-${TARGET}-preimage \ -pczf rootfs-${TARGET}-preimage.tgz \ . \ ; fi #--------------------------------------------------------------- # packages ## # This is where the original "bootstrap.sh" script runs. The aim # of this step is to produce '.apk' files that were built using # non-Adélie-packaged tools, but would install Adélie tools. # if ! test -f rootfs-${TARGET}-packages.tgz; then # FIXME: no hard code rm -fr rootfs-${TARGET}-packages; mkdir rootfs-${TARGET}-packages; tar -C rootfs-${TARGET}-packages \ -xzf rootfs-${TARGET}-preimage.tgz \ ; PROOT_NO_SECCOMP=1 \ PATH="${DEF_PATH}" \ SHELL=/bin/sh \ TARGET_ARCH=${TARGET} \ CURL_CA_BUNDLE=/cacert.pem \ ${MTOOLS}/sys/emus/bin/proot \ -R "${BASE}"/rootfs-${TARGET}-packages \ -i 1000:300 \ -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \ -b "${HERE}" \ "${HERE}"/setup-packages \ ; fi # TODO: create tarball of this directory #--------------------------------------------------------------- # image creator # TODO # apk --root /foo --arch armv7 --initdb add # apk --root /foo add adelie-core dash-binsh build-tools