diff options
| -rw-r--r-- | binsub.c | 277 | ||||
| -rwxr-xr-x | bootstrap | 763 | ||||
| -rwxr-xr-x | setup-abuild | 362 | ||||
| -rwxr-xr-x | setup-packages | 13 | ||||
| -rwxr-xr-x | setup-preimage | 35 |
5 files changed, 1450 insertions, 0 deletions
diff --git a/binsub.c b/binsub.c new file mode 100644 index 0000000..c66148c --- /dev/null +++ b/binsub.c @@ -0,0 +1,277 @@ +/** + * binsub.c / 2022-12-09 + * + * (C) 2022 Zach van Rijn <me@zv.io> + * + * MIT License + * + * This utility truncates or replaces needles in an input file; + * truncation meaning the replacement string is empty. + * + * Replacement string length must be less than or equal to that + * of the needle because the file length must remain unchanged. + * + * For efficient operation, consider deploying this on a '.tar' + * file instead of individual files within a directory. + */ + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +/** + * Basic memory structure. + */ +struct buffer +{ + char *data; + size_t len; +}; + + +/** + * Given a needle ('find') and optional replacement ('repl'), if + * the needle is found, truncate it, inject the replacement, and + * pad the tail end of the matching string with null bytes. + * + * The file length remains the same, and we hope that nobody is + * relying on precomputed offsets into the strings. Mega kludge! + * + * Replacement = "": + * + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * in |.|.|.|N|E|E|D|L|E|.|.|.|O|T|H|E|R| |D|A|T|A|.|.|.|0| + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * ^ shift data ^ + * +-------------------------------+ + * + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * out |.|.|.|.|.|.|O|T|H|E|R| |D|A|T|A|.|.|.|0|0|0|0|0|0|0| + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * ^ shift data ^ + * +-------------------------------+ + * + * Replacement = "FOO": + * + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * in |.|.|.|N|E|E|D|L|E|.|.|.|O|T|H|E|R| |D|A|T|A|.|.|.|0| + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * ^ shift data ^ + * +-------------------------------+ + * + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * out |.|.|.|F|O|O|.|.|.|O|T|H|E|R| |D|A|T|A|.|.|.|0|0|0|0| + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * ^ FOO ^ shift data ^ + * +=====+-------------------------------+ + */ +void +replace (struct buffer *buf, const char *find, const char *repl) +{ + char *match; /* pointer to found needle */ + size_t idx; /* cursor into file buffer */ + + size_t nlen; /* length of needle */ + size_t rlen = 0; /* length of replacement */ + + size_t mlen; /* length of matching string */ + + nlen = strlen(find); + + /** + * Iterate over each character in the current string in the + * buffer, because multiple full matches may be possible. If + * we cannot find a match in the current string, skip to the + * next string. I don't think there is a more optimal way? + */ + for (idx = 0; idx <= buf->len; idx++) + { + /** + * Does the current string contain the needle? + */ + match = strstr(buf->data + idx, find); + if (match) + { + /** + * How long is the current string? We need to search + * it until we cannot find any more matches. + */ + mlen = strlen(match); + printf("%10zu bytes at offset 0x%010lx (%02ld%%)\n", + mlen, + (match - buf->data), + (100 * (match - buf->data)) / buf->len + ); + + /** + * The replacement string is l.e. the length of the + * needle, so if it is non-empty, inject it first. + */ + if (repl) + { + rlen = strlen(repl); + memcpy(match, repl, rlen); + } + + /** + * The replacement length may be zero (if empty). In + * any case, copy the non-needle string remainder to + * the current matched (needle) location plus offset + * of any injected replacement. Zero out the tail. + */ + memmove(match + rlen, match + nlen, mlen - nlen); + memset(match + mlen - nlen + rlen, 0, nlen - rlen); + } + else + { + /** + * This is a partial optimization. Don't bother with + * searching for needles in the rest of this string; + * we already know none exist. + */ + idx += strlen(buf->data + idx); + } + } +} + + +/** + * Read the contents of a file into a newly allocated buffer. It + * is possible to 'mmap()', but it leaves less room for checks. + */ +void +scanner (const char *file, const char *find, const char *repl) +{ + FILE *fp = NULL; + size_t nb = 0; + + struct buffer buf; + memset(&buf, 0, sizeof(struct buffer)); + + fp = fopen(file, "rb+"); + if (!fp) + { + fprintf(stderr, + "E: Could not open FILE '%s' for reading!\n", + file + ); + return; + } + + fseek(fp, 0, SEEK_END); + buf.len = ftell(fp); + fseek(fp, 0, SEEK_SET); + + /** + * Allocate memory for the entire file at once. This is not + * ideal, but we don't expect large files for our use case. + */ + buf.data = malloc(buf.len + 1); + if (!buf.data) + { + fclose(fp); + fprintf(stderr, + "E: Could not allocate '%zu' bytes for file '%s'\n", + buf.len, + file + ); + return; + } + buf.data[buf.len] = 0; /* extra byte needs to be nil */ + + nb = fread(buf.data, 1, buf.len, fp); + if (nb != buf.len) + { + free(buf.data); + buf.data = NULL; + fclose(fp); + fprintf(stderr, + "E: Only read '%zu' / '%zu' bytes of file '%s'\n", + nb, + buf.len, + file + ); + return; + } + fclose(fp); + + printf("Examining file '%s'...\n", file); + replace(&buf, find, repl); + + fp = fopen(file, "wb"); + if (!fp) + { + fprintf(stderr, + "E: Could not open FILE '%s' for writing!\n", + file + ); + return; + } + + nb = fwrite(buf.data, 1, buf.len, fp); + if (nb != buf.len) + { + free(buf.data); + buf.data = NULL; + fclose(fp); + fprintf(stderr, + "E: Only wrote '%zu' / '%zu' bytes of file '%s'\n", + nb, + buf.len, + file + ); + return; + } + fclose(fp); + + free(buf.data); +} + + +/** + * WARNING! + * + * This program replaces all occurrences of NEEDLE within string + * sections of an input file with the string REPLACE. The input + * file is overwritten. Few, if any, sanity checks are in place. + */ +int +main (int argc, char **argv) +{ + char *prog = NULL; + char *file = NULL; + char *find = NULL; + char *repl = NULL; + + prog = argv[0]; + + switch (argc) + { + case 3: + file = argv[1]; + find = argv[2]; + break; + case 4: + file = argv[1]; + find = argv[2]; + repl = argv[3]; + if (strlen(repl) > strlen(find)) + { + fprintf(stderr, + "E: REPLACE cannot be longer than NEEDLE\n" + ); + return 1; + } + break; + default: + fprintf(stderr, + "Usage: %s FILE NEEDLE [REPLACE]\n", + prog + ); + return 1; + } + + scanner(file, find, repl); + + return 0; +} diff --git a/bootstrap b/bootstrap new file mode 100755 index 0000000..d2fcf08 --- /dev/null +++ b/bootstrap @@ -0,0 +1,763 @@ +#!/bin/sh -e + +#=============================================================== +# Filename : bootstrap +# Purpose : Bootstraps Adélie from source for any architecture. +# Authors : Zach van Rijn <me@zv.io> +# License : MIT +# Revision : 20221209 +#=============================================================== + +#=============================================================== +# README +#=============================================================== +# +# overview +# -------- +# +# Given a basic development environment ("Baseline System") that +# contains standard system utilities, this script bootstraps the +# Adélie Linux distribution for any suported target architecture +# (assuming that musl, gcc, etc. have been ported to it) WITHOUT +# requiring 'root' privileges. This process is slow* and costly, +# but demonstrates that bootstrapping from source is possible. +# +# Effort has been focused on correctness instead of performance. +# This means, some files can be deleted sooner and some of the +# binaries can run natively on the build machine. We know and do +# appreciate there are different opinions on how to approach the +# bootstrap process. This meets our needs and hopefully offers a +# different perspective on how to simplify cross compilation. +# +# (*) See the "requirements" section for mitigations/discussion. +# +# +# features +# -------- +# +# * One-click bootstrap. Just start the script and walk away. +# +# $ ./bootstrap.sh ppc64 /path/to/new/scratch/dir +# +# * Can be done without root privileges: no 'chroot(8)'. +# +# * This is the real deal. No "seed" binaries are required to +# go from start to finish. The final output is what we use +# to set up our official build servers totally* from source. +# +# * Can be done on a wide range of platforms, almost certainly +# those supported by mcmtools (want to contribute???). +# +# * Minimal dependencies. Nothing fancy. Shell scripts only. +# +# * Can be adapted to assist with porting Adélie to new target +# architectures. TODO: write a tool to automatically do so. +# +# (*) We don't count binaries in your starting environment, and +# recommend https://www.gnu.org/software/mes/ for the brave. +# +# +# design +# ------ +# +# The goal of this script is to support as wide a range of uses +# as possible. For example, a full cross-platform bootstrap that +# needs to be done without target hardware or virtual machines. +# +# There are a few different strategies, depending on your needs +# and what type of privileges/hardware you have access to. +# +# In this diagram, indentation refers to an output product, and +# moving down vertically refers to an input step. CPU A is the +# build architecture, and CPU B is the target architecture. Your +# mileage may vary, and you can mix/match other approaches.. +# +# * CPU A --> CPU B (slowest: no privilege, no CPU B hardware) +# --> Bootstrap 1 (Cross Libc + Partial Cross CPU B) +# --> CPU A + CPU B Mixed Rootfs +# --> Partial Emulation (PRoot) +# --> Bootstrap 2 (Full Cross CPU B) +# --> CPU B Host Rootfs +# --> Full Emulation (PRoot) +# --> Adélie Bootstrap +# --> Adélie Rootfs for CPU B +# +# * CPU A --> CPU B (slower: privilege, no CPU B hardware) +# --> Bootstrap 1 (Cross Libc + Partial Cross CPU B) +# --> CPU A + CPU B Mixed Rootfs +# --> Partial Emulation (registered binfmt_misc) +# --> Bootstrap 2 (Full Cross CPU B) +# --> CPU B Host Rootfs +# --> Full Emulation (registered binfmt_misc) +# --> Adélie Bootstrap +# --> Adélie Rootfs for CPU B +# +# * CPU A --> CPU B (faster; no privilege, CPU B hardware) +# --> Bootstrap 1 (Cross Libc + Partial Cross CPU B) +# --> CPU A + CPU B Mixed Rootfs +# --> Partial Emulation (PRoot) +# --> Bootstrap 2 (Full Cross CPU B) +# --> CPU B Host Rootfs +# --> Copy to Native CPU B Hardware +# --> Adélie Bootstrap +# --> Adélie Rootfs for CPU B +# +# * CPU B --> CPU B (fastest; QEMU-system or native hardware) +# --> Bootstrap 1 (Cross Libc) +# --> CPU B Mixed Rootfs +# --> Bootstrap 2 (Remove Contamination) +# --> CPU B Host Rootfs +# --> Adélie Bootstrap +# --> Adélie Rootfs for CPU B +# +# The resulting Adélie Rootfs is capable of building the entire +# Adélie Linux distribution from source. We use it to seed our +# official build servers. This removes the dependency on using +# previous binary releases of our distribution, which had been +# manually bootstrapped way back from Gentoo and/or on a G5. +# +# +# requirements +# ------------ +# +# TL;DR: You must be able to run the mcmtools bootstrap script. +# +# Cross builds take up a lot of TIME and SPACE, and this cannot +# be improved by throwing hardware at it. This is primarily due +# to the tradeoff of not requiring root privileges at any point, +# and the decision to emulate a native environment instead of to +# force explicit cross-compilation at each step in the process. +# +# Targeting an architecture of opposite endianness will be even +# slower; this is a QEMU limitation. +# +# If you wish to bootstrap to the same target CPU architecture, +# or a compatible mode (e.g. i686 on x86_64), it will be faster. +# +# mcmtools is a hard dependency for our bootstrap process now: +# +# https://git.zv.io/toolchains/bootstrap +# +# It is a simple analog to Buildroot or Yocto. Those tools could +# be used, too, and would provide the "host rootfs" environment. +# +# Internet access is required, but (as an exercise left to the +# reader) it is possible to pre-download all required sources if +# you provide your own rootfs for the static QEMU build process. +# +# If you are in a position to use native hardware, then you are +# able to get away with only the final stages of bootstrapping. +# To do this, you'd essentially comment out the first stages or +# copy the results of the first stages elsewhere and continue. +# +# Other requirements that you should be aware of, estimated: +# +# * As many CPU cores as you can throw at it; +# +# * ~15 GB for each mcmtools rootfs ("seed" and "host"); +# +# * ~ 3 GB for toolchains; +# +# * ~ 2 GB for each intermediate rootfs; +# +# * ~ 2 GB for the "system/" package repository, when built; +# +# * Please refer to the README in 'bootstrap', linked above, +# for more information about performance. Most of this can +# be gained back if you adapt this script to use 'chroot(8)' +# instead of 'PRoot', at the expense of requiring privilege, +# with the correct registration of QEMU with 'binfmt_misc'. +# +# In brief, there is an approximate factor of 13 slowdown on +# practical workloads when using QEMU user and 'PRoot' to do +# dynamic binary translation and emulate a foreign machine. +# +# +# process +# ------- +# +# The illustration below outlines the complete bootstrap process +# and roughly corresponds to the script layout/organization. Our +# terminology is not perfectly consistent; please excuse this. +# +# The term "build" is shorthand for "build-native" CPU, which +# is the machine on which you are performing the bootstrap. +# +# The term "native" is shorthand for "foreign-native" CPU, which +# is the machine to which you are targeting the bootstrap, and +# relates to the "host" CPU on which the code will run, but that +# with the help of dynamic binary translation, runs "natively". +# +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# build (unstable, everything provided by user) +# +# +-----------------+ User-provided tools. Dependency +# | Baseline System | of 'mcmtools', which will verify +# +-----------------+ that these tools are available. +# | +# +----------+ Script to build pinned versions +# | mcmtools | of common system utilities, a +# +----------+ host-arch host-libc toolchain, a +# | host-arch musl-libc toolchain... +# | +# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - +# mixed | (stable versions, unstable libc) +# | +# +-------------+ ...and a 'chroot'-able rootfs. A +# | seed rootfs | sane, but not clean, environment +# +-------------+ in which we begin the bootstrap. +# | +# +-----------+ Script to build 'PRoot' and its +# | emulators | dependencies, as well as static +# +-----------+ QEMU user binaries. Add to seed. +# | +# +-----------------+ Static musl toolchains targeting +# | musl toolchains | a given architecture: cross from +# +-----------------+ the host CPU & foreign "native". +# | Output binaries will run via the +# | 'binfmt_misc' mechanism + QEMU. +# | +# +-------------+ Script to build Alpine Package +# | build tools | Keeper (APK) and dependencies. +# +-------------+ All binaries are cross-compiled! +# | This step is a sanity check. +# | +# +----------+ Script to build pinned versions +# | mcmtools | of common system utilities. This +# +----------+ time, all utilities are target- +# | native. Build more toolchains... +# | +# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - +# host | (stable versions, stable libc) +# | +# +-------------+ ...and a 'chroot'-able rootfs. A +# | host rootfs | sane, clean, foreign "native" +# +-------------+ rootfs that requires QEMU, or is +# | able to run on native hardware. +# | +# +-------------+ Script to build Alpine Package +# | build tools | Keeper (APK) and dependencies. +# +-------------+ These binaries are native built! +# | This step is required! +# | +# +-------------+ Script to build the Adélie Linux +# | system repo | "system/" package repository. It +# +-------------+ is used to build core packages. +# | +# +---------------+ Script to install packages into +# | image creator | a clean rootfs. Carryover from +# +---------------+ the mcmtools process is removed. +# | +# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - +# verify | (optional verification) +# | +# ... The "host" stage can be repeated +# | zero or more times to ensure the +# | final image is not contaminated. +# | +# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - +# output | (Welcome to Adélie Linux) +# | +# +---------------+ This is a minimal Adélie rootfs. +# | adelie rootfs | Copy it to native hardware, then +# +---------------+ use it as a builder 'chroot'. It +# is used to seed official Adélie +# build boxen (e.g. autobuilder). +# +# limitations +# ----------- +# +# The builds are not hermetically sealed. That is not the point. +# The build environment should not *need* anything from outside, +# but it is not *prevented* from accessing anything. You should +# run this on a clean, trusted machine. +# +# There is no guarantee of byte-for-byte reproducible builds at +# this time. This is, in part, due to timestamps and tar headers +# but may involve a lack of determinism in parallel builds. +# +# QEMU user emulation may cause subtle incompatibilites with the +# target CPU hardware. For example, CPU feature (mis)detection. +# +# It is not currently possible to safely reuse any of the output +# if the target architecture is changed. You will need to start +# from scratch if you wish to change the target. This is a TODO. +# +# +# todo +# ---- +# +# * Do not inherit the build environment; avoid contamination. +# +# * Finish the "system/" build; the "bootstrap.sh" script will +# require some work. Current status: preimage is finished. +# + +HERE="$(dirname $(readlink -f ${0}))"; + + +#--------------------------------------------------------------- +# initialization + +## +# Haaaalp! +# +usage () +{ + printf "Usage: %s ARCH BASE\n\n" "${0}"; + cat <<EOF + ARCH { aarch64, armv7, ppc64, ppc, x86_64, pmmx } + + BASE an absolute path to bootstrap out of. + ** MAY CAUSE DATA LOSS IF SET INCORRECTLY! ** +EOF + exit 0; +} + + +## +# argv[1]: ARCH +# +# ARCH is the Adélie Linux target. This is the first step in the +# porting process, so e.g. mips64, sparc64, riscv64 will need to +# be added to this table when the time comes to port to them. +# +# ARCH is translated to canonical GCC and QEMU machine types. +# +case "${1}" in +# adelie gcc qemu +# ------ --- ---- + aarch64) m=aarch64: ; q=aarch64 ; ;; + armv7) m=armv7l:eabihf ; q=arm ; ;; + ppc) m=powerpc: ; q=ppc ; ;; + ppc64) m=powerpc64: ; q=ppc64 ; ;; + riscv64) m=riscv64: ; q=riscv64 ; ;; + x86_64) m=x86_64: ; q=x86_64 ; ;; + pmmx) m=i586: ; q=i386 ; ;; + *) usage ;; +esac +shift; + + +## +# argv[2]: BASE +# +# BASE is a semi-permanent scratch directory. It is where all of +# the magic happens, and probably cannot be relocated easily. Be +# careful to not set it incorrectly or to a place you'd regret +# being overwritten, corrupted, or deleted. +# +case "${1}" in + /*) BASE="${1}"; + ;; + *) printf "BASE not set, or not an absolute path!\n"; + exit 1; + ;; +esac +shift; + + +## +# Internal variables. Do not modify this section directly. +# +CHAINS=https://git.zv.io/toolchains; +printf "CHAINS=%s\n" "${CHAINS}"; + +SYSTEM="-adelie-linux-musl"; # we only target musl on Linux +printf "SYSTEM=%s\n" "${SYSTEM}"; + +NATIVE=$(cc -dumpmachine); # host arch, host libc +printf "NATIVE=%s\n" "${NATIVE}"; + +BUILDS="${NATIVE%%-*}${SYSTEM}"; # host arch, musl libc +printf "BUILDS=%s\n" "${BUILDS}"; + +TARGET="${m%:*}${SYSTEM}${m#*:}"; # ultimate Adélie Linux target +printf "TARGET=%s\n" "${TARGET}"; + +MTOOLS=${MCMTOOLS:-"${BASE}/mcmtools"}; # CAREFUL! MAY CAUSE DATA LOSS! +printf "MTOOLS=%s\n" "${MTOOLS}"; + +## +# Default 'PATH' for use inside various rootfs environments. +# +DEF_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +mkdir -p "${BASE}"; + + +#--------------------------------------------------------------- +# mcmtools (seed) + +## +# Allow the user to supply an existing mcmtools installation. It +# is not ideal but we can allow the user to save some CPU cycles +# at the cost of adding new tools to the existing installation. +# +# If they wish to keep the existing mcmtools installation clean, +# the 'binsub' tool can be used to relocate it. A temporary dir +# is used if this environment variable is omitted. Another case +# for providing a custom value is if '/tmp' is mounted weird. +# +if ! test -d "${MTOOLS}"/sys/emus/bin; then # FIXME: no hard code + cd "${BASE}"; + + test -d bootstrap \ + || git clone ${CHAINS}/bootstrap.git; + cd bootstrap; + git checkout 371f5e2624acd89dd05a9bd8c68f369b16dfdde6; + + ## seed rootfs + # + # This will build absolutely everything that is needed to be + # self-reliant, except for some build deps for QEMU. + # + # We copy 'config.mak' from mcmtools bootstrap to the rootfs + # so that when we build "real" toolchain they are the same. + # + DEST="${MTOOLS}" \ + ARCH=${BUILDS} \ + ./bootstrap \ + ; + test -f "${MTOOLS}"/config.mak || \ + cp "${MTOOLS}"/tmp/musl-cross-make/config.mak \ + "${MTOOLS}"/config.mak \ + ; + # cleaning + ( + cd "${MTOOLS}"; + rm -fr tmp; # save 10 GB + ) + + # is any of this actually needed? + ( + cd "${MTOOLS}"/sys; + mkdir -p dev; + mkdir -p proc; + mkdir -p sys; + rm -fr usr; + ln -s . usr; + ) + + ## emulators + # + # Dependencies are built with the mcmtools host toolchain; a + # reason to not force musl here is in the event that these + # cannot be built statically and the host libc is different. + # Our priority is to obtain a functioning 'PRoot' above all. + # + # QEMU itself is built inside an Alpine Linux rootfs; we do + # this because we still need Python 3 to build it. You can + # manually provide your own static QEMU user binaries and be + # on your way without Alpine, but it is a good 'PRoot' test. + # + test -d "${MTOOLS}/sys/emus/bin" || \ + PATH="${MTOOLS}/host/bin:${MTOOLS}/sys/bin" \ + DEST="${MTOOLS}" \ + ./prootemu \ + ; +fi + + +#--------------------------------------------------------------- +# musl toolchains + +## +# We have a musl-targeting toolchain that runs on the host, but +# it is linked to the host libc and may not run in this chroot. +# +# We need to build the same toolchain, but static. There are two +# possible directions to go: (1) a cross toolchain that runs at +# full speed on the build machine, at the cost of having to tell +# downstream build scripts how to cross compile, or (2) a native +# toolchain for the foreign (target) architecture that runs slow +# in QEMU and still requires the cross toolchain to build it. +# +# Build both. We will eventually need both toolchains, anyway. +# +if ! test -d "${MTOOLS}"/sys/tc/native; then # FIXME: no hard code + cd "${BASE}"; + + test -d musl-cross-make \ + || git clone ${CHAINS}/musl-cross-make.git; + cd musl-cross-make; + git pull; # always use the latest + + ## + # Ensure consistent 'config.mak' for all toolchain builds. + # + cp "${MTOOLS}"/config.mak config.mak; + + ## musl toolchains + # + # Build these toolchains statically using the musl toolchain + # from the seed rootfs so that it is known to work correctly + # (the original musl toolchain itself may itself be linked + # with glibc or be unsafe to use in some contexts). + # + # Note: "native" is for the foreign target CPU architecture. + # + MCMTOOLS="${MTOOLS}" \ + ./scripts/build ${TARGET} \ + ; + for k in cross native; do + rm -fr "${MTOOLS}"/sys/tc/${k}; + mkdir "${MTOOLS}"/sys/tc/${k}; + tar -C "${MTOOLS}"/sys/tc/${k} \ + --strip-components=1 \ + -xzf output/${TARGET}-${k}.tgz \ + ; + done +fi + + +#--------------------------------------------------------------- +# build tools (cross) + +## +# Build 'abuild', its dependencies, and other utilities. +# +PROOT_NO_SECCOMP=1 \ +PATH="/tc/cross/bin:${DEF_PATH}" \ +SHELL=/bin/sh \ +DEST=/usr/local \ +CURL_CA_BUNDLE=/cacert.pem \ +CC=${TARGET}-gcc \ +CXX=${TARGET}-g++ \ +AR=${TARGET}-ar \ +LD=${TARGET}-ld \ +CPP=${TARGET}-cpp \ +RANLIB=${TARGET}-ranlib \ +${MTOOLS}/sys/emus/bin/proot \ + -S "${MTOOLS}"/sys \ + -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \ + -b "${HERE}" \ + "${HERE}"/setup-abuild \ + ; + + +#--------------------------------------------------------------- +# rootfs: build + +## +# Create a tarball of the build rootfs. The image creator could +# use this as input if the target architecture matches. +# +cd "${BASE}"; +if ! test -f rootfs-${BUILDS}.tgz; then + tar -C mcmtools/sys \ + -pczf rootfs-${BUILDS}.tgz \ + . \ + ; +fi + + +#--------------------------------------------------------------- +# mcmtools (host) + +## +# NOTE: The 'PATH' order is *really* important. Foreign "native" +# toolchains must be the first ones found; e.g. '/usr/bin/gcc' +# is a symlink to 'ccache' and isn't a functional compiler. +# +# NOTE: 'LD_LIBRARY_PATH' is needed because of nonstandard path +# of toolchain, which otherwise produces these errors: +# +# Error loading shared library libstdc++.so.6: +# Exec format error (needed by ...) +# Error loading shared library libgcc_s.so.1: +# Exec format error (needed by ...) +# +# NOTE: We *do* need to build 'musl-cross-make' (even though it +# may seem redundant) so that we can safely install the dynamic +# loader. If we do not do this, we're on the hook to symlink it. +# +# Once 'coreutils' is built, 'uname' will return correctly, then +# future software should(tm) build as if it is built natively. +# +# FIXME: is Linux 3.2.0 really appropriate here? +# +cd "${BASE}"; +if ! test -d mcmtools-${TARGET}/sys/tc/musl/bin; then # FIXME: no hard code +PROOT_NO_SECCOMP=1 \ +PATH="/tc/native/bin:${DEF_PATH}" \ +LD_LIBRARY_PATH=/tc/native/lib \ +SHELL=/bin/sh \ +BASE="${BASE}" \ +DEST="${BASE}"/mcmtools-${TARGET} \ +ARCH=${TARGET} \ +CURL_CA_BUNDLE=/cacert.pem \ +${MTOOLS}/sys/emus/bin/proot \ + -S "${MTOOLS}"/sys \ + -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \ + -b "${HERE}" \ + -k "3.2.0" \ + "${BASE}"/bootstrap/bootstrap \ + ; +fi + +# cleaning +( + cd mcmtools-${TARGET}; + rm -fr tmp; # save 10 GB +) + +## +# Install the native static toolchain over the native rootfs for +# a quick way to have the dynamic loader. Overwrite the symlink! +# +if test -h mcmtools-${TARGET}/sys/bin/gcc; then # FIXME: no hard code + tar -C mcmtools-${TARGET}/sys \ + --strip-components=1 \ + -xzf musl-cross-make/output/${TARGET}-native.tgz \ + ; +fi + + +#--------------------------------------------------------------- +# build tools (host) + +## +# Build 'abuild', its dependencies, and other utilities. +# +PROOT_NO_SECCOMP=1 \ +PATH="/tc/musl/bin:${DEF_PATH}" \ +SHELL=/bin/sh \ +DEST=/usr/local \ +CURL_CA_BUNDLE=/cacert.pem \ +CC=gcc \ +CXX=g++ \ +AR=ar \ +LD=ld \ +CPP=cpp \ +${MTOOLS}/sys/emus/bin/proot \ + -S "${BASE}"/mcmtools-${TARGET}/sys \ + -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \ + -b "${HERE}" \ + "${HERE}"/setup-abuild \ + ; + + +#--------------------------------------------------------------- +# rootfs: host + +## +# Create a tarball of the host rootfs. This is a safety measure. +# +cd "${BASE}"; +if ! test -f rootfs-${TARGET}.tgz; then + tar -C mcmtools-${TARGET}/sys \ + -pczf rootfs-${TARGET}.tgz \ + . \ + ; +fi + + +#--------------------------------------------------------------- +# patch rootfs + +## +# Some paths are hard-coded into various binaries. Since we need +# to "relocate" everything to run at '/', we can decompress the +# '.tar' file and perform the appropriate substititons at once. +# +cd "${BASE}"; +if ! test -f rootfs-${TARGET}-patched.tgz; then + gzip -dk rootfs-${TARGET}.tgz; + + ## + # Build 'binsub', a string patch tool. + # + "${MTOOLS}"/musl/bin/gcc -static -o binsub binsub.c -O3; + + ## + # Replace with explicit '/' ('/foo/bar/baz' --> '/' instead + # of the empty string) to avoid gotchas. Assumes '//' is the + # same as '/'. I can't think of a counterexample. + # + ./binsub rootfs-${TARGET}.tar \ + "${BASE}"/mcmtools-${TARGET}/sys \ + / \ + ; + + gzip -c9 rootfs-${TARGET}.tar > rootfs-${TARGET}-patched.tgz; + + rm -f rootfs-${TARGET}.tar; +fi + + +#--------------------------------------------------------------- +# preimage + +## +# The preimage has some modifications to support using the real +# "bootstrap.sh" script. Note that we need to bind-mount and use +# a different working directory because PRoot has a few defaults +# that would otherwise shadow paths we're trying to create. +# +cd "${BASE}"; +if ! test -f rootfs-${TARGET}-preimage.tgz; then # FIXME: no hard code + rm -fr rootfs-${TARGET}-preimage; + mkdir rootfs-${TARGET}-preimage; + tar -C rootfs-${TARGET}-preimage \ + -xzf rootfs-${TARGET}-patched.tgz \ + || true; # we're somehow messing up some bzip binaries? + +PROOT_NO_SECCOMP=1 \ +PATH="${DEF_PATH}" \ +SHELL=/bin/sh \ +CURL_CA_BUNDLE=/cacert.pem \ +${MTOOLS}/sys/emus/bin/proot \ + -S "${MTOOLS}"/sys \ + -b "${BASE}"/rootfs-${TARGET}-preimage \ + -w "${BASE}"/rootfs-${TARGET}-preimage \ + -b "${HERE}" \ + "${HERE}"/setup-preimage \ + ; + + tar -C rootfs-${TARGET}-preimage \ + -pczf rootfs-${TARGET}-preimage.tgz \ + . \ + ; +fi + + +#--------------------------------------------------------------- +# packages + +## +# This is where the original "bootstrap.sh" script runs. The aim +# of this step is to produce '.apk' files that were built using +# non-Adélie-packaged tools, but would install Adélie tools. +# +if ! test -f rootfs-${TARGET}-packages.tgz; then # FIXME: no hard code + rm -fr rootfs-${TARGET}-packages; + mkdir rootfs-${TARGET}-packages; + tar -C rootfs-${TARGET}-packages \ + -xzf rootfs-${TARGET}-preimage.tgz \ + ; + +PROOT_NO_SECCOMP=1 \ +PATH="${DEF_PATH}" \ +SHELL=/bin/sh \ +TARGET_ARCH=${TARGET} \ +CURL_CA_BUNDLE=/cacert.pem \ +${MTOOLS}/sys/emus/bin/proot \ + -R "${BASE}"/rootfs-${TARGET}-packages \ + -i 1000:300 \ + -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \ + -b "${HERE}" \ + "${HERE}"/setup-packages \ + ; +fi + +# TODO: create tarball of this directory + + +#--------------------------------------------------------------- +# image creator + +# TODO + +# apk --root /foo --arch armv7 --initdb add +# apk --root /foo add adelie-core dash-binsh build-tools diff --git a/setup-abuild b/setup-abuild new file mode 100755 index 0000000..686df6d --- /dev/null +++ b/setup-abuild @@ -0,0 +1,362 @@ +#!/bin/sh -e + +HERE="$(dirname $(readlink -f ${0}))"; +DEST=/usr/local; + +git config --global http.sslCAInfo "${CURL_CA_BUNDLE}"; + + +mkdir -p "${DEST}"; +cd "${DEST}"; # this directory will already exist if correct + +## +# musl +# +# This provides the dynamic loader for the foreign (target) arch +# so that we do not have to force building static musl binaries. +# +nmus=musl; +vmus=1.2.3; +test ! -f ._${nmus}-${vmus} && \ +( + test ! -d ${nmus}-${vmus} \ + && curl -s https://musl.libc.org/releases/${nmus}-${vmus}.tar.gz \ + | tar -xzf - \ + ; + cd ${nmus}-${vmus}; + rm -fr x; mkdir x; cd x; + ../configure \ + --prefix=/usr \ + --enable-static \ + --enable-shared \ + ; + make -j$(nproc); + make install; +) +touch ._${nmus}-${vmus}; +rm -fr ${nmus}-${vmus}; + + +## +# OpenSSL +# +nssl=openssl; +vssl=1.1.1s; +test ! -f ._${nssl}-${vssl} && \ +( + test ! -d ${nssl}-${vssl} \ + && curl -s https://www.openssl.org/source/${nssl}-${vssl}.tar.gz \ + | tar -xzf - \ + ; + cd ${nssl}-${vssl}; + rm -fr x; mkdir x; cd x; + ../Configure cc \ + --prefix="${DEST}" \ + --openssldir="${DEST}" \ + no-shared \ + ; + make -j$(nproc); + make install_sw install_ssldirs; +) +touch ._${nssl}-${vssl}; +rm -fr ${nssl}-${vssl}; + + +## +# zlib +# +nzlb=zlib; +vzlb=1.2.13; +test ! -f ._${nzlb}-${vzlb} && \ +( + test ! -d ${nzlb}-${vzlb} \ + && curl -s https://www.zlib.net/${nzlb}-${vzlb}.tar.gz \ + | tar -xzf - \ + ; + cd ${nzlb}-${vzlb}; + rm -fr x; mkdir x; cd x; + ../configure \ + --prefix="${DEST}" \ + --static \ + ; + make -j$(nproc); + make install; +) +touch ._${nzlb}-${vzlb}; +rm -fr ${nzlb}-${vzlb}; + + +## +# abuild +# +nbld=abuild; +vbld=f75ebd1953f6e41416b5037c4049b55b9a21a1ea; +test ! -f ._${nbld}-${vbld} && \ +( + test ! -d ${nbld}-${vbld} \ + && git clone https://git.adelielinux.org/adelie/${nbld}.git ${nbld}-${vbld} \ + ; + cd abuild-${vbld}; + git checkout ${vbld}; + while read k; do curl -s ${k} | patch -p1 || true; done <<EOF +http://ix.io/4ifN +EOF + rm -fr x; mkdir x; cd x; + export SSL_CFLAGS="-I${DEST}/include"; + export SSL_LDFLAGS="-L${DEST}/lib"; + export SSL_LIBS="-lssl -lcrypto"; # not in mcmtools + export ZLIB_LIBS="-lz"; # do not use from mcmtools + export LDFLAGS="-L${DEST}/lib -lssl -lcrypto"; + export CFLAGS="-static -I${DEST}/include"; + sed -i "${DEST}/abuild-${vbld}/abuild-sudo.c" \ + -e "s@__DEST__@${DEST}@" \ + ; # hardcoded + make -j$(nproc) -C .. install \ + prefix="${DEST}" \ + sysconfdir="${DEST}" \ + SCDOC=true \ + ; + sed -i "${DEST}/bin/abuild" \ + -e 's@/bin/ash -e@/usr/bin/env bash@' \ + ; # hardcoded +) +touch ._${nbld}-${vbld}; +rm -fr ${nbld}-${vbld}; + + +## +# util-linux (for 'getopt' used by 'abuild-keygen') +# +nutl=util-linux; +vutl=08431acdf5b3accd0887ab550bfa4efabed751d6; +test ! -f ._${nutl}-${vutl} && \ +( + test ! -d ${nutl}-${vutl} \ + && mkdir ${nutl}-${vutl} \ + && git clone https://github.com/karelzak/${nutl}.git ${nutl}-${vutl} \ + ; + cd ${nutl}-${vutl}; + git checkout ${vutl}; + test -f configure || ./autogen.sh; + rm -fr x; mkdir x; cd x; + ../configure \ + --prefix="${DEST}" \ + --host="$(${CC} -dumpmachine)" \ + --enable-static \ + --disable-shared \ + ; + sed -i Makefile \ + -e 's/chgrp/-chgrp/g' \ + -e 's/chmod/-chmod/g' \ + -e 's/chown/-chown/g' \ + ; # allow non-root installation + make -j$(nproc) install; +) +touch ._${nutl}-${vutl}; +rm -fr ${nutl}-${vutl}; + + +## +# pkgconf (pkg-config replacement) +# +npkg=pkgconf; +vpkg=623b8f7851648a5c476de904a8ffed7b7b679aab; # until autoconf 2.71 +test ! -f ._${npkg}-${vpkg} && +( + test ! -d ${npkg}-${vpkg} \ + && git clone https://github.com/${npkg}/${npkg}.git ${npkg}-${vpkg} \ + ; + cd ${npkg}-${vpkg}; + git checkout ${vpkg}; + ./autogen.sh; + ./configure \ + --prefix="${DEST}" \ + --host="$(${CC} -dumpmachine)" \ + --enable-static \ + --disable-shared \ + --with-system-libdir=/lib:/usr/lib \ + --with-system-includedir=/usr/include \ + ; + make -j$(nproc) install; + ln -s pkgconf "${DEST}"/bin/pkg-config +) +touch ._${npkg}-${vpkg}; +rm -fr ${npkg}-${vpkg}; + + +## +# samurai (ninja replacement) +# +nsam=samurai; +vsam=4cc8f4a3654b72c977e0e62367d1ab6d5b873def; +test ! -f ._${nsam}-${vsam} && +( + test ! -d ${nsam}-${vsam} \ + && git clone https://github.com/michaelforney/${nsam}.git ${nsam}-${vsam} \ + ; + cd ${nsam}-${vsam}; + make -j$(nproc) install \ + PREFIX="" \ + DESTDIR="${DEST}" \ + ; + +) +touch ._${nsam}-${vsam}; +rm -fr ${nsam}-${vsam}; + + +## +# muon (meson replacement) +# +nmun=muon; +vmun=62ce4561b4444e5020dc39aad0381655afeda0d6; +test ! -f ._${nmun}-${vmun} && +( + test ! -d ${nmun}-${vmun} \ + && git clone https://git.sr.ht/~lattis/${nmun} ${nmun}-${vmun} \ + ; + cd ${nmun}-${vmun}; + sed -i bootstrap.sh \ + -e 's/if.*then/if false; then/g' \ + ; + ./bootstrap.sh \ + bootstrap \ + ; + bootstrap/muon setup \ + -Dstatic=true \ + build \ + ; + sed -i build/build.ninja \ + -e "s@\bar\b@${AR}@g" \ + ; + samu -C build; + cp build/muon "${DEST}/bin"; +) +touch ._${nmun}-${vmun}; +rm -fr ${nmun}-${vmun}; + + +## +# apk-tools +# +# Ariadne says use meson, and 'muon' doesn't work so... kludges! +# Also, '-j' will break the build. +# +natl=apk-tools; +vatl=be4ce40797af9056c79be4dc74ff978f1f4957e4; +test ! -f ._${natl}-${vatl} && \ +( + test ! -d ${natl}-${vatl} \ + && git clone https://git.alpinelinux.org/${natl} ${natl}-${vatl} \ + ; + cd ${natl}-${vatl}; + sed -i Make.rules \ + -e '/targets += $(__shlibs) $(shobjs)/d' \ + ; # disable shared libs + sed -i Make.rules \ + -e '/CC.*:=/d' \ + -e '/AR.*:=/d' \ + -e '/LD.*:=/d' \ + ; # inherit from environment + sed -i src/Makefile \ + -e 's/$(install-libapk_so)//g' -e 's/$(libapk_so)//g' \ + -e 's/ version.o/ version.o strlcpy.o/' \ + ; # disable shared libs, hack the hack + sed -i src/context.c \ + -e "s@var/log@${DEST}/${1}/var/log@" \ + ; # hardcoded + ln -sf ../portability/strlcpy.c src/strlcpy.c; + export LUA=no; # documentation requires lua + make clean; + make \ + INSTALLDIR="${DEST}" \ + CFLAGS="-I${DEST}/include -DNEED_STRLCPY -Wno-error" \ + LDFLAGS="-L${DEST}/lib -L${DEST}/${natl}-${vatl}/libfetch" \ + LIBS="-lapk -lfetch -lssl -lcrypto -lz" \ + ; + cp src/apk "${DEST}/bin"; +) +touch ._${natl}-${vatl}; +rm -fr ${natl}-${vatl}; + + +## +# pax-utils +# +npax=pax-utils; +vpax=974b9359c2f89d57e69598572aafcd8f920d79e2; +test ! -f ._${npax}-${vpax} && \ +( + test ! -d ${npax}-${vpax} \ + && git clone https://anongit.gentoo.org/git/proj/${npax}.git ${npax}-${vpax} \ + ; + cd ${npax}-${vpax}; + git checkout ${vpax}; + muon setup build; + samu -C build; + muon -C build install; +) +touch ._${npax}-${vpax}; +rm -fr ${npax}-${vpax}; + + +## +# fakeroot +# +nfrt=fakeroot; +vfrt=8c0260009e85264fd1ea282fbb22063fc694c552; # until autoconf 2.71 +test ! -f ._${nfrt}-${vfrt#*:} && \ +( + test ! -d ${nfrt}-${vfrt} \ + && git clone https://salsa.debian.org/clint/${nfrt}.git ${nfrt}-${vfrt} \ + ; + cd ${nfrt}-${vfrt}; + git checkout ${vfrt}; + while read k; do curl -s ${k} | patch -p1 || true; done <<EOF +https://git.alpinelinux.org/aports/plain/main/fakeroot/do-not-redefine-id_t.patch?id=bb497eeb2155d0332284942105692bc05fec25a9 +https://git.alpinelinux.org/aports/plain/main/fakeroot/fakeroot-no64.patch?id=bb497eeb2155d0332284942105692bc05fec25a9 +https://git.alpinelinux.org/aports/plain/main/fakeroot/fakeroot-stdint.patch?id=bb497eeb2155d0332284942105692bc05fec25a9 +https://git.alpinelinux.org/aports/plain/main/fakeroot/fix-format.patch?id=bb497eeb2155d0332284942105692bc05fec25a9 +https://git.alpinelinux.org/aports/plain/main/fakeroot/fix-shell-in-fakeroot.patch?id=bb497eeb2155d0332284942105692bc05fec25a9 +EOF + ./bootstrap; + f=$(mktemp); # needed due to "error: unknown type name 'cap_user_header_t'" + echo > ${f} "#include <linux/capability.h>"; + cat libfakeroot.c >> ${f}; + mv ${f} libfakeroot.c; + rm -fr x; mkdir x; cd x; + CFLAGS="-D_STAT_VER=0 $CFLAGS" \ + ../configure \ + --prefix="${DEST}" \ + ; + sed -i Makefile \ + -e '/^SUBDIRS =/ s/doc//g' \ + ; + make -j$(nproc) install; +) +touch ._${nfrt}-${vfrt#*:}; +rm -fr ${nfrt}-${vfrt#*:}; + + +## +# attr +# +natt=attr; +vatt=2979615e71fb53b3f2f9085eea516d4e2b3174ea; +test ! -f ._${natt}-${vatt} && +( + test ! -d ${natt}-${vatt} \ + && git clone https://git.savannah.nongnu.org/git/${natt}.git ${natt}-${vatt} \ + ; + cd ${natt}-${vatt}; + git checkout ${vatt}; + ./autogen.sh; + ./configure \ + --prefix="${DEST}" \ + --enable-static \ + --disable-shared \ + ; + make -j$(nproc) install; +) +touch ._${natt}-${vatt}; +rm -fr ${natt}-${vatt}; diff --git a/setup-packages b/setup-packages new file mode 100755 index 0000000..ea449dc --- /dev/null +++ b/setup-packages @@ -0,0 +1,13 @@ +#!/bin/sh -e + +d=$(mktemp -d); +cd "${d}"; +pwd; + +git clone https://git.adelielinux.org/adelie/packages.git; +cd packages; +git checkout 198e0130d845f7466c09a61f3829905d95313c06; + +# FIXME: this script needs a lot of work +#./scripts/bootstrap.sh ${TARGET_ARCH}; +bash diff --git a/setup-preimage b/setup-preimage new file mode 100755 index 0000000..7892a0d --- /dev/null +++ b/setup-preimage @@ -0,0 +1,35 @@ +#!/bin/sh -e + +set -x; + +# these will require proper mounting when actually used +mkdir -p dev proc sys tmp; + +# overwrite leftovers from mcmtools bootstrap +cat > root/.bashrc <<EOF +PATH="${PATH}"; +EOF + +# handle git backend issue (hardcoded paths) +# git config --global http.sslCAInfo /cacert.pem +cat > root/.gitconfig <<EOF +[http] + sslCAInfo = "${CURL_CA_BUNDLE}" +EOF + +# handle CA certificate issue (this or 'CURL_CA_BUNDLE' env var) +cat > root/.curlrc <<EOF +--cacert "${CURL_CA_BUNDLE}" +EOF + +# hack to create abuild user and group +# FIXME: have to use a patched 'abuild' anyway, debug me! +cat > etc/group <<EOF +abuild:x:300:builder +EOF +cat > etc/passwd <<EOF +builder:x:1000:300::/usr/src:/bin/sh +EOF + +# maybe not necessary? +chown -R 1000:300 root etc; |