diff options
| -rwxr-xr-x | scripts/bootstrap-abuild | 351 | ||||
| -rwxr-xr-x | scripts/bootstrap-host | 4 | ||||
| -rwxr-xr-x | scripts/bootstrap.sh | 546 | ||||
| -rw-r--r--[-rwxr-xr-x] | scripts/setup-abuild | 2 |
4 files changed, 674 insertions, 229 deletions
diff --git a/scripts/bootstrap-abuild b/scripts/bootstrap-abuild new file mode 100755 index 000000000..4333113a8 --- /dev/null +++ b/scripts/bootstrap-abuild @@ -0,0 +1,351 @@ +#!/bin/sh -e + +## +# This script is to be called from 'bootstrap.sh', not sourced. +# PATH is fully contained. We install to 'MCMTOOLS/sys', which +# will add (and may overwrite) in some cases. This process is +# destructive. We do this so that we may easily 'chroot' later. +# +HERE="$(dirname $(readlink -f ${0}))"; +DEST=/usr/local; + +git config --global http.sslCAInfo "${CURL_CA_BUNDLE}"; + +mkdir -p "${DEST}"; +cd "${DEST}"; # this directory will already exist if correct + +if false; then # provided by latest 'bootstrap' +## +# Perl +# +nprl=perl; +vprl=5.36.0; +test ! -f ._${vprl}-${vprl} && \ +( + test ! -d ${nprl}-${vprl} \ + && curl -s https://www.cpan.org/src/${vprl%%.*}.0/perl-${vprl}.tar.gz \ + | tar -xzf - \ + ; + cd ${nprl}-${vprl}; + rm -fr x; mkdir x; cd x; + ../Configure -des; + make -j$(nproc); + make install; +) +touch ._${nprl}-${vprl}; +rm -fr ${nprl}-${vprl}; +fi + +## +# musl +# +# This provides the dynamic loader for the foreign (target) arch +# so that we do not have to force building static musl binaries. +# +nmus=musl; +vmus=1.2.3; +test ! -f ._${nmus}-${vmus} && \ +( + test ! -d ${nmus}-${vmus} \ + && curl -s https://musl.libc.org/releases/${nmus}-${vmus}.tar.gz \ + | tar -xzf - \ + ; + cd ${nmus}-${vmus}; + rm -fr x; mkdir x; cd x; + ../configure \ + --prefix="${DEST}" \ + --enable-static \ + --enable-shared \ + ; + make -j$(nproc); + make install; +) +touch ._${nmus}-${vmus}; +rm -fr ${nmus}-${vmus}; + + +## +# OpenSSL +# +nssl=openssl; +vssl=1.1.1s; +test ! -f ._${nssl}-${vssl} && \ +( + test ! -d ${nssl}-${vssl} \ + && curl -s https://www.openssl.org/source/${nssl}-${vssl}.tar.gz \ + | tar -xzf - \ + ; + cd ${nssl}-${vssl}; + rm -fr x; mkdir x; cd x; + ../Configure cc \ + --prefix="${DEST}" \ + --openssldir="${DEST}" \ + no-shared \ + ; + make -j$(nproc); + make install_sw install_ssldirs; +) +touch ._${nssl}-${vssl}; +rm -fr ${nssl}-${vssl}; + + +## +# zlib +# +nzlb=zlib; +vzlb=1.2.13; +test ! -f ._${nzlb}-${vzlb} && \ +( + test ! -d ${nzlb}-${vzlb} \ + && curl -s https://www.zlib.net/${nzlb}-${vzlb}.tar.gz \ + | tar -xzf - \ + ; + cd ${nzlb}-${vzlb}; + rm -fr x; mkdir x; cd x; + ../configure \ + --prefix="${DEST}" \ + --static \ + ; + make -j$(nproc); + make install; +) +touch ._${nzlb}-${vzlb}; +rm -fr ${nzlb}-${vzlb}; + + +## +# abuild +# +nbld=abuild; +vbld=ee13f777d56e8f3e8848fec9a422430a66292cc1; +test ! -f ._${nbld}-${vbld} && \ +( + test ! -d ${nbld}-${vbld} \ + && git clone https://git.alpinelinux.org/${nbld} ${nbld}-${vbld} \ + ; + cd abuild-${vbld}; + git checkout ${vbld}; +# patch -p1 --forward < "${HERE}/patches/0001-allow-untrusted.diff" || true; # FIXME +# patch -p1 --forward < "${HERE}/patches/0001-etc-apk-keys.diff" || true; # FIXME +# patch -p1 --forward < "${HERE}/patches/0001-extra-lib-paths.diff" || true; # FIXME + rm -fr x; mkdir x; cd x; + export SSL_CFLAGS="-I${DEST}/include"; + export SSL_LDFLAGS="-L${DEST}/lib"; + export SSL_LIBS="-lssl -lcrypto"; # not in mcmtools + export ZLIB_LIBS="-lz"; # do not use from mcmtools + export LDFLAGS="-L${DEST}/lib -lssl -lcrypto"; + export CFLAGS="-DABUILD_GROUP=\\\"$(id -gn)\\\" -static -I${DEST}/include"; # default 'abuild' if undefined + sed -i "${DEST}/abuild-${vbld}/abuild-sudo.c" \ + -e "s@/sbin/apk@${DEST}/bin/apk@" \ + ; # hardcoded + make -j$(nproc) -C .. install \ + prefix="${DEST}" \ + sysconfdir="${DEST}" \ + SCDOC=true \ + ; + sed -i "${DEST}/bin/abuild" \ + -e 's@/bin/ash -e@/usr/bin/env bash@' \ + ; # hardcoded +) +touch ._${nbld}-${vbld}; +rm -fr ${nbld}-${vbld}; + + +## +# util-linux (for 'getopt' used by 'abuild-keygen') +# +nutl=util-linux; +vutl=08431acdf5b3accd0887ab550bfa4efabed751d6; +test ! -f ._${nutl}-${vutl} && \ +( + test ! -d ${nutl}-${vutl} \ + && mkdir ${nutl}-${vutl} \ + && git clone https://github.com/karelzak/${nutl}.git ${nutl}-${vutl} \ + ; + cd ${nutl}-${vutl}; + git checkout ${vutl}; + test -f configure || ./autogen.sh; + rm -fr x; mkdir x; cd x; + ../configure \ + --prefix="${DEST}" \ + --host="$(${CC} -dumpmachine)" \ + --enable-static \ + --disable-shared \ + ; + sed -i Makefile \ + -e 's/chgrp/-chgrp/g' \ + -e 's/chmod/-chmod/g' \ + -e 's/chown/-chown/g' \ + ; # allow non-root installation + make -j$(nproc) install; +) +touch ._${nutl}-${vutl}; +rm -fr ${nutl}-${vutl}; + + +## +# pkgconf (pkg-config replacement) +# +npkg=pkgconf; +vpkg=623b8f7851648a5c476de904a8ffed7b7b679aab; # until autoconf 2.71 +test ! -f ._${npkg}-${vpkg} && +( + test ! -d ${npkg}-${vpkg} \ + && git clone https://github.com/${npkg}/${npkg}.git ${npkg}-${vpkg} \ + ; + cd ${npkg}-${vpkg}; + git checkout ${vpkg}; + ./autogen.sh; + ./configure \ + --prefix="${DEST}" \ + --host="$(${CC} -dumpmachine)" \ + --enable-static \ + --disable-shared \ + --with-system-libdir=/lib:/usr/lib \ + --with-system-includedir=/usr/include \ + ; + make -j$(nproc) install; +) +touch ._${npkg}-${vpkg}; +rm -fr ${npkg}-${vpkg}; + + +## +# samurai (ninja replacement) +# +nsam=samurai; +vsam=4cc8f4a3654b72c977e0e62367d1ab6d5b873def; +test ! -f ._${nsam}-${vsam} && +( + test ! -d ${nsam}-${vsam} \ + && git clone https://github.com/michaelforney/${nsam}.git ${nsam}-${vsam} \ + ; + cd ${nsam}-${vsam}; + make -j$(nproc) install \ + PREFIX="" \ + DESTDIR="${DEST}" \ + ; + +) +touch ._${nsam}-${vsam}; +rm -fr ${nsam}-${vsam}; + + +## +# muon (meson replacement) +# +nmun=muon; +vmun=62ce4561b4444e5020dc39aad0381655afeda0d6; +test ! -f ._${nmun}-${vmun} && +( + test ! -d ${nmun}-${vmun} \ + && git clone https://git.sr.ht/~lattis/${nmun} ${nmun}-${vmun} \ + ; + cd ${nmun}-${vmun}; + sed -i bootstrap.sh \ + -e 's/if.*then/if false; then/g' \ + ; + ./bootstrap.sh \ + bootstrap \ + ; + bootstrap/muon setup \ + -Dstatic=true \ + build \ + ; + sed -i build/build.ninja \ + -e "s@\bar\b@${AR}@g" \ + ; + samu -C build; + cp build/muon "${DEST}/bin"; +) +touch ._${nmun}-${vmun}; +rm -fr ${nmun}-${vmun}; + + +## +# apk-tools +# +# Ariadne says use meson, and 'muon' doesn't work so... kludges! +# Also, '-j' will break the build. +# +natl=apk-tools; +vatl=be4ce40797af9056c79be4dc74ff978f1f4957e4; +test ! -f ._${natl}-${vatl} && \ +( + test ! -d ${natl}-${vatl} \ + && git clone https://git.alpinelinux.org/${natl} ${natl}-${vatl} \ + ; + cd ${natl}-${vatl}; + sed -i Make.rules \ + -e '/targets += $(__shlibs) $(shobjs)/d' \ + ; # disable shared libs + sed -i src/Makefile \ + -e 's/$(install-libapk_so)//g' -e 's/$(libapk_so)//g' \ + -e 's/ version.o/ version.o strlcpy.o/' \ + ; # disable shared libs, hack the hack + sed -i src/context.c \ + -e "s@var/log@${DEST}/${1}/var/log@" \ + ; # hardcoded + ln -sf ../portability/strlcpy.c src/strlcpy.c; + export LUA=no; # documentation requires lua + make clean; + make \ + INSTALLDIR="${DEST}" \ + CFLAGS="-I${DEST}/include -DNEED_STRLCPY -Wno-error" \ + LDFLAGS="-L${DEST}/lib -L${DEST}/${natl}-${vatl}/libfetch" \ + LIBS="-lapk -lfetch -lssl -lcrypto -lz" \ + ; + cp src/apk "${DEST}/bin"; +) +touch ._${natl}-${vatl}; +rm -fr ${natl}-${vatl}; + + +## +# pax-utils +# +npax=pax-utils; +vpax=974b9359c2f89d57e69598572aafcd8f920d79e2; +test ! -f ._${npax}-${vpax} && \ +( + test ! -d ${npax}-${vpax} \ + && git clone https://anongit.gentoo.org/git/proj/${npax}.git ${npax}-${vpax} \ + ; + cd ${npax}-${vpax}; + git checkout ${vpax}; + muon setup build; +) +touch ._${npax}-${vpax}; +rm -fr ${npax}-${vpax}; + + +## +# fakeroot +# +nfrt=fakeroot; +vfrt=8c0260009e85264fd1ea282fbb22063fc694c552; # until autoconf 2.71 +test ! -f ._${nfrt}-${vfrt#*:} && \ +( + test ! -d ${nfrt}-${vfrt} \ + && git clone https://salsa.debian.org/clint/${nfrt}.git ${nfrt}-${vfrt} \ + ; + cd ${nfrt}-${vfrt}; + git checkout ${vfrt}; + ./bootstrap; + f=$(mktemp); # needed due to "error: unknown type name 'cap_user_header_t'" + echo > ${f} "#include <linux/capability.h>"; + cat libfakeroot.c >> ${f}; + mv ${f} libfakeroot.c; + rm -fr x; mkdir x; cd x; + # -D_ID_T is for "error: conflicting types for 'id_t'; have 'int'" + CFLAGS="-D_ID_T" \ + ../configure \ + --prefix="${DEST}" \ + ; + sed -i Makefile \ + -e '/^SUBDIRS =/ s/doc//g' \ + ; + make -j$(nproc) install; +) +touch ._${nfrt}-${vfrt#*:}; +rm -fr ${nfrt}-${vfrt#*:}; diff --git a/scripts/bootstrap-host b/scripts/bootstrap-host new file mode 100755 index 000000000..7619f68a9 --- /dev/null +++ b/scripts/bootstrap-host @@ -0,0 +1,4 @@ +#!/bin/sh -e + +cd "${BASE}"/bootstrap; +./bootstrap; diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index d5334b11b..21156c5ad 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,6 +1,14 @@ #!/bin/sh -e #=============================================================== +# Filename : scripts/bootstrap.sh (part of Adélie package repo). +# Purpose : Bootstraps Adélie from source for any architecture. +# Authors : Zach van Rijn <me@zv.io> +# License : MIT +# Revision : 20221206 +#=============================================================== + +#=============================================================== # README #=============================================================== # @@ -11,10 +19,109 @@ # contains standard system utilities, this script bootstraps the # Adélie Linux distribution for any suported target architecture # (assuming that musl, gcc, etc. have been ported to it) without -# requiring 'root' privileges. The procedure is outlined below: +# requiring 'root' privileges. This process is slow* and costly, +# but demonstrates that bootstrapping from source is possible. +# +# Effort has been focused on correctness instead of performance. +# This means, some files can be deleted sooner and some of the +# binaries can run natively on the build machine. We know and do +# appreciate there are different opinions on how to approach the +# bootstrap process. This meets our needs and hopefully offers a +# different perspective. +# +# This process takes up a lot of TIME and SPACE, and this cannot +# be improved by throwing hardware at it. This is primarily due +# to the tradeoff of not requiring root privileges at any point, +# and the decision to emulate a native environment instead of to +# force explicit cross-compilation at each step in the process. +# +# (*) See the "requirements" section for mitigations/discussion. +# +# +# features +# -------- +# +# * One-click bootstrap. Just start the script and walk away. +# +# $ ./scripts/bootstrap.sh ppc64 /path/to/new/scratch/dir +# +# * Can be done without root privileges: no 'chroot(8)'. +# +# * This is the real deal. No "seed" binaries are required to +# go from start to finish. The final output is what we use +# to set up our official build servers totally* from source. +# +# * Can be done on a wide range of platforms, almost certainly +# those supported by mcmtools (want to contribute???). +# +# * Minimal dependencies. Nothing fancy. Shell scripts only. +# +# * Can be adapted to assist with porting Adélie to new target +# architectures. TODO: write a tool to automatically do it. +# +# (*) We don't count binaries in your starting environment, and +# recommend https://www.gnu.org/software/mes/ for the brave. +# +# +# requirements +# ------------ +# +# TL;DR: You must be able to run the mcmtools bootstrap script. +# +# mcmtools is a hard dependency for our bootstrap process now: +# +# https://git.zv.io/toolchains/bootstrap +# +# It is a simple analog to Buildroot or Yocto. Those tools could +# be used, too, and would provide the "host rootfs" environment. +# +# Internet access is required, but (as an exercise left to the +# reader) it is possible to pre-download all required sources if +# you provide your own rootfs for the static QEMU build process. +# +# If you are in a position to use native hardware, then you are +# able to get away with only the final stages of bootstrapping. +# To do this, you'd essentially comment out the first stages or +# copy the results of the first stages elsewhere and continue. +# +# Other requirements that you should be aware of, estimated: +# +# * As many CPU cores as you can throw at it; +# +# * ~15 GB for each rootfs ("seed" and "host"), so ~30GB; +# +# * ~ 3 GB for toolchains; +# +# * ~ 2 GB for the "system/" package repository, when built; +# +# * Please refer to the README in 'bootstrap', linked above, +# for more information about performance. Most of this can +# be gained back if you adapt this script to use 'chroot(8)' +# instead of 'PRoot', at the expense of requiring privilege, +# with the correct registration of QEMU with 'binfmt_misc'. +# +# In brief, there is an approximate factor of 13 slowdown on +# practical workloads when using QEMU user and 'PRoot' to do +# dynamic binary translation and emulate a foreign machine. +# +# +# process +# ------- +# +# The illustration below outlines the complete bootstrap process +# and roughly corresponds to the script layout/organization. Our +# terminology is not perfectly consistent; please excuse this. +# +# The term "build" is shorthand for "build-native" CPU, which +# is the machine on which you are performing the bootstrap. +# +# The term "native" is shorthand for "foreign-native" CPU, which +# is the machine to which you are targeting the bootstrap, and +# relates to the "host" CPU on which the code will run, but that +# with the help of dynamic binary translation, runs "natively". # # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# (unstable, everything provided by user) +# build (unstable, everything provided by user) # # +-----------------+ User-provided tools. Dependency # | Baseline System | of 'mcmtools', which will verify @@ -26,7 +133,7 @@ # | host-arch musl-libc toolchain... # | # - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - -# | (stable versions, unstable libc) +# mixed | (stable versions, unstable libc) # | # +-------------+ ...and a 'chroot'-able rootfs. A # | seed rootfs | sane, but not clean, environment @@ -36,51 +143,101 @@ # | emulators | dependencies, as well as static # +-----------+ QEMU user binaries. Add to seed. # | -# +---------------+ Static musl libc cross toolchain -# | musl cross tc | targeting a given architecture. -# +---------------+ Output binaries will run via the +# +-----------------+ Static musl toolchains targeting +# | musl toolchains | a given architecture: cross from +# +-----------------+ the host CPU & foreign "native". +# | Output binaries will run via the # | 'binfmt_misc' mechanism + QEMU. # | # +-------------+ Script to build Alpine Package # | build tools | Keeper (APK) and dependencies. # +-------------+ All binaries are cross-compiled! +# | This step is a sanity check. +# | +# +----------+ Script to build pinned versions +# | mcmtools | of common system utilities. This +# +----------+ time, all utilities are target- +# | native. Build more toolchains... +# | +# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - +# host | (stable versions, stable libc) # | +# +-------------+ ...and a 'chroot'-able rootfs. A +# | host rootfs | sane, clean, foreign "native" +# +-------------+ rootfs that requires QEMU, or is +# | able to run on native hardware. +# | +# +-------------+ Script to build Alpine Package +# | build tools | Keeper (APK) and dependencies. +# +-------------+ These binaries are native built! +# | This step is required! +# | +# +-------------+ Script to build the Adélie Linux +# | system repo | "system/" package repository. It +# +-------------+ is used to build core packages. +# | +# +---------------+ Script to install packages into +# | image creator | a clean rootfs. Carryover from +# +---------------+ the mcmtools process is removed. +# | +# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - +# verify | (optional verification) +# | +# ... The "host" stage can be repeated +# | zero or more times to ensure the +# | final image is not contaminated. +# | +# - - - - -|- - - - - - - - - - - - - - - - - - - - - - - - +# output | (Welcome to Adélie Linux) # | -# +-------------+ Script to build the minimum set -# | seed rootfs | of tools to start building the -# +-------------+ "system/" package repository. +# +---------------+ This is a minimal Adélie rootfs. +# | adelie rootfs | Copy it to native hardware, then +# +---------------+ use it as a builder 'chroot'. It +# is used to seed official Adélie +# build boxen (e.g. autobuilder). # -# mcmtools; a separate script -# that -# mcmtools provides almost everything needed to build 'abuild', -# and it is a hard dependency for our bootstrap process now. +# limitations +# ----------- # -# https://git.zv.io/toolchains/bootstrap +# The builds are not hermetically sealed. That is not the point. +# The build environment should not *need* anything from outside, +# but it is not *prevented* from accessing anything. You should +# run this on a clean, trusted machine. +# +# There is no guarantee of byte-for-byte reproducible builds at +# this time. This is, in part, due to timestamps and tar headers +# but may involve a lack of determinism in parallel builds. # -# The output of this step is a host-native toolchain with fixed -# versions of development utilities, and a host-architecture hosted toolchain that cross-compiles to both musl and/or -# a foreign CPU architecture. (Table above prevents this). +# QEMU user emulation may cause subtle incompatibilites with the +# target CPU hardware. For example, CPU feature (mis)detection. # +# It is not currently possible to safely reuse any of the output +# if the target architecture is changed. You will need to start +# from scratch if you wish to change the target. This is a TODO. +# HERE="$(dirname $(readlink -f ${0}))"; -TEMP="$(mktemp -d)"; # do not change if not expert! + #--------------------------------------------------------------- # initialization +## +# Haaaalp! +# usage () { - printf "Usage: %s ARCH\n\n" "${0}"; + printf "Usage: %s ARCH BASE\n\n" "${0}"; cat <<EOF - ARCH is { aarch64, armv7, ppc64, ppc, x86_64, pmmx } + ARCH { aarch64, armv7, ppc64, ppc, x86_64, pmmx } -Optional environment variables: - - MCMTOOLS=/path/to/existing/mcmtools/ **MAY CAUSE DATA LOSS** + BASE an absolute path to bootstrap out of. + ** MAY CAUSE DATA LOSS IF SET INCORRECTLY! ** EOF exit 0; } + ## # argv[1]: ARCH # @@ -91,19 +248,35 @@ EOF # ARCH is translated to canonical GCC and QEMU machine types. # case "${1}" in -# adelie gcc qemu -# ------ --- ---- - aarch64) m=aarch64; q=aarch64; ;; - armv7) m=armv7l ; q=arm ; ;; - ppc) m=ppc ; q=ppc ; ;; - ppc64) m=ppc64 ; q=ppc64 ; ;; - x86_64) m=x86_64 ; q=x86_64 ; ;; - pmmx) m=i686 ; q=i386 ; ;; +# adelie gcc qemu +# ------ --- ---- + aarch64) m=aarch64 ; q=aarch64 ; ;; + armv7) m=armv7l ; q=arm ; ;; + ppc) m=powerpc ; q=ppc ; ;; + ppc64) m=powerpc64 ; q=ppc64 ; ;; + x86_64) m=x86_64 ; q=x86_64 ; ;; + pmmx) m=i586 ; q=i386 ; ;; *) usage ;; esac shift; -test ! -n "${m}" && printf "Invoking '%s TARGET' where 'TARGET=%s' is not valid.\n" "${0}" "${m}" && exit 1; + +## +# argv[2]: BASE +# +# BASE is a semi-permanent scratch directory. It is where all of +# the magic happens, and probably cannot be relocated easily. Be +# careful to not set it incorrectly or to a place you'd regret +# being overwritten, corrupted, or deleted. +# +case "${1}" in + /*) BASE="${1}"; + ;; + *) printf "BASE not set, or not an absolute path!\n"; + exit 1; + ;; +esac + ## # Internal variables. Do not modify this section directly. @@ -123,12 +296,17 @@ printf "BUILDS=%s\n" "${BUILDS}"; TARGET="${m}${SYSTEM}"; # ultimate Adélie Linux target printf "TARGET=%s\n" "${TARGET}"; -MTOOLS=${MCMTOOLS:-"${TEMP}/mcmtools"}; # CAREFUL! MAY CAUSE DATA LOSS! +MTOOLS=${MCMTOOLS:-"${BASE}/mcmtools"}; # CAREFUL! MAY CAUSE DATA LOSS! printf "MTOOLS=%s\n" "${MTOOLS}"; +## +# Default 'PATH' for use inside various rootfs environments. +# +DEF_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + #--------------------------------------------------------------- -# mcmtools +# mcmtools (seed) ## # Allow the user to supply an existing mcmtools installation. It @@ -138,19 +316,22 @@ printf "MTOOLS=%s\n" "${MTOOLS}"; # is used if this environment variable is omitted. Another case # for providing a custom value is if '/tmp' is mounted weird. # -if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code - cd "${TEMP}"; +if ! test -d "${MTOOLS}/sys/emus/bin"; then # FIXME: no hard code + cd "${BASE}"; test -d bootstrap \ || git clone ${CHAINS}/bootstrap.git; cd bootstrap; - git checkout 773363265da441f472c207fb59c1acecc8ba2eb4; + git checkout d148555321a391fd6f23a289fa51c2dfa26f4b10; ## seed rootfs # # This will build absolutely everything that is needed to be # self-reliant, except for some build deps for QEMU. # + # We copy 'config.mak' from mcmtools bootstrap to the rootfs + # so that when we build "real" toolchain they are the same. + # test -d "${MTOOLS}" || \ DEST="${MTOOLS}" \ ARCH=${BUILDS} \ @@ -160,9 +341,19 @@ if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code cp "${MTOOLS}"/tmp/musl-cross-make/config.mak \ "${MTOOLS}"/config.mak \ ; - - ## - # emulators + #rm -fr "${MTOOLS}"/tmp; # save 10 GB (FIXME: make safe) + + # is any of this actually needed? + ( + cd "${MTOOLS}"/sys; + mkdir -p dev; + mkdir -p proc; + mkdir -p sys; + rm -fr usr; + ln -s . usr; + ) + + ## emulators # # Dependencies are built with the mcmtools host toolchain; a # reason to not force musl here is in the event that these @@ -174,7 +365,7 @@ if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code # manually provide your own static QEMU user binaries and be # on your way without Alpine, but it is a good 'PRoot' test. # - test -d "${MTOOLS}/emus/bin" || \ + test -d "${MTOOLS}/sys/emus/bin" || \ PATH="${MTOOLS}/host/bin:${MTOOLS}/sys/bin" \ DEST="${MTOOLS}" \ ./prootemu \ @@ -182,225 +373,124 @@ if ! test -d "${MTOOLS}/emus/bin"; then # FIXME: no hard code fi +#--------------------------------------------------------------- +# musl toolchains + ## -# Now we have a musl-targeting toolchain that runs on the host. -# We need to build the same toolchain, but static. +# We have a musl-targeting toolchain that runs on the host, but +# it is linked to the host libc and may not run in this chroot. +# +# We need to build the same toolchain, but static. There are two +# possible directions to go: (1) a cross toolchain that runs at +# full speed on the build machine, at the cost of having to tell +# downstream build scripts how to cross compile, or (2) a native +# toolchain for the foreign (target) architecture that runs slow +# in QEMU and still requires the cross toolchain to build it. # -# This is required for it to run inside 'PRoot' if it differs in -# architecture. +# Build both. We will eventually need both toolchains, anyway. # -if ! test -d "${MTOOLS}/sys/tc/cross"; then # FIXME: no hard code - cd "${TEMP}"; +if ! test -d "${MTOOLS}/sys/tc/native"; then # FIXME: no hard code + cd "${BASE}"; test -d musl-cross-make \ || git clone ${CHAINS}/musl-cross-make.git; cd musl-cross-make; git pull; # always use the latest + ## + # Ensure consistent 'config.mak' for all toolchain builds. + # cp "${MTOOLS}"/config.mak config.mak; - ## musl cross tc + ## musl toolchains # - # Build this toolchain statically using the musl toolchain + # Build these toolchains statically using the musl toolchain # from the seed rootfs so that it is known to work correctly # (the original musl toolchain itself may itself be linked # with glibc or be unsafe to use in some contexts). # - PATH="${MTOOLS}/musl/bin:${MTOOLS}/sys/bin" \ - ./scripts/buildcross ${TARGET} \ - ; - cd output; - mkdir -p "${MTOOLS}"/sys/tc/cross; - tar -C "${MTOOLS}"/sys/tc/cross \ - --strip-components=1 \ - -xzf ${TARGET}-cross.tgz \ + # Note: "native" is for the foreign target CPU architecture. + # + PATH="${MTOOLS}/musl/bin:${MTOOLS}/sys/bin" \ + ./scripts/build ${TARGET} \ ; + for k in cross native; do + rm -fr "${MTOOLS}"/sys/tc/${k}; + mkdir "${MTOOLS}"/sys/tc/${k}; + tar -C "${MTOOLS}"/sys/tc/${k} \ + --strip-components=1 \ + -xzf output/${TARGET}-${k}.tgz \ + ; + done fi -## -# Finish setting up the rootfs to support what we're doing. -# -# FIXME: what of this is actually needed? -# -( - cd "${MTOOLS}"/sys; - mkdir -p dev; - mkdir -p proc; - mkdir -p sys; - rm -fr usr; - ln -s . usr; -) +#--------------------------------------------------------------- +# build tools (cross) ## # Build 'abuild', its dependencies, and other utilities. -# Once finished, add them to PATH. -# - -PROOT_NO_SECCOMP=1 \ -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tc/cross/bin \ -CROSS_COMPILE=${TARGET}- \ -SHELL=/bin/sh \ -DEST=/usr/local \ -CURL_CA_BUNDLE=/cacert.pem \ -${MTOOLS}/sys/emus/bin/proot \ - -S "${MTOOLS}"/sys \ - -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \ - -b "${HERE}" \ - "${HERE}"/setup-abuild \ +# +PROOT_NO_SECCOMP=1 \ +PATH="/tc/cross/bin:${DEF_PATH}" \ +SHELL=/bin/sh \ +DEST=/usr/local \ +CURL_CA_BUNDLE=/cacert.pem \ +CROSS_COMPILE=${TARGET}- \ +CC=${TARGET}-gcc \ +CXX=${TARGET}-g++ \ +AR=${TARGET}-ar \ +LD=${TARGET}-ld \ +CPP=${TARGET}-cpp \ +${MTOOLS}/sys/emus/bin/proot \ + -S "${MTOOLS}"/sys \ + -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \ + -b "${HERE}" \ + "${HERE}"/bootstrap-abuild \ ; -echo ok -exit 0; - -export PATH="${MTOOLS}/abuild/bin:${PATH}"; -bash - -## -# Additional configuration. -# -export PACKAGER="Zach van Rijn <me@zv.io>"; -export CBUILD=${TARGET_ARCH}; -export CBUILDROOT="${MCMTOOLS}/abuild/${TARGET_ARCH}"; -export ABUILD_USERDIR="${CBUILDROOT}.conf"; -export SRCDEST="${MCMTOOLS}/abuild/src"; -export REPODEST="${MCMTOOLS}/abuild/apk"; -export ABUILD_APK_INDEX_OPTS="--allow-untrusted"; # FIXME -#export BUILD_ROOT="${CBUILDROOT}"; - -SUDO_APK=abuild-apk - -# get abuild configurables -[ -e "${MCMTOOLS}/abuild/share/abuild/functions.sh" ] || (echo "abuild not found" ; exit 1) -CBUILDROOT="$(CTARGET=$TARGET_ARCH . ${MCMTOOLS}/abuild/share/abuild/functions.sh ; echo $CBUILDROOT)" -. "${MCMTOOLS}/abuild/share/abuild/functions.sh" -[ -z "$CBUILD_ARCH" ] && die "abuild is too old (use 2.29.0 or later)" -[ -z "$CBUILDROOT" ] && die "CBUILDROOT not set for $TARGET_ARCH" - -# deduce aports directory -[ -z "$APORTS" ] && APORTS=$(realpath $(dirname $0)/../) -[ -e "$APORTS/system/build-tools" ] || die "Unable to deduce packages directory" - -apkbuildname() { - local repo="${1%%/*}" - local pkg="${1##*/}" - [ "$repo" = "$1" ] && repo="system" - echo $APORTS/$repo/$pkg/APKBUILD -} - -msg() { - [ -n "$quiet" ] && return 0 - local prompt="$GREEN>>>${NORMAL}" - local name="${BLUE}bootstrap-${TARGET_ARCH}${NORMAL}" - printf "${prompt} ${name}: %s\n" "$1" >&2 -} - -if [ -z "$TARGET_ARCH" ]; then - program=$(basename $0) - cat <<EOF -usage: $program TARGET_ARCH - -This script creates a local cross-compiler, and uses it to -cross-compile an Adélie Linux base system for new architecture. - -Steps for introducing new architecture include: -- adding the compiler tripler and arch type to abuild -- adding the arch type detection to apk-tools -- adjusting build rules for packages that are arch aware: - gcc, musl, binutils, easy-kernel -- create new kernel config for easy-kernel - -After these steps the initial cross-build can be completed -by running this with the target arch as parameter, e.g.: - ./$program aarch64 - -EOF - return 1 -fi +#--------------------------------------------------------------- +# mcmtools (host) ## -# Package signing keys. Public and Private keys are stored in a -# different location; variables for which are installed to arch- -# specific 'abuild.conf' file. -# -if [ ! -d "$CBUILDROOT/etc/apk/keys" ] || [ -n "$(find $CBUILDROOT -type f -name '*.rsa')" ]; then - msg "Creating sysroot in $CBUILDROOT" - mkdir -p "$CBUILDROOT/etc/apk/keys" - abuild-keygen -an; - p=$(find "${ABUILD_USERDIR}" -type f -name "*.rsa.pub"); - mv "${p}" "$CBUILDROOT/etc/apk/keys"; - grep 1>/dev/null PACKAGER_PUBKEY= "${ABUILD_USERDIR}/abuild.conf" || printf >> "${ABUILD_USERDIR}/abuild.conf" "PACKAGER_PUBKEY=\"%s\"\n" "$CBUILDROOT/etc/apk/keys/${p##*/}"; -fi +# Build the 'system/' repository. +# +# NOTE: The 'PATH' order is *really* important. Foreign "native" +# toolchains must be the first ones found; e.g. '/usr/bin/gcc' +# is a symlink to 'ccache' and isn't a functional compiler. +# +# Once 'coreutils' is built, 'uname' will return correctly, then +# future software should(tm) build as if it is built natively. +# +# FIXME: is Linux 3.2.0 really appropriate here? +# +PROOT_NO_SECCOMP=1 \ +PATH="/tc/native/bin:${DEF_PATH}" \ +SHELL=/bin/sh \ +BASE="${BASE}" \ +DEST="${BASE}"/mcmtools-${TARGET} \ +ARCH=${TARGET} \ +CURL_CA_BUNDLE=/cacert.pem \ +${MTOOLS}/sys/emus/bin/proot \ + -S "${MTOOLS}"/sys \ + -q "${MTOOLS}"/sys/emus/bin/qemu-${q} \ + -b "${HERE}" \ + -k "3.2.0" \ + "${HERE}"/bootstrap-host \ + ; -## -# APK database. -# -if [ ! -f "${CBUILDROOT}/._database-${TARGET_ARCH}" ]; then - mkdir -p "${CBUILDROOT}/var/log"; # why not created by default? - ${SUDO_APK} add --quiet --initdb --arch $TARGET_ARCH --root $CBUILDROOT - touch "${CBUILDROOT}/._database-${TARGET_ARCH}"; -fi +#--------------------------------------------------------------- +# system repository -msg "Building cross-compiler" +# TODO -# Build and install cross binutils (--with-sysroot) -CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname binutils) abuild -r -exit -if ! CHOST=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname musl) abuild up2date 2>/dev/null; then - # C-library headers for target - CHOST=$TARGET_ARCH BOOTSTRAP=nocc APKBUILD=$(apkbuildname musl) abuild -r - # Minimal cross GCC - EXTRADEPENDS_HOST="musl-dev" \ - CTARGET=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname gcc) abuild -r +#--------------------------------------------------------------- +# image creator - # Cross build bootstrap C-library for the target - EXTRADEPENDS_BUILD="gcc-pass2-$TARGET_ARCH" \ - CHOST=$TARGET_ARCH BOOTSTRAP=nolibc APKBUILD=$(apkbuildname musl) abuild -r -fi +# TODO -# Full cross GCC -EXTRADEPENDS_TARGET="musl musl-dev" \ -CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname gcc) abuild -r - -# Cross build-base -CTARGET=$TARGET_ARCH BOOTSTRAP=nobase APKBUILD=$(apkbuildname build-base) abuild -r - -msg "Cross building base system" -msg "Change your abuild.conf NOW to avoid build errors!!" -read - -# add implicit target prerequisite packages -apk info --quiet --installed --root "$CBUILDROOT" libgcc libstdc++ musl-dev || \ - ${SUDO_APK} --root "$CBUILDROOT" add --repository "$REPODEST/system" libgcc libstdc++ musl-dev - -# ordered cross-build -for PKG in musl pkgconf zlib \ - gettext-tiny ncurses bash binutils make bison flex m4 \ - openssl apk-tools linux-pam shadow \ - gmp mpfr3 mpc1 isl gcc ca-certificates \ - openrc libcap-ng coreutils sed gzip bzip2 diffutils \ - attr libcap patch sudo acl fakeroot libarchive mawk \ - pax-utils abuild grep findutils patch lzip unzip autoconf automake libtool \ - ncurses util-linux lvm2 popt xz lddtree libssh2 curl build-tools pcre \ - debianutils file shimmy procps zsh sharutils net-tools check kbd sysklogd vim db groff libpipeline man-db psmisc less \ - adelie-base \ - ; do - - CHOST=$TARGET_ARCH BOOTSTRAP=bootimage APKBUILD=$(apkbuildname $PKG) abuild -r - - case "$PKG" in - fortify-headers | libc-dev | build-base) - # headers packages which are implicit but mandatory dependency - apk info --quiet --installed --root "$CBUILDROOT" $PKG || \ - ${SUDO_APK} --update --root "$CBUILDROOT" --repository "$REPODEST/system" add $PKG - ;; - musl | gcc) - # target libraries rebuilt, force upgrade - [ "$(apk upgrade --root "$CBUILDROOT" --repository "$REPODEST/main" --available --simulate | wc -l)" -gt 1 ] && - ${SUDO_APK} upgrade --root "$CBUILDROOT" --repository "$REPODEST/main" --available - ;; - esac -done +# apk --root /foo --arch armv7 --initdb add +# apk --root /foo add adelie-core dash-binsh build-tools diff --git a/scripts/setup-abuild b/scripts/setup-abuild index 4333113a8..f35ddd114 100755..100644 --- a/scripts/setup-abuild +++ b/scripts/setup-abuild @@ -53,7 +53,7 @@ test ! -f ._${nmus}-${vmus} && \ cd ${nmus}-${vmus}; rm -fr x; mkdir x; cd x; ../configure \ - --prefix="${DEST}" \ + --prefix=/usr \ --enable-static \ --enable-shared \ ; |