unvendor
This commit is contained in:
Dawid Sobczak 2025-04-18 12:41:04 +01:00
parent 9a4b261179
commit 404aa8ebbb
139 changed files with 8091 additions and 1178 deletions

70
06/helpers/builddir Executable file
View file

@ -0,0 +1,70 @@
#!/bin/sh
: ${DISORDER=0}
set -uex
verb=$1
front=$2
back=$front-back
[ "tmp/build/${front#tmp/build/}" = "$front" ]
[ "tmp/build/${back#tmp/build/}" = "$back" ]
remove_hard() {
if [ -e "$1" ]; then
sudo umount "$1" 2>/dev/null || true
if ! rm -rf "$1" 2>/dev/null; then
chmod -R +w "$1"
rm -rf "$1"
fi
fi
}
if [ "$verb" = create ]; then
echo "### helpers/builddir: creating $front..."
remove_hard "$front"; remove_hard "$back"
mkdir -p "$front"
if [ "$DISORDER" = 1 ]; then
sudo umount "$back" || true
rm -rf "$back"
mkdir -p "$front" "$back"
sudo mount -o size=16G -t tmpfs tmpfs "$back"
sudo mount --bind "$back" "$front"
[ $(findmnt -no FSTYPE "$front") = tmpfs ]
fi
elif [ "$verb" = pre-build ]; then
[ -e "$front" ]
if [ "$DISORDER" = 1 ]; then
echo "### helpers/builddir: disordering $front..."
[ -e "$back" ]
[ $(findmnt -no FSTYPE "$back") = tmpfs ]
sudo umount "$front"
sudo disorderfs --shuffle-dirents=yes \
-o allow_other --multi-user=yes \
"$back" "$front"
[ $(findmnt -no FSTYPE "$front") = fuse.disorderfs ]
fi
elif [ "$verb" = post-build ]; then
if [ "$DISORDER" = 1 ]; then
echo "### helpers/builddir: ordering $front..."
[ ! -e "$front" ] && [ ! -e "$back" ]
[ $(findmnt -no FSTYPE "$front") = fuse.disorderfs ]
[ $(findmnt -no FSTYPE "$back") = tmpfs ]
sudo umount "$front"
sudo mount --bind "$back" "$front"
fi
elif [ "$verb" = remove ]; then
echo "### helpers/builddir: removing $front..."
if [ "$DISORDER" = 1 ]; then
[ $(findmnt -no FSTYPE "$back") = tmpfs ]
sudo umount "$front"
sudo umount "$back"
rm -d "$front" "$back"
else
remove_hard "$front"; remove_hard "$back"
fi
[ ! -e "$front" ] && [ ! -e "$back" ]
else
exit 9
fi

49
06/helpers/cheat Executable file
View file

@ -0,0 +1,49 @@
#!/bin/sh
set -ue
NIXPKGS=nixpkgs/21f524672f25f8c3e7a0b5775e6505fee8fe43ce
: ${DESTDIR:=stage}
mkdir -p $DESTDIR/cheat
if [ ! -e $DESTDIR/cheat/make ]; then
nix build "$NIXPKGS#pkgsStatic.gnumake"
cp result/bin/make $DESTDIR/cheat/make
rm result
fi
if [ ! -e $DESTDIR/cheat/bash ]; then
nix build "$NIXPKGS#pkgsStatic.bash"
cp result/bin/bash $DESTDIR/cheat/bash
rm result
fi
if [ ! -e $DESTDIR/cheat/strace ]; then
nix build "$NIXPKGS#pkgsStatic.strace"
cp result/bin/strace $DESTDIR/cheat/
rm result
fi
if [ ! -e $DESTDIR/cheat/busybox ]; then
nix build "$NIXPKGS#pkgsStatic.busybox"
cp result/bin/busybox $DESTDIR/cheat/busybox
for f in $(ls result/bin/); do
[ $(basename $f) = busybox ] ||
ln -s /cheat/busybox $DESTDIR/cheat/$(basename $f)
done
rm result
fi
if [ -z "$@" ]; then
_PATH=''
for bindir in $DESTDIR/store/*/bin $DESTDIR/store/*/*/bin; do
_PATH="${bindir##$DESTDIR}:$_PATH"
done
LIBC=$(find $DESTDIR/store/*/lib/libc.so | tail -n1)
if [ -n "$LIBC" ] && [ ! -h $DESTDIR/cheat/ldd ]; then
ln -s "${LIBC##$DESTDIR}" $DESTDIR/cheat/ldd
fi
helpers/chroot /cheat/env "PATH=$_PATH" /cheat/ash
else
helpers/chroot "$@"
fi

18
06/helpers/chroot Executable file
View file

@ -0,0 +1,18 @@
#!/usr/bin/env bash
# Run a command chrooted inside $DESTDIR w/o network, with /dev/null, outline:
#
# [helpers/chroot, outer script]
# / unshare
# | -n # without network
# | -r # with EUID=EGID=0
# \ -m # separate mount namespace
# [helpers/chroot-inner, this script]
# mount --bind /dev/null $DESTDIR/dev/null # unprivileged /dev/null!
# &&
# env -i # with env unset
# chroot $DESTDIR # unprivileged chroot!
set -uex
exec unshare -nrm helpers/chroot-inner "$@"

40
06/helpers/chroot-inner Executable file
View file

@ -0,0 +1,40 @@
#!/usr/bin/env bash
# Run a command chrooted inside $DESTDIR w/o network, with /dev/null, outline:
#
# [helpers/chroot, outer script]
# unshare
# -n # without network
# -r # with EUID=EGID=0
# -m # separate mount namespace
# [helpers/chroot-inner, this script]
# / mount --bind /dev/null $DESTDIR/dev/null # unprivileged /dev/null!
# | &&
# | env -i # with env unset
# \ chroot $DESTDIR # unprivileged chroot!
set -uex
: ${DESTDIR:=stage}
: ${NPROC:=1}
: ${SOURCE_DATE_EPOCH:=0}
CHROOT=$(command -v chroot)
if [[ ! -x "$CHROOT" ]]; then
if [[ -x /sbin/chroot ]]; then
CHROOT=/sbin/chroot
elif [[ -x /usr/sbin/chroot ]]; then
CHROOT=/usr/sbin/chroot
fi
fi
if [[ -e /run/wrappers/bin/mount.real ]]; then
MOUNT=$(cat /run/wrappers/bin/mount.real)
else
MOUNT=mount
fi
mkdir -p "$DESTDIR/dev"; :> "$DESTDIR/dev/null"
$MOUNT --bind /dev/null "$DESTDIR/dev/null"
exec env -i "NPROC=$NPROC" "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" \
$CHROOT "$DESTDIR" "$@"

20
06/helpers/inject Executable file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -uex
DESTDIR=$1
shift
for arg; do
if [[ "$arg" =~ .*\.pkg ]]; then
# That one's for package dependencies
tar -xf "$arg" -Izstd -C "$DESTDIR"
else
# That one's for copying over downloads and sources
if [[ ! -e "./$DESTDIR/$arg" || "$arg" -nt "./$DESTDIR/$arg" ]]
then
echo "Copying $arg into $DESTDIR..."
mkdir -p "$DESTDIR/$(dirname "$arg")"
cp -a --reflink=auto "$arg" "$DESTDIR/$arg"
fi
fi
done

View file

@ -0,0 +1,97 @@
#!/usr/bin/env bash
set -uexo pipefail
CUSTOM_CORE_DIR=$1
# Determine where to start from
if [[ -e pkgs/custom-stage5.pkg ]]; then
BASE=pkgs/custom-stage5.pkg
elif [[ -e pkgs/5-go-beyond-using-nix.pkg ]]; then
BASE=pkgs/5-go-beyond-using-nix.pkg
elif [[ -e pkgs/4-rebootstrap-using-nix.pkg ]]; then
BASE=pkgs/4-rebootstrap-using-nix.pkg
else
[[ "${USE_CCACHE-}" = 1 ]] && FLAGS='USE_CCACHE=1' || FLAGS=''
[[ -n "${NPROC-}" ]] && FLAGS="$FLAGS NPROC=$NPROC" || FLAGS=''
make $FLAGS pkgs/5-go-beyond-using-nix.pkg
BASE=pkgs/5-rebootstrap-using-nix.pkg
fi
# Inject stage5 pkg dependencies
DESTDIR=tmp/build/custom-stage5
helpers/builddir create "$DESTDIR"
grep -E '^pkgs/5-go-beyond-using-nix.pkg:' Makefile \
| grep -v '^pkgs/5-go-beyond-using-nix.pkg: downloads/' \
| grep -v 'pkgs/5-go-beyond-using-nix.pkg: pkgs/_2a0-ccache.pkg$' \
| sed 's|^pkgs/5-go-beyond-using-nix.pkg: ||' \
| while IFS= read -r dep; do helpers/inject "$DESTDIR" "$dep"; done
helpers/inject "$DESTDIR" recipes/5-go-beyond-using-nix.sh
# Inject BASE and its nixdb
helpers/inject "$DESTDIR" "$BASE"
mkdir -p "$DESTDIR/prev/"
tar --strip-components=2 -xf $BASE -C "$DESTDIR/prev/"
rm -f "$DESTDIR/store/5-go-beyond-using-nix/ZilchOS-core.iso"
# Inject ccache data, all stage5 ccache data we can find
if [[ "${USE_CCACHE-}" = 1 ]]; then
helpers/inject "$DESTDIR" pkgs/_2a0-ccache.pkg
unshare -nr chroot "$DESTDIR" /store/_2a0-ccache/bin/ccache -z
if [[ -e tmp/ccache/5-go-beyond-using-nix.tar.zstd ]]; then
tar -Izstd -xf tmp/ccache/5-go-beyond-using-nix.tar.zstd \
-C "$DESTDIR/ccache"
fi
if [[ -e tmp/ccache/custom-stage5.tar.zstd ]]; then
tar -Izstd -xf tmp/ccache/custom-stage5.tar.zstd \
-C "$DESTDIR/ccache"
fi
ln -sf /store/_2a0-ccache/wrap-available "$DESTDIR/ccache/setup"
ln -sf /store/_2a0-ccache/bin "$DESTDIR/ccache/bin"
fi
helpers/builddir pre-build "$DESTDIR"
# Inject custom ZilchOS Core code
DESTCORE="$DESTDIR/tmp/5-go-beyond-using-nix/ZilchOS-core"
mkdir -p "$DESTCORE"
rm -rf "$DESTCORE" # TODO: REMOVE
cp -r "$CUSTOM_CORE_DIR" "$DESTCORE"
rm -rf "$DESTCORE/.git"
[[ -e $DESTCORE/flake.nix ]]
hashes=$(cat $CUSTOM_CORE_DIR/.maint/hashes)
# Inject custom required downloads
DESTDIR="$DESTDIR/downloads" ./download.sh $(find $DESTCORE -name '*.nix')
DESTDIR="$DESTDIR" NPROC="${NPROC-}" \
./helpers/chroot /recipes/5-go-beyond-using-nix.sh
cat tmp/build/custom-stage5/store/5-go-beyond-using-nix/hashes \
> nix-checksums-stage5-custom
# Collect results back
tar -Izstd -cf "pkgs/custom-stage5.pkg" \
-C "$DESTDIR" "store/5-go-beyond-using-nix"
# Collect ccache back
if [[ "${USE_CCACHE-}" = 1 ]]; then
unshare -nr chroot "$DESTDIR" /store/_2a0-ccache/bin/ccache -sz
tar -Izstd -cf "tmp/ccache/custom-stage5.tar.zstd" \
-C "$DESTDIR/ccache" .
fi
helpers/builddir post-build "$DESTDIR"
helpers/builddir remove "$DESTDIR"
set +x
status=true
while IFS=' ' read ref_hash pkg; do
stage5_hash=$(grep " $pkg$" nix-checksums-stage5-custom \
| sed -E 's|^([a-z0-9]{32}) .*|\1|');
if [[ "$ref_hash" == "$stage5_hash" ]]; then
echo " $ref_hash $pkg";
else
status=false;
echo "- $ref_hash $pkg";
echo "+ $stage5_hash $pkg";
fi
done <<<"$hashes"; $status

View file

@ -0,0 +1,66 @@
#!/usr/bin/env bash
set -ueo pipefail
COMMIT=${1-HEAD}
commit=$(git show -s --no-notes "$COMMIT")
echo "$commit"
if grep -q '^\s*Builds-required: none$' <<<"$commit"; then
echo '---'
echo '`Builds-required: none` in commit message, skipping commit'
exit 0
fi
echo '---'
if grep -q '^\s*Builds-required: ' <<<"$commit"; then
verification_line=$(grep '^\s*Builds-required:' <<<"$commit")
verspec=$(sed 's|^\s*Builds-required:||' <<<"$verification_line")
if ! grep -Eq '^ make=[0-9]+ raw=[0-9]+ nix=[0-9]+$' <<<"$verspec"; then
echo 'Malformed `Builds-required:` line' >&2
exit 7
fi
required_make=$(sed -E 's|.* make=([0-9]+).*|\1|' <<<"$verspec")
(( required_make >= 0 ))
required_raw=$(sed -E 's|.* raw=([0-9]+).*|\1|' <<<"$verspec")
(( required_raw >= 0 ))
required_nix=$(sed -E 's|.* nix=([0-9]+).*|\1|' <<<"$verspec")
(( required_nix >= 0 ))
else
required_make=2
required_raw=1
required_nix=2
fi
git fetch origin refs/notes/commits:refs/notes/commits
notes=$(git notes show "$COMMIT" \
| grep ^Built-on: \
| grep -v USE_CCACHE \
| sort -u)
using_make=$(grep -Fw how=make <<<"$notes" | wc -l) || true
using_raw=$(grep -Fw how=raw <<<"$notes" | wc -l) || true
using_nix=$(grep -Fw how=nix <<<"$notes" | wc -l) || true
echo "$notes"
status=true
text=""
if (( using_make < required_make )); then
text+="Not enough how=make commits: $using_make < $required_make\n"
status=false
fi
if (( using_raw < required_raw )); then
text+="Not enough how=raw commits: $using_raw < $required_raw\n"
status=false
fi
if (( using_nix < required_nix )); then
text+="Not enough how=nix commits: $using_nix < $required_nix\n"
status=false
fi
echo '---'
echo "${using_make} out of ${required_make} required how=make builds"
echo "${using_raw} out of ${required_raw} required how=raw builds"
echo "${using_nix} out of ${required_nix} required how=nix builds"
if ! $status; then
echo -ne "---\n$text"
fi
$status

29
06/helpers/maint/notes Executable file
View file

@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -ueo pipefail
if [[ $# == 0 || "$1" == 'show' ]]; then
exec git tree --show-notes --decorate=short --color=always \
| grep -v '^|\s*$' \
| grep -v 'Notes:$' \
| sed 's/ Built-on: /^ Built-on: /' \
| less
elif [[ "$1" == 'mark-built' ]] && (( $# >= 3 )) ; then
commit=$2; hostname=$3; how=$4; shift 4; extra="$@"
git fetch origin refs/notes/commits:refs/notes/commits
if ! grep -Eqx 'how=(raw|make|nix)' <<<$how; then
echo 'how must be one of: raw, make, nix' >/dev/stderr
exit 1
fi
msg="Built-on: $hostname $how $extra"
if grep -Fqx "$msg" <(git notes show "$commit" 2>/dev/null); then
echo "$commit is already marked with '$msg'" >/dev/stderr
exit
fi
git notes append -m "$msg" "$commit"
git push origin refs/notes/commits
else
echo 'Usage: ' >/dev/stderr
echo ' helpers/notes [show]' >/dev/stderr
echo -n ' helpers/notes mark-built ' >/dev/stderr
echo '<commit> <hostname> how=[raw|make|nix] <extra stuff>' >/dev/stderr
exit 1
fi

61
06/helpers/maint/release Executable file
View file

@ -0,0 +1,61 @@
#!/usr/bin/env bash
set -uex
# check that two extra files have not been modified
seeder_path=$(nix eval --impure --expr '"${recipes/1-stage1/seed.host-executed.sh}"')
grep "stage1_seeder_reference = $seeder_path;$" using-nix/0.nix
syscall_h_path=$(nix eval --impure --expr '"${recipes/1-stage1/syscall.h}"')
grep "syscall_h_reference = $syscall_h_path;$" using-nix/0.nix
# check that both downloads in using-nix/0-prebuilt refer to the same tag
lines=$(grep 'seeding-files-r[0-9]*' using-nix/0-prebuilt.nix \
| sed 's/.*r\([0-9][0-9][0-9]\).*/\1/')
sort <<<"$lines"
[[ $(wc -l <<<"$lines") == 2 ]]
[[ $(sort <<<"$lines" | uniq | wc -l) == 1 ]]
# build nars, calculate hashes, check both 0-from-nixpkgs and 0-prebuilt
[[ ! -L result* ]]
tinycc_=$(nix-build using-nix/0-from-nixpkgs.nix -A tinycc --no-out-link)
tinycc=$(nix-build using-nix/0-from-nixpkgs.nix -A tinycc --no-out-link --check)
[[ "$tinycc_" == "$tinycc" ]]
nix store dump-path $tinycc > tinycc-liberated.nar
tinycc_hash=$(nix hash file tinycc-liberated.nar)
grep "outputHash = \"$tinycc_hash\";$" using-nix/0-from-nixpkgs.nix
grep "sha256 = \"$tinycc_hash\";$" using-nix/0-prebuilt.nix
protosrc_=$(nix-build using-nix/0-from-nixpkgs.nix -A protosrc --no-out-link)
protosrc=$(nix-build using-nix/0-from-nixpkgs.nix -A protosrc --no-out-link \
--check)
[[ "$protosrc_" == "$protosrc" ]]
nix store dump-path $protosrc > protosrc.nar
protosrc_hash=$(nix hash file protosrc.nar)
grep "outputHash = \"$protosrc_hash\";$" using-nix/0-from-nixpkgs.nix
grep "sha256 = \"$protosrc_hash\";$" using-nix/0-prebuilt.nix
[[ ! -L result* ]]
# print release notes
set +x
cat <<\EOF
protosrc (patched sources of stage1 tinycc/protomusl/protobusybox) has to come from somewhere, and there are three main options: build as part of bootstrap, build using nixpkgs or take them from here.
You don't need these files neither when you do the whole bootstrap-Nix-included route using ./build.sh/make, nor when you can afford pulling nixpkgs to build them. But there's at least a case for building in Hydra under restricted-eval mode where you can neither inject stuff externally nor IFD. These allow kickstarting Hydra builds as long as recipes/1-stage1/seed.host-executed.sh and recipes/1-stage1/syscall.h weren't modified.
See using-nix/0.nix for more explanations of all three seeding options, using-nix/0-from-nixpkgs for how these were built.
EOF
echo "$tinycc_hash" tinycc-liberated.nar
echo "$protosrc_hash" protosrc.nar