289 lines
9.0 KiB
Bash
Executable File
289 lines
9.0 KiB
Bash
Executable File
#!/bin/sh
|
|
# Experimental alternative to build2 that can run steps in parallel if their
|
|
# dependencies are met. Caveats:
|
|
#
|
|
# 1. We rely on the OS to schedule steps, so thousands of independent tasks
|
|
# will likely be counter-productive.
|
|
# 2. Can run out of virtual memory if you spawn too many say $CC processes.
|
|
# 3. Compilation errors can cause the script to hang. We tag the most common
|
|
# suspects with '|| quit', but can't eliminate the possibility entirely.
|
|
# 4. Ugly as heck! This version really benefits from comparisons with its
|
|
# 'upstream', build2. And even then, diff gets confused.
|
|
# 5. There's a mechanical difficulty: we use mktemp to reliably create
|
|
# temporary filenames, which has the side effect of also creating the
|
|
# files. So zero-size files are considered equivalent to non-existent
|
|
# files. When some commands have trouble with this (e.g. ar) we need to
|
|
# delete the empty file, which can expose us to a race condition wrt
|
|
# mktemp.
|
|
|
|
set -e # stop immediately on error
|
|
|
|
# Some environment variables that can be passed in. For example, to turn off
|
|
# optimization:
|
|
# $ CFLAGS=-g ./build4
|
|
test "$CXX" || export CXX=c++
|
|
test "$CC" || export CC=cc
|
|
test "$CFLAGS" || export CFLAGS="-g -O2"
|
|
export CFLAGS="$CFLAGS -Wall -Wextra -ftrapv -fno-strict-aliasing"
|
|
export CXXFLAGS="-std=c++98 $CFLAGS" # CI has an ancient version; don't expect recent dialects
|
|
|
|
# Outline:
|
|
# [0-9]*.cc -> mu.cc -> .build/*.cc -> .build/*.o -> .build/mu_bin
|
|
# (layers) | | | |
|
|
# tangle cleave $CXX $CXX
|
|
|
|
# can also be called with a layer to only build until
|
|
# $ ./build4 --until 050
|
|
UNTIL_LAYER=${2:-zzz}
|
|
|
|
# there's two mechanisms for fast builds here:
|
|
# - if a command is quick to run, always run it but update the result only on any change
|
|
# - otherwise run it only if the output is 'older_than' the inputs
|
|
#
|
|
# avoid combining both mechanisms for a single file
|
|
# otherwise you'll see spurious messages about files being updated
|
|
# risk: a file may unnecessarily update without changes, causing unnecessary work downstream
|
|
|
|
# return 1 if $1 is older than _any_ of the remaining args
|
|
older_than() {
|
|
local target=$1
|
|
shift
|
|
if [ ! -s $target ]
|
|
then
|
|
#? echo "$target has size zero"
|
|
#? echo "updating $target" >&2
|
|
return 0 # success
|
|
fi
|
|
local f
|
|
for f in $*
|
|
do
|
|
if [ $f -nt $target ]
|
|
then
|
|
#? echo "updating $target" >&2
|
|
return 0 # success
|
|
fi
|
|
done
|
|
return 1 # failure
|
|
}
|
|
|
|
# redirect to $1, unless it's already identical
|
|
update() {
|
|
if [ ! -e $1 ]
|
|
then
|
|
cat > $1
|
|
else
|
|
cat > $1.tmp
|
|
diff -q $1 $1.tmp >/dev/null && rm $1.tmp || mv $1.tmp $1
|
|
fi
|
|
}
|
|
|
|
update_cp() {
|
|
if [ ! -e $2/$1 ]
|
|
then
|
|
cp $1 $2
|
|
elif [ $1 -nt $2/$1 ]
|
|
then
|
|
cp $1 $2
|
|
fi
|
|
}
|
|
|
|
noisy_cd() {
|
|
cd $1
|
|
echo "-- `pwd`" >&2
|
|
}
|
|
|
|
mv_if_exists() {
|
|
test -e $1 && mv $1 $2
|
|
return 0
|
|
}
|
|
|
|
# wait for all the given filenames to exist
|
|
# exit immediately if a special file called '.quit' exists
|
|
QUITFILE=`pwd`/.quit
|
|
rm -f $QUITFILE
|
|
wait_for_all() {
|
|
# could use inotify on Linux
|
|
while ! all_exist "$@"
|
|
do
|
|
#? echo waiting: $*
|
|
test -e $QUITFILE && return 1 # some step had an error; stop all waiting steps using errexit
|
|
sleep 1
|
|
done
|
|
return 0
|
|
}
|
|
quit() {
|
|
touch $QUITFILE
|
|
exit 1
|
|
}
|
|
|
|
all_exist() {
|
|
for f in "$@"
|
|
do
|
|
test -e $f || return 1
|
|
done
|
|
return 0
|
|
}
|
|
|
|
TMP=`mktemp`
|
|
mv_if_exists ../../linux/bootstrap/tools/enumerate $TMP
|
|
(
|
|
wait_for_all ../../linux/bootstrap/tools/enumerate.cc
|
|
older_than $TMP ../../linux/bootstrap/tools/enumerate.cc && {
|
|
echo "building enumerate"
|
|
$CXX $CXXFLAGS ../../linux/bootstrap/tools/enumerate.cc -o $TMP || quit
|
|
echo "done building enumerate"
|
|
}
|
|
mv $TMP ../../linux/bootstrap/tools/enumerate
|
|
) &
|
|
|
|
TMP=`mktemp`
|
|
mv_if_exists ../../linux/bootstrap/tools/tangle $TMP
|
|
(
|
|
wait_for_all ../../linux/bootstrap/tools/tangle.cc
|
|
older_than $TMP ../../linux/bootstrap/tools/tangle.cc && {
|
|
echo "building tangle"
|
|
cd ../../linux/bootstrap/tools
|
|
# auto-generate various lists (ending in '_list' by convention) {
|
|
# list of function declarations, so I can define them in any order
|
|
grep -h "^[^ #].*) {" tangle.cc |sed 's/ {.*/;/' |update tangle.function_list
|
|
# list of tests to run
|
|
grep -h "^[[:space:]]*void test_" tangle.cc |sed 's/^\s*void \(.*\)() {$/\1,/' |update tangle.test_list
|
|
grep -h "^\s*void test_" tangle.cc |sed 's/^\s*void \(.*\)() {.*/"\1",/' |update tangle.test_name_list
|
|
# }
|
|
# Now that we have all the _lists, compile 'tangle'
|
|
$CXX $CXXFLAGS tangle.cc -o $TMP || quit
|
|
$TMP test
|
|
cd ../../../archive/1.vm
|
|
echo "done building tangle"
|
|
}
|
|
mv $TMP ../../linux/bootstrap/tools/tangle
|
|
) &
|
|
|
|
wait_for_all ../../linux/bootstrap/tools/enumerate
|
|
echo "enumerating layers"
|
|
LAYERS=$(../../linux/bootstrap/tools/enumerate --until $UNTIL_LAYER |grep '\.cc$')
|
|
echo "done enumerating layers"
|
|
|
|
TMP=`mktemp`
|
|
mv_if_exists mu.cc $TMP
|
|
(
|
|
wait_for_all $LAYERS ../../linux/bootstrap/tools/enumerate ../../linux/bootstrap/tools/tangle
|
|
older_than $TMP $LAYERS ../../linux/bootstrap/tools/enumerate ../../linux/bootstrap/tools/tangle && {
|
|
echo "running tangle"
|
|
# no update here; rely on 'update' calls downstream
|
|
../../linux/bootstrap/tools/tangle $LAYERS >$TMP || quit
|
|
echo "done running tangle"
|
|
}
|
|
mv $TMP mu.cc
|
|
) &
|
|
|
|
TMP=`mktemp`
|
|
mv_if_exists cleave/cleave $TMP
|
|
(
|
|
wait_for_all cleave/cleave.cc
|
|
older_than $TMP cleave/cleave.cc && {
|
|
echo "building cleave"
|
|
$CXX $CXXFLAGS cleave/cleave.cc -o $TMP || quit
|
|
rm -rf .build
|
|
echo "done building cleave"
|
|
}
|
|
mv $TMP cleave/cleave
|
|
) &
|
|
|
|
wait_for_all mu.cc cleave/cleave # cleave/cleave just for the .build cleanup
|
|
mkdir -p .build
|
|
# auto-generate function declarations, so I can define them in any order
|
|
# functions start out unindented, have all args on the same line, and end in ') {'
|
|
#
|
|
# \/ ignore struct/class methods
|
|
grep -h "^[^[:space:]#].*) {$" mu.cc |grep -v ":.*(" |sed 's/ {.*/;/' |update .build/function_list
|
|
# auto-generate list of tests to run
|
|
grep -h "^\s*void test_" mu.cc |sed 's/^\s*void \(.*\)() {.*/\1,/' |update .build/test_list
|
|
grep -h "^\s*void test_" mu.cc |sed 's/^\s*void \(.*\)() {.*/"\1",/' |update .build/test_name_list
|
|
mkdir -p .build/termbox
|
|
update_cp termbox/termbox.h .build/termbox
|
|
|
|
TMP=`mktemp`
|
|
mv_if_exists mu_bin $TMP
|
|
(
|
|
wait_for_all mu.cc cleave/cleave termbox/*.c termbox/*.h termbox/*.inl
|
|
older_than $TMP mu.cc *_list cleave/cleave termbox/* && {
|
|
echo "building mu_bin"
|
|
cleave/cleave mu.cc .build || quit
|
|
cd .build
|
|
# create the list of global variable declarations from the corresponding definitions
|
|
grep ';' global_definitions_list |sed 's/[=(].*/;/' |sed 's/^[^\/# ]/extern &/' |sed 's/^extern extern /extern /' |update global_declarations_list
|
|
for f in mu_*.cc
|
|
do
|
|
OBJ=`echo $f |sed 's/\.cc$/.o/'`
|
|
TMP=`mktemp`
|
|
mv_if_exists $OBJ $TMP
|
|
(
|
|
older_than $TMP $f header global_declarations_list function_list test_list && {
|
|
echo "building $OBJ"
|
|
$CXX $CXXFLAGS -c $f -o $TMP || quit
|
|
echo "done building $OBJ"
|
|
}
|
|
mv $TMP $OBJ
|
|
) &
|
|
done
|
|
cd ../termbox
|
|
TMP=`mktemp`
|
|
mv_if_exists utf8.o $TMP
|
|
(
|
|
older_than $TMP utf8.c && {
|
|
echo "building termbox/utf8.o"
|
|
$CC $CFLAGS -c utf8.c -o $TMP || quit
|
|
echo "done building termbox/utf8.o"
|
|
}
|
|
mv $TMP utf8.o
|
|
) &
|
|
TMP=`mktemp`
|
|
mv_if_exists termbox.o $TMP
|
|
(
|
|
older_than $TMP termbox.c termbox.h input.inl output.inl bytebuffer.inl && {
|
|
echo "building termbox/termbox.o"
|
|
$CC $CFLAGS -c termbox.c -o $TMP || quit
|
|
echo "done building termbox/termbox.o"
|
|
}
|
|
mv $TMP termbox.o
|
|
) &
|
|
TMP=`mktemp`
|
|
mv_if_exists libtermbox.a $TMP
|
|
(
|
|
wait_for_all termbox.o utf8.o
|
|
older_than $TMP termbox.o utf8.o && {
|
|
echo "building termbox/libtermbox.a"
|
|
rm $TMP; ar rcs $TMP termbox.o utf8.o || quit # race condition; later mktemp may end up reusing this file
|
|
echo "done building termbox/libtermbox.a"
|
|
}
|
|
mv $TMP libtermbox.a
|
|
) &
|
|
cd ..
|
|
MU_OBJS=`echo .build/mu_*.cc |sed 's/\.cc/.o/g'`
|
|
echo wait_for_all $MU_OBJS termbox/libtermbox.a
|
|
wait_for_all $MU_OBJS termbox/libtermbox.a
|
|
echo "building .build/mu_bin"
|
|
$CXX $CXXFLAGS $MU_OBJS termbox/libtermbox.a -o $TMP || quit
|
|
echo "done building .build/mu_bin"
|
|
echo "done building mu_bin"
|
|
}
|
|
mv $TMP mu_bin
|
|
) &
|
|
|
|
## [0-9]*.mu -> core.mu
|
|
|
|
wait_for_all ../../linux/bootstrap/tools/enumerate
|
|
echo "building core.mu"
|
|
MU_LAYERS=$(../../linux/bootstrap/tools/enumerate --until $UNTIL_LAYER |grep '\.mu$') || exit 0 # ok if no .mu files
|
|
cat $MU_LAYERS |update core.mu
|
|
echo "done building core.mu"
|
|
|
|
wait_for_all mu_bin
|
|
exit 0
|
|
|
|
# scenarios considered:
|
|
# 0 status when nothing needs updating
|
|
# only a single layer is recompiled when changing a C++ function
|
|
# stop immediately after any failure
|