Specification and translation files for forgebuild
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

153 lines
5.8 KiB

#! /bin/bash
# ./test.sh [IMPL..]
# Run the forgebuild test suite against a number of implementations.
# When no argument is passed, the test suite is run against `forgebuild` in $PATH
# When arguments are passed:
# - if they are executable, the test suite is run against it and results are not saved to disk
# - if they are folders, prepare.sh is run in the folder, and should output the path to the
# executable to test; the basename of this executable is the implementation name (eg. forgebuild.sh)
# Results are logged to $RESULTSDIR, if it exists
# There is no check to ensure executables and folders are not mixed. Please be careful.
# To bootstrap test results for the website in ~/forge/site/static/tests/:
# $ RESULTSDIR=~/forge/site/static/tests ./test.sh PATH/TO/build.sh/ PATH/TO/build.rs/
# Please don't do weird things
shopt -s nullglob
# Run the test suite against a binary
# $1 is executable
# $2, if any, is test results folder
runTests() {
if [ ! -x "$1" ]; then
echo "ERROR: No executable permission for "$1""
echo "Try chmod +x "$1""
return 1
fi
echo "Running tests for "$1""
SUCCESS=1
if [ $# -gt 1 ]; then
# Output folder set
TESTDIR="$2"
# Save full log to a folder based on current timestamp
TIMESTAMP="$(date +%s)"
# Folder may already exist from another run (within same second)
[ ! -d "$TESTDIR"/$TIMESTAMP ] && mkdir "$TESTDIR"/$TIMESTAMP
echo "SAVING RESULTS TO "$TESTDIR"/$TIMESTAMP"
# latest should contain current date - commit
# Calculate commit
GITDIR="$(dirname "$FORGEBUILD")"
PREVDIR="$(pwd)"
cd $GITDIR
commit=$(git rev-parse HEAD)
cd $PREVDIR
if [ ! $? -eq 0 ]; then
echo "WARN: $GITDIR is not a git repository. Commit information not saved."
[ -f "$TESTDIR"/results.toml ] && commit="NOTGIT"
fi
# Test identifier is TIMESTAMP/COMMIT. It's saved in latest file
# so zola can load it programmatically
# and ti's served as table for test results in results.toml
echo "[$TIMESTAMP-$commit]" >> $TESTDIR/results.toml
echo "$TIMESTAMP-$commit" > "$TESTDIR"/latest
for test in tests/*.bats; do
testname="$(basename $test .bats)"
echo " Testfile "$testname""
OUTPUT="$(FORGEBUILD="$FORGEBUILD" bats "$test" 2>&1)"
if [ $? -eq 0 ]; then
echo "$testname = \"✓\"" >> $TESTDIR/results.toml
else
echo "$OUTPUT"
echo "$testname = \"⨉\"" >> $TESTDIR/results.toml
SUCCESS=0
fi
# Save full test log
echo "$OUTPUT" > "$TESTDIR"/$TIMESTAMP/$testname.txt
done
else
# No output folder
for test in tests/*.bats; do
testname="$(basename $test .bats)"
echo " Testfile "$testname""
# No output folder, don't save to disk
FORGEBUILD="$FORGEBUILD" bats "$test" || SUCCESS=0
done
fi
if [ $SUCCESS -eq 0 ]; then
echo "FAIL: Some tests have failed."
return 2;
fi
}
# Check for dependencies
# TODO: maybe include a copy of bats in the repository?
if ! command -v bats > /dev/null; then
echo "bats testing tool not found. Try sudo apt install bats?"
exit 1
fi
if [ $# -lt 1 ]; then
# If there is no first argument, check forgebuild in $PATH or fail
FORGEBUILD="$(command -v forgebuild)"
if [ ! $? -eq 0 ]; then
echo "forgebuild not found in \$PATH. You can pass a path as argument to this script."
exit 2
fi
runTests "$FORGEBUILD"
else
# Run a loop for every argument
while [ $# -gt 0 ]; do
# If there is a first argument, it's either a program
FORGEBUILD="$(readlink -m "$1")"
# Shift arguments for the next iteration of the loop
shift
echo ""
echo "-------------------"
echo ""
if [ -f $FORGEBUILD ]; then
if [ -x $FORGEBUILD ]; then
runTests "$FORGEBUILD"
else
echo "ERROR: Not executable: $FORGEBUILD"
echo "Maybe try chmod +x $FORGEBUILD"
exit 2
fi
elif [ -d $FORGEBUILD ]; then
# Check if there is a prepare.sh script to build the program
# and give us a full path to work with
if [ -x $FORGEBUILD/prepare.sh ]; then
# Go there and come back
PREVDIR="$(pwd)"
cd $FORGEBUILD
OUTPUT="$($FORGEBUILD/prepare.sh)"
cd $PREVDIR
if [ $? -eq 0 ]; then
# Preparing was successful, we have a path
FORGEBUILD="$OUTPUT"
# Check if we should save to disk
if [ "$RESULTSDIR" = "" ]; then
runTests "$FORGEBUILD"
else
# Where to store results for this specific implementation
IMPLRESULTS="$RESULTSDIR"/"$(basename $FORGEBUILD)"
[ ! -d "$IMPLRESULTS" ] && mkdir "$IMPLRESULTS"
runTests "$FORGEBUILD" "$IMPLRESULTS"
fi
else
# Error occured, abort
echo "ERROR: Failed to run the prepare script $FORGEBUILD/prepare.sh. Output:"
echo "$OUTPUT" | sed 's/^/ /'
exit 3
fi
else
echo "ERROR: No prepare.sh script in folder $FORGEBUILD"
exit 4
fi
else
echo "ERROR: $FORGEBUILD is neither an executable or a folder."
exit 3
fi
done
fi