Newer, better test script
This commit is contained in:
parent
dc256b15d4
commit
ad55c62383
145
test.sh
145
test.sh
|
@ -1,22 +1,153 @@
|
|||
#! /bin/bash
|
||||
|
||||
# ./test.sh [IMPL..]
|
||||
# Run the forgebuild test suite against a number of implementations.
|
||||
# When no argument is passed, the test suite is run against `forgebuild` in $PATH
|
||||
# When arguments are passed:
|
||||
# - if they are executable, the test suite is run against it and results are not saved to disk
|
||||
# - if they are folders, prepare.sh is run in the folder, and should output the path to the
|
||||
# executable to test; the basename of this executable is the implementation name (eg. forgebuild.sh)
|
||||
# Results are logged to $RESULTSDIR, if it exists
|
||||
# There is no check to ensure executables and folders are not mixed. Please be careful.
|
||||
# To bootstrap test results for the website in ~/forge/site/static/tests/:
|
||||
# $ RESULTSDIR=~/forge/site/static/tests ./test.sh PATH/TO/build.sh/ PATH/TO/build.rs/
|
||||
|
||||
# Please don't do weird things
|
||||
shopt -s nullglob
|
||||
|
||||
# Run the test suite against a binary
|
||||
# $1 is executable
|
||||
# $2, if any, is test results folder
|
||||
runTests() {
|
||||
if [ ! -x "$1" ]; then
|
||||
echo "ERROR: No executable permission for "$1""
|
||||
echo "Try chmod +x "$1""
|
||||
return 1
|
||||
fi
|
||||
echo "Running tests for "$1""
|
||||
SUCCESS=1
|
||||
if [ $# -gt 1 ]; then
|
||||
# Output folder set
|
||||
TESTDIR="$2"
|
||||
# Save full log to a folder based on current timestamp
|
||||
TIMESTAMP="$(date +%s)"
|
||||
# Folder may already exist from another run (within same second)
|
||||
[ ! -d "$TESTDIR"/$TIMESTAMP ] && mkdir "$TESTDIR"/$TIMESTAMP
|
||||
echo "SAVING RESULTS TO "$TESTDIR"/$TIMESTAMP"
|
||||
# latest should contain current date - commit
|
||||
# Calculate commit
|
||||
GITDIR="$(dirname "$FORGEBUILD")"
|
||||
PREVDIR="$(pwd)"
|
||||
cd $GITDIR
|
||||
commit=$(git rev-parse HEAD)
|
||||
cd $PREVDIR
|
||||
if [ ! $? -eq 0 ]; then
|
||||
echo "WARN: $GITDIR is not a git repository. Commit information not saved."
|
||||
[ -f "$TESTDIR"/results.toml ] && commit="NOTGIT"
|
||||
fi
|
||||
# Test identifier is TIMESTAMP/COMMIT. It's saved in latest file
|
||||
# so zola can load it programmatically
|
||||
# and ti's served as table for test results in results.toml
|
||||
echo "[$TIMESTAMP-$commit]" >> $TESTDIR/results.toml
|
||||
echo "$TIMESTAMP-$commit" > "$TESTDIR"/latest
|
||||
for test in tests/*.bats; do
|
||||
testname="$(basename $test .bats)"
|
||||
echo " Testfile "$testname""
|
||||
|
||||
OUTPUT="$(FORGEBUILD="$FORGEBUILD" bats "$test" 2>&1)"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "$testname = \"✓\"" >> $TESTDIR/results.toml
|
||||
else
|
||||
echo "$OUTPUT"
|
||||
echo "$testname = \"⨉\"" >> $TESTDIR/results.toml
|
||||
SUCCESS=0
|
||||
fi
|
||||
# Save full test log
|
||||
echo "$OUTPUT" > "$TESTDIR"/$TIMESTAMP/$testname.txt
|
||||
done
|
||||
else
|
||||
# No output folder
|
||||
for test in tests/*.bats; do
|
||||
testname="$(basename $test .bats)"
|
||||
echo " Testfile "$testname""
|
||||
# No output folder, don't save to disk
|
||||
FORGEBUILD="$FORGEBUILD" bats "$test" || SUCCESS=0
|
||||
done
|
||||
fi
|
||||
|
||||
if [ $SUCCESS -eq 0 ]; then
|
||||
echo "FAIL: Some tests have failed."
|
||||
return 2;
|
||||
fi
|
||||
}
|
||||
|
||||
# Check for dependencies
|
||||
# TODO: maybe include a copy of bats in the repository?
|
||||
if ! command -v bats > /dev/null; then
|
||||
echo "bats testing tool not found. Try sudo apt install bats?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $# -lt 1 ]; then
|
||||
# If there is no first argument, check forgebuild in $PATH or fail
|
||||
FORGEBUILD="$(command -v forgebuild)"
|
||||
if [ ! $? -eq 0 ]; then
|
||||
echo "forgebuild not found in \$PATH. You can pass a path as argument to this script."
|
||||
exit 2
|
||||
fi
|
||||
runTests "$FORGEBUILD"
|
||||
else
|
||||
FORGEBUILD="$(readlink -m "$1")"
|
||||
if [ ! -x $FORGEBUILD ]; then
|
||||
echo "File $FORGEBUILD doesn't exist or is not executable."
|
||||
exit 3
|
||||
fi
|
||||
# Run a loop for every argument
|
||||
while [ $# -gt 0 ]; do
|
||||
# If there is a first argument, it's either a program
|
||||
FORGEBUILD="$(readlink -m "$1")"
|
||||
# Shift arguments for the next iteration of the loop
|
||||
shift
|
||||
echo ""
|
||||
echo "-------------------"
|
||||
echo ""
|
||||
if [ -f $FORGEBUILD ]; then
|
||||
if [ -x $FORGEBUILD ]; then
|
||||
runTests "$FORGEBUILD"
|
||||
else
|
||||
echo "ERROR: Not executable: $FORGEBUILD"
|
||||
echo "Maybe try chmod +x $FORGEBUILD"
|
||||
exit 2
|
||||
fi
|
||||
elif [ -d $FORGEBUILD ]; then
|
||||
# Check if there is a prepare.sh script to build the program
|
||||
# and give us a full path to work with
|
||||
if [ -x $FORGEBUILD/prepare.sh ]; then
|
||||
# Go there and come back
|
||||
PREVDIR="$(pwd)"
|
||||
cd $FORGEBUILD
|
||||
OUTPUT="$($FORGEBUILD/prepare.sh)"
|
||||
cd $PREVDIR
|
||||
if [ $? -eq 0 ]; then
|
||||
# Preparing was successful, we have a path
|
||||
FORGEBUILD="$OUTPUT"
|
||||
# Check if we should save to disk
|
||||
if [ "$RESULTSDIR" = "" ]; then
|
||||
runTests "$FORGEBUILD"
|
||||
else
|
||||
# Where to store results for this specific implementation
|
||||
IMPLRESULTS="$RESULTSDIR"/"$(basename $FORGEBUILD)"
|
||||
[ ! -d "$IMPLRESULTS" ] && mkdir "$IMPLRESULTS"
|
||||
runTests "$FORGEBUILD" "$IMPLRESULTS"
|
||||
fi
|
||||
else
|
||||
# Error occured, abort
|
||||
echo "ERROR: Failed to run the prepare script $FORGEBUILD/prepare.sh. Output:"
|
||||
echo "$OUTPUT" | sed 's/^/ /'
|
||||
exit 3
|
||||
fi
|
||||
else
|
||||
echo "ERROR: No prepare.sh script in folder $FORGEBUILD"
|
||||
exit 4
|
||||
fi
|
||||
else
|
||||
echo "ERROR: $FORGEBUILD is neither an executable or a folder."
|
||||
exit 3
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
FORGEBUILD="$FORGEBUILD" bats "$(dirname "$0")"/tests
|
||||
|
|
Loading…
Reference in New Issue