Newer, better test script
This commit is contained in:
parent
dc256b15d4
commit
ad55c62383
145
test.sh
145
test.sh
|
@ -1,22 +1,153 @@
|
||||||
#! /bin/bash
|
#! /bin/bash
|
||||||
|
|
||||||
|
# ./test.sh [IMPL..]
|
||||||
|
# Run the forgebuild test suite against a number of implementations.
|
||||||
|
# When no argument is passed, the test suite is run against `forgebuild` in $PATH
|
||||||
|
# When arguments are passed:
|
||||||
|
# - if they are executable, the test suite is run against it and results are not saved to disk
|
||||||
|
# - if they are folders, prepare.sh is run in the folder, and should output the path to the
|
||||||
|
# executable to test; the basename of this executable is the implementation name (eg. forgebuild.sh)
|
||||||
|
# Results are logged to $RESULTSDIR, if it exists
|
||||||
|
# There is no check to ensure executables and folders are not mixed. Please be careful.
|
||||||
|
# To bootstrap test results for the website in ~/forge/site/static/tests/:
|
||||||
|
# $ RESULTSDIR=~/forge/site/static/tests ./test.sh PATH/TO/build.sh/ PATH/TO/build.rs/
|
||||||
|
|
||||||
|
# Please don't do weird things
|
||||||
|
shopt -s nullglob
|
||||||
|
|
||||||
|
# Run the test suite against a binary
|
||||||
|
# $1 is executable
|
||||||
|
# $2, if any, is test results folder
|
||||||
|
runTests() {
|
||||||
|
if [ ! -x "$1" ]; then
|
||||||
|
echo "ERROR: No executable permission for "$1""
|
||||||
|
echo "Try chmod +x "$1""
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "Running tests for "$1""
|
||||||
|
SUCCESS=1
|
||||||
|
if [ $# -gt 1 ]; then
|
||||||
|
# Output folder set
|
||||||
|
TESTDIR="$2"
|
||||||
|
# Save full log to a folder based on current timestamp
|
||||||
|
TIMESTAMP="$(date +%s)"
|
||||||
|
# Folder may already exist from another run (within same second)
|
||||||
|
[ ! -d "$TESTDIR"/$TIMESTAMP ] && mkdir "$TESTDIR"/$TIMESTAMP
|
||||||
|
echo "SAVING RESULTS TO "$TESTDIR"/$TIMESTAMP"
|
||||||
|
# latest should contain current date - commit
|
||||||
|
# Calculate commit
|
||||||
|
GITDIR="$(dirname "$FORGEBUILD")"
|
||||||
|
PREVDIR="$(pwd)"
|
||||||
|
cd $GITDIR
|
||||||
|
commit=$(git rev-parse HEAD)
|
||||||
|
cd $PREVDIR
|
||||||
|
if [ ! $? -eq 0 ]; then
|
||||||
|
echo "WARN: $GITDIR is not a git repository. Commit information not saved."
|
||||||
|
[ -f "$TESTDIR"/results.toml ] && commit="NOTGIT"
|
||||||
|
fi
|
||||||
|
# Test identifier is TIMESTAMP/COMMIT. It's saved in latest file
|
||||||
|
# so zola can load it programmatically
|
||||||
|
# and ti's served as table for test results in results.toml
|
||||||
|
echo "[$TIMESTAMP-$commit]" >> $TESTDIR/results.toml
|
||||||
|
echo "$TIMESTAMP-$commit" > "$TESTDIR"/latest
|
||||||
|
for test in tests/*.bats; do
|
||||||
|
testname="$(basename $test .bats)"
|
||||||
|
echo " Testfile "$testname""
|
||||||
|
|
||||||
|
OUTPUT="$(FORGEBUILD="$FORGEBUILD" bats "$test" 2>&1)"
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
echo "$testname = \"✓\"" >> $TESTDIR/results.toml
|
||||||
|
else
|
||||||
|
echo "$OUTPUT"
|
||||||
|
echo "$testname = \"⨉\"" >> $TESTDIR/results.toml
|
||||||
|
SUCCESS=0
|
||||||
|
fi
|
||||||
|
# Save full test log
|
||||||
|
echo "$OUTPUT" > "$TESTDIR"/$TIMESTAMP/$testname.txt
|
||||||
|
done
|
||||||
|
else
|
||||||
|
# No output folder
|
||||||
|
for test in tests/*.bats; do
|
||||||
|
testname="$(basename $test .bats)"
|
||||||
|
echo " Testfile "$testname""
|
||||||
|
# No output folder, don't save to disk
|
||||||
|
FORGEBUILD="$FORGEBUILD" bats "$test" || SUCCESS=0
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $SUCCESS -eq 0 ]; then
|
||||||
|
echo "FAIL: Some tests have failed."
|
||||||
|
return 2;
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check for dependencies
|
||||||
|
# TODO: maybe include a copy of bats in the repository?
|
||||||
if ! command -v bats > /dev/null; then
|
if ! command -v bats > /dev/null; then
|
||||||
echo "bats testing tool not found. Try sudo apt install bats?"
|
echo "bats testing tool not found. Try sudo apt install bats?"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ $# -lt 1 ]; then
|
if [ $# -lt 1 ]; then
|
||||||
|
# If there is no first argument, check forgebuild in $PATH or fail
|
||||||
FORGEBUILD="$(command -v forgebuild)"
|
FORGEBUILD="$(command -v forgebuild)"
|
||||||
if [ ! $? -eq 0 ]; then
|
if [ ! $? -eq 0 ]; then
|
||||||
echo "forgebuild not found in \$PATH. You can pass a path as argument to this script."
|
echo "forgebuild not found in \$PATH. You can pass a path as argument to this script."
|
||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
|
runTests "$FORGEBUILD"
|
||||||
else
|
else
|
||||||
FORGEBUILD="$(readlink -m "$1")"
|
# Run a loop for every argument
|
||||||
if [ ! -x $FORGEBUILD ]; then
|
while [ $# -gt 0 ]; do
|
||||||
echo "File $FORGEBUILD doesn't exist or is not executable."
|
# If there is a first argument, it's either a program
|
||||||
exit 3
|
FORGEBUILD="$(readlink -m "$1")"
|
||||||
fi
|
# Shift arguments for the next iteration of the loop
|
||||||
|
shift
|
||||||
|
echo ""
|
||||||
|
echo "-------------------"
|
||||||
|
echo ""
|
||||||
|
if [ -f $FORGEBUILD ]; then
|
||||||
|
if [ -x $FORGEBUILD ]; then
|
||||||
|
runTests "$FORGEBUILD"
|
||||||
|
else
|
||||||
|
echo "ERROR: Not executable: $FORGEBUILD"
|
||||||
|
echo "Maybe try chmod +x $FORGEBUILD"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
elif [ -d $FORGEBUILD ]; then
|
||||||
|
# Check if there is a prepare.sh script to build the program
|
||||||
|
# and give us a full path to work with
|
||||||
|
if [ -x $FORGEBUILD/prepare.sh ]; then
|
||||||
|
# Go there and come back
|
||||||
|
PREVDIR="$(pwd)"
|
||||||
|
cd $FORGEBUILD
|
||||||
|
OUTPUT="$($FORGEBUILD/prepare.sh)"
|
||||||
|
cd $PREVDIR
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
# Preparing was successful, we have a path
|
||||||
|
FORGEBUILD="$OUTPUT"
|
||||||
|
# Check if we should save to disk
|
||||||
|
if [ "$RESULTSDIR" = "" ]; then
|
||||||
|
runTests "$FORGEBUILD"
|
||||||
|
else
|
||||||
|
# Where to store results for this specific implementation
|
||||||
|
IMPLRESULTS="$RESULTSDIR"/"$(basename $FORGEBUILD)"
|
||||||
|
[ ! -d "$IMPLRESULTS" ] && mkdir "$IMPLRESULTS"
|
||||||
|
runTests "$FORGEBUILD" "$IMPLRESULTS"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Error occured, abort
|
||||||
|
echo "ERROR: Failed to run the prepare script $FORGEBUILD/prepare.sh. Output:"
|
||||||
|
echo "$OUTPUT" | sed 's/^/ /'
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "ERROR: No prepare.sh script in folder $FORGEBUILD"
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "ERROR: $FORGEBUILD is neither an executable or a folder."
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
FORGEBUILD="$FORGEBUILD" bats "$(dirname "$0")"/tests
|
|
||||||
|
|
Loading…
Reference in New Issue