From ad55c62383dc399d1d8a99c10bb2522a337d3b54 Mon Sep 17 00:00:00 2001 From: southerntofu Date: Tue, 5 Jan 2021 13:57:42 +0100 Subject: [PATCH] Newer, better test script --- test.sh | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 138 insertions(+), 7 deletions(-) diff --git a/test.sh b/test.sh index 609f824..9914b7d 100755 --- a/test.sh +++ b/test.sh @@ -1,22 +1,153 @@ #! /bin/bash +# ./test.sh [IMPL..] +# Run the forgebuild test suite against a number of implementations. +# When no argument is passed, the test suite is run against `forgebuild` in $PATH +# When arguments are passed: +# - if they are executable, the test suite is run against it and results are not saved to disk +# - if they are folders, prepare.sh is run in the folder, and should output the path to the +# executable to test; the basename of this executable is the implementation name (eg. forgebuild.sh) +# Results are logged to $RESULTSDIR, if it exists +# There is no check to ensure executables and folders are not mixed. Please be careful. +# To bootstrap test results for the website in ~/forge/site/static/tests/: +# $ RESULTSDIR=~/forge/site/static/tests ./test.sh PATH/TO/build.sh/ PATH/TO/build.rs/ + +# Please don't do weird things +shopt -s nullglob + +# Run the test suite against a binary +# $1 is executable +# $2, if any, is test results folder +runTests() { + if [ ! -x "$1" ]; then + echo "ERROR: No executable permission for "$1"" + echo "Try chmod +x "$1"" + return 1 + fi + echo "Running tests for "$1"" + SUCCESS=1 + if [ $# -gt 1 ]; then + # Output folder set + TESTDIR="$2" + # Save full log to a folder based on current timestamp + TIMESTAMP="$(date +%s)" + # Folder may already exist from another run (within same second) + [ ! -d "$TESTDIR"/$TIMESTAMP ] && mkdir "$TESTDIR"/$TIMESTAMP + echo "SAVING RESULTS TO "$TESTDIR"/$TIMESTAMP" + # latest should contain current date - commit + # Calculate commit + GITDIR="$(dirname "$FORGEBUILD")" + PREVDIR="$(pwd)" + cd $GITDIR + commit=$(git rev-parse HEAD) + cd $PREVDIR + if [ ! $? -eq 0 ]; then + echo "WARN: $GITDIR is not a git repository. Commit information not saved." + [ -f "$TESTDIR"/results.toml ] && commit="NOTGIT" + fi + # Test identifier is TIMESTAMP/COMMIT. It's saved in latest file + # so zola can load it programmatically + # and ti's served as table for test results in results.toml + echo "[$TIMESTAMP-$commit]" >> $TESTDIR/results.toml + echo "$TIMESTAMP-$commit" > "$TESTDIR"/latest + for test in tests/*.bats; do + testname="$(basename $test .bats)" + echo " Testfile "$testname"" + + OUTPUT="$(FORGEBUILD="$FORGEBUILD" bats "$test" 2>&1)" + if [ $? -eq 0 ]; then + echo "$testname = \"✓\"" >> $TESTDIR/results.toml + else + echo "$OUTPUT" + echo "$testname = \"⨉\"" >> $TESTDIR/results.toml + SUCCESS=0 + fi + # Save full test log + echo "$OUTPUT" > "$TESTDIR"/$TIMESTAMP/$testname.txt + done + else + # No output folder + for test in tests/*.bats; do + testname="$(basename $test .bats)" + echo " Testfile "$testname"" + # No output folder, don't save to disk + FORGEBUILD="$FORGEBUILD" bats "$test" || SUCCESS=0 + done + fi + + if [ $SUCCESS -eq 0 ]; then + echo "FAIL: Some tests have failed." + return 2; + fi +} + +# Check for dependencies +# TODO: maybe include a copy of bats in the repository? if ! command -v bats > /dev/null; then echo "bats testing tool not found. Try sudo apt install bats?" exit 1 fi if [ $# -lt 1 ]; then + # If there is no first argument, check forgebuild in $PATH or fail FORGEBUILD="$(command -v forgebuild)" if [ ! $? -eq 0 ]; then echo "forgebuild not found in \$PATH. You can pass a path as argument to this script." exit 2 fi + runTests "$FORGEBUILD" else - FORGEBUILD="$(readlink -m "$1")" - if [ ! -x $FORGEBUILD ]; then - echo "File $FORGEBUILD doesn't exist or is not executable." - exit 3 - fi + # Run a loop for every argument + while [ $# -gt 0 ]; do + # If there is a first argument, it's either a program + FORGEBUILD="$(readlink -m "$1")" + # Shift arguments for the next iteration of the loop + shift + echo "" + echo "-------------------" + echo "" + if [ -f $FORGEBUILD ]; then + if [ -x $FORGEBUILD ]; then + runTests "$FORGEBUILD" + else + echo "ERROR: Not executable: $FORGEBUILD" + echo "Maybe try chmod +x $FORGEBUILD" + exit 2 + fi + elif [ -d $FORGEBUILD ]; then + # Check if there is a prepare.sh script to build the program + # and give us a full path to work with + if [ -x $FORGEBUILD/prepare.sh ]; then + # Go there and come back + PREVDIR="$(pwd)" + cd $FORGEBUILD + OUTPUT="$($FORGEBUILD/prepare.sh)" + cd $PREVDIR + if [ $? -eq 0 ]; then + # Preparing was successful, we have a path + FORGEBUILD="$OUTPUT" + # Check if we should save to disk + if [ "$RESULTSDIR" = "" ]; then + runTests "$FORGEBUILD" + else + # Where to store results for this specific implementation + IMPLRESULTS="$RESULTSDIR"/"$(basename $FORGEBUILD)" + [ ! -d "$IMPLRESULTS" ] && mkdir "$IMPLRESULTS" + runTests "$FORGEBUILD" "$IMPLRESULTS" + fi + else + # Error occured, abort + echo "ERROR: Failed to run the prepare script $FORGEBUILD/prepare.sh. Output:" + echo "$OUTPUT" | sed 's/^/ /' + exit 3 + fi + else + echo "ERROR: No prepare.sh script in folder $FORGEBUILD" + exit 4 + fi + else + echo "ERROR: $FORGEBUILD is neither an executable or a folder." + exit 3 + fi + done fi - -FORGEBUILD="$FORGEBUILD" bats "$(dirname "$0")"/tests