build/test.sh

#! /bin/bash

# ./test.sh [IMPL..]
#   Run the forgebuild test suite against a number of implementations.
#   When no argument is passed, the test suite is run against `forgebuild` in $PATH
#   When arguments are passed:
#    - if they are executable, the test suite is run against it and results are not saved to disk
#    - if they are folders, prepare.sh is run in the folder, and should output the path to the
#      executable to test; the basename of this executable is the implementation name (eg. forgebuild.sh)
#      Results are logged to $RESULTSDIR, if it exists
#   There is no check to ensure executables and folders are not mixed. Please be careful.
#   To bootstrap test results for the website in ~/forge/site/static/tests/:
#     $ RESULTSDIR=~/forge/site/static/tests ./test.sh PATH/TO/build.sh/ PATH/TO/build.rs/

# Please don't do weird things
shopt -s nullglob

# Find where test.sh is so that we can find the tests/ folder relative to it
SPECDIR="$(dirname "$0")"

# Run the test suite against a binary
#   $1 is executable
#   $2, if any, is test results folder
runTests() {
    if [ ! -x "$1" ]; then
        echo "ERROR: No executable permission for "$1""
        echo "Try chmod +x "$1""
        return 1
    fi
    echo "Running tests for "$1""
    SUCCESS=1
    if [ $# -gt 1 ]; then
        # Output folder set
        TESTDIR="$2"
        # Save full log to a folder based on current timestamp
        TIMESTAMP="$(date +%s)"
        # Folder may already exist from another run (within same second)
        [ ! -d "$TESTDIR"/$TIMESTAMP ] && mkdir "$TESTDIR"/$TIMESTAMP
        echo "SAVING RESULTS TO "$TESTDIR"/$TIMESTAMP"
        # latest should contain current date - commit
        # Calculate commit
        GITDIR="$(dirname "$FORGEBUILD")"
        PREVDIR="$(pwd)"
        cd $GITDIR
        commit=$(git rev-parse HEAD)
        cd $PREVDIR
        if [ ! $? -eq 0 ]; then
            echo "WARN: $GITDIR is not a git repository. Commit information not saved."
            [ -f "$TESTDIR"/results.toml ] && commit="NOTGIT"
        fi
        # Test identifier is TIMESTAMP/COMMIT. It's saved in latest file
        # so zola can load it programmatically
        # and ti's served as table for test results in results.toml
        echo "[$TIMESTAMP-$commit]" >> $TESTDIR/results.toml
        echo "$TIMESTAMP-$commit" > "$TESTDIR"/latest
        for test in "$SPECDIR"/tests/*.bats; do
            testname="$(basename $test .bats)"
            echo "  Testfile "$testname""

            OUTPUT="$(FORGEBUILD="$FORGEBUILD" bats "$test" 2>&1)"
            if [ $? -eq 0 ]; then
               echo "$testname = \"✓\"" >> $TESTDIR/results.toml
            else
                echo "$OUTPUT"
                echo "$testname = \"⨉\"" >> $TESTDIR/results.toml
                SUCCESS=0
            fi
            # Save full test log
            echo "$OUTPUT" > "$TESTDIR"/$TIMESTAMP/$testname.txt
        done
    else
        # No output folder
        for test in "$SPECDIR"/tests/*.bats; do
            testname="$(basename $test .bats)"
            echo "  Testfile "$testname""
            # No output folder, don't save to disk
            FORGEBUILD="$FORGEBUILD" bats "$test" || SUCCESS=0
        done
    fi

    if [ $SUCCESS -eq 0 ]; then
        echo "FAIL: Some tests have failed."
        return 2;
    fi
}

# Check for dependencies
# TODO: maybe include a copy of bats in the repository?
if ! command -v bats > /dev/null; then
    echo "bats testing tool not found. Try sudo apt install bats?"
    exit 1
fi

if [ $# -lt 1 ]; then
    # If there is no first argument, check forgebuild in $PATH or fail
    FORGEBUILD="$(command -v forgebuild)"
    if [ ! $? -eq 0 ]; then
        echo "forgebuild not found in \$PATH. You can pass a path as argument to this script."
        exit 2
    fi
    runTests "$FORGEBUILD"
else
    # Run a loop for every argument
    while [ $# -gt 0 ]; do
        # If there is a first argument, it's either a program
        FORGEBUILD="$(readlink -m "$1")"
        # Shift arguments for the next iteration of the loop
        shift
        echo ""
        echo "-------------------"
        echo ""
        if [ -f $FORGEBUILD ]; then
            if [ -x $FORGEBUILD ]; then
                runTests "$FORGEBUILD"
            else
                echo "ERROR: Not executable: $FORGEBUILD"
                echo "Maybe try chmod +x $FORGEBUILD"
                exit 2
            fi
        elif [ -d $FORGEBUILD ]; then
            # Check if there is a prepare.sh script to build the program
            # and give us a full path to work with
            if [ -x $FORGEBUILD/prepare.sh ]; then
                # Go there and come back
                PREVDIR="$(pwd)"
                cd $FORGEBUILD
                OUTPUT="$($FORGEBUILD/prepare.sh)"
                cd $PREVDIR
                if [ $? -eq 0 ]; then
                    # Preparing was successful, we have a path
                    FORGEBUILD="$OUTPUT"
                    # Check if we should save to disk
                    if [ "$RESULTSDIR" = "" ]; then
                        runTests "$FORGEBUILD"
                    else
                        # Where to store results for this specific implementation
                        IMPLRESULTS="$RESULTSDIR"/"$(basename $FORGEBUILD)"
                        [ ! -d "$IMPLRESULTS" ] && mkdir "$IMPLRESULTS"
                        runTests "$FORGEBUILD" "$IMPLRESULTS"
                    fi
                else
                    # Error occured, abort
                    echo "ERROR: Failed to run the prepare script $FORGEBUILD/prepare.sh. Output:"
                    echo "$OUTPUT" | sed 's/^/  /'
                    exit 3
                fi
            else
                echo "ERROR: No prepare.sh script in folder $FORGEBUILD"
                exit 4
            fi
        else
            echo "ERROR: $FORGEBUILD is neither an executable or a folder."
            exit 3
        fi
    done
fi