Rework first tests for progressive implementation

This commit is contained in:
southerntofu 2020-12-01 18:01:55 +01:00
parent c8f5471ccd
commit 2bfd3a7fe0
4 changed files with 498 additions and 81 deletions

View File

@ -1,12 +1,13 @@
#! /usr/bin/env bats
# This is a test file for use with the bats testing framework
# https://github.com/bats-core/bats-core
# On Debian: apt install bats
# 01-cli-flags.bats
# In this first test suite for forgebuild, we disable translations entirely.
# If you're just getting started writing your implementation, good! No need
# to disable anything. Otherwise, please make sure LANG=NONE disables
# translations entirely. eg. "$(trans error)$(trans missing_basedir)"
# outputs "errormissing_basedir"
# output.bats tests that output is uniform across forgebuild implementations,
# by using the special value "NONE" as LANG environment variable when calling
# forgebuild.
load helpers
function setup {
ORIGDIR="$(pwd)"
@ -24,15 +25,8 @@ function setup {
echo "echo \$(pwd) > $TMPDIR/forgebuild-pwd" > $FORGEBUILDDIR/pwd
chmod +x $FORGEBUILDDIR/pwd
# Setup settings
mkdir $FORGEBUILDDIR/config
echo "default" > $FORGEBUILDDIR/config/entry
mkdir $FORGEBUILDDIR/$HOSTNAME
echo "host" > $FORGEBUILDDIR/$HOSTNAME/entry
# Disable translations
export LANG="NONE"
#export LANG="JSON"
# Enable debug for full log
export LOG="debug"
@ -40,99 +34,238 @@ function setup {
function teardown {
cd $ORIGDIR
clean
}
function clean {
if [ -d $TMPDIR ]; then rm -rf $TMPDIR; fi
}
# Emulate translation, without actual escape codes (colors),
# variable substitution, or actual translations
warn() {
echo "warning${1}"
# forgebuild being a CLI application, first we'll check that it can display
# a use help message when asked to. When called with the -h|--help flag,
# forgebuild should display a help message and quit without error.
# The first line of the help message should contain "forgebuild VER",
# where VER is the semantic version of the software.
@test "Can provide help" {
run $FORGEBUILD --help
echo "$output"
# forgebuild returns 0 exit code (no error)
[ $status = 0 ]
first_output="$output"
# the first line contains name/version eg. forgebuild 0.2.3
echo "${lines[0]}" | grep -P "forgebuild \d+[\.\d+]*"
# -h should return exactly the same output
run $FORGEBUILD -h
[ $status = 0 ]
eq "$first_output" "$output"
}
info() {
if [[ "$LOG" = "info" ]] || [[ "$LOG" = "debug" ]] || [[ "$LOG" = "" ]]; then
echo "[git-build] ${1}"
fi
# In the same manner, forgebuild should return version information
# when called with the -V|--version flag. Note that "-V" uses a capital
# letter V, as -v is traditionally reserved for verbosity level (loglevel).
# The version information should output only one line: "forgebuild VER",
# where VER is a semantic version.
@test "Can provide version information" {
run $FORGEBUILD --version
[ $status = 0 ]
first_output="$output"
echo "$output" | run grep -P "forgebuild \d+[\.\d+]*"
[ $status = 0 ]
run $FORGEBUILD -V
[ $status = 0 ]
eq "$first_output" "$output"
}
debug() {
[[ "$LOG" = "debug" ]] && echo "debug${1}"
}
error() {
echo "error${1}"
}
# Simple repository
@test "missing basedir" {
# The basedir is the path from which we are going to load tasks.
# It is either passed with the `-b|--basedir` flag, or defaults
# to ~/.basedir. When this folder does not exist, forgebuild
# should output "errormissing_basedir" and return with a non-zero
# exit code.
@test "Detects missing basedir" {
run $FORGEBUILD -b /tmp/THISPATHDOESNOTEXIST
[ $status != 0 ]
echo "$output"
[ "$output" = "$(error missing_basedir)" ]
eq "$output" "$(error missing_basedir)"
if [ ! -d $HOME/.forgebuild ]; then
run $FORGEBUILD
[ $status != 0 ]
echo "$output"
eq "$output" "$(error missing_basedir)"
fi
}
@test "unknown task" {
# The positional arguments passed to forgebuild are a list of tasks
# to be triggered or not depending on further configuration. In this
# test suite, two tasks are declared (time, pwd) and are configured
# to run. If an unknown task is requested, forgebuild will output
# "errorunknown_arg" and return with a non-zero exit code.
@test "Doesn't recognize unknown task" {
run $FORGEBUILD -b $FORGEBUILDDIR THISTASKDOESNOTEXIST
[ $status != 0 ]
echo "$output"
[ "$output" = "$(error unknown_arg)" ]
eq "$output" "$(error unknown_arg)"
}
@test "unknown argument" {
# When called with an unknown flag, forgebuild will output
# "error_unknownarg" and return with a non-zero exit code.
@test "Doesn't recognize unknown argument" {
run $FORGEBUILD -WTF -b $FORGEBUILDDIR
[ $status != 0 ]
echo "$output"
# First line is unknown_arg error, then maybe help message?
# But each implementation may have a different help so we don't
# check that precisely
[ "${lines[0]}" = "$(error unknown_arg)" ]
eq "${lines[0]}" "$(error unknown_arg)"
}
@test "clone failed" {
echo "/tmp/THISPATHDOESNOTEXIST" > $FORGEBUILDDIR/time.source
run $FORGEBUILD -b $FORGEBUILDDIR time
echo "$output"
[ "${lines[0]}" = "$(debug found_task)" ]
[ "${lines[1]}" = "$(info config)" ]
[ "${lines[2]}" = "$(debug start_proc)" ]
[ "${lines[3]}" = "$(info process)" ]
[ "${lines[4]}" = "$(info clone)" ]
[ "${lines[5]}" = "$(error clone_failed)" ]
}
@test "simple sourceless run" {
run $FORGEBUILD -b $FORGEBUILDDIR time
echo "$output"
[ "${lines[0]}" = "$(debug found_task)" ]
[ "${lines[1]}" = "$(info config)" ]
[ "${lines[2]}" = "$(debug start_proc)" ]
[ "${lines[3]}" = "$(info process)" ]
[ "${lines[4]}" = "$(info run)" ]
}
@test "no tasks requested" {
# When no tasks are requested as positional arguments, and no
# special flag is passed, forgebuild will output "infono_task",
# then for each executable file T found in the basedir, will
# output "debugfound_task" when LOG=debug.
@test "No tasks requested finds tasks in basedir" {
run $FORGEBUILD -b $FORGEBUILDDIR
[ $status = 0 ]
echo "$output"
[ "${lines[0]}" = "$(info no_task)" ]
[ "${lines[1]}" = "$(debug found_task)" ]
[ "${lines[2]}" = "$(debug found_task)" ]
[ "${lines[3]}" = "$(info config)" ]
[ "${lines[4]}" = "$(debug start_proc)" ]
[ "${lines[5]}" = "$(info process)" ]
[ "${lines[6]}" = "$(info run)" ]
[ "${lines[7]}" = "$(debug start_proc)" ]
[ "${lines[8]}" = "$(info process)" ]
[ "${lines[9]}" = "$(info run)" ]
eq "${lines[0]}" "$(info no_task)"
eq "${lines[1]}" "$(debug found_task)"
eq "${lines[2]}" "$(debug found_task)"
}
@test "no tasks exist" {
# When no tasks are requested, and no tasks are found in the basedir
# forgebuild simply prints "infono_task".
# TODO: maybe introduce a "nothing to do" message here?
@test "No task exists does not fail" {
rm -r $FORGEBUILDDIR/*
run $FORGEBUILD -b $FORGEBUILDDIR
[ $status = 0 ]
echo "$output"
[ "${lines[0]}" = "$(info no_task)" ]
[ "${lines[1]}" = "$(info config)" ]
# TODO: maybe introduce a "nothing to do" message here?
eq "${lines[0]}" "$(info no_task)"
}
# When no tasks are requested, but some tasks are found
# in the basedir, forgebuild starts processing the tasks,
# printing "debugstart_proc", "infoprocess" then "inforun"
# for each task found.
@test "Claims to run all tasks in the basedir" {
run $FORGEBUILD -b $FORGEBUILDDIR
[ $status = 0 ]
echo "$output"
eq "${lines[0]}" "$(info no_task)"
eq "${lines[1]}" "$(debug found_task)"
eq "${lines[2]}" "$(debug found_task)"
# Keep a copy of the lines to match against later.
# WTF i don't know how to copy an array in bash!! Everything
# i try gives me only one entry in the array...
# Quick hack: copy the entire output..
prev_output="$output"
# Maybe there's some additional output (like config)
# to suppress from this test.
CONTEXT="$output" run find_line "$(debug start_proc)"
[ $status = 0 ]
# The line number matching start_proc
i="$output"
# Get the lines again from the original forgebuild output
run echo "$prev_output"
# Now, for each of the two tasks triggered we should receive
# debugstart_proc infoprocess and inforun
eq "${lines[i++]}" "$(debug start_proc)"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
eq "${lines[i++]}" "$(debug start_proc)"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
}
# When a single task is requested via a positional argument, only
# a single task is processed.
@test "Claims to run single task" {
run $FORGEBUILD -b $FORGEBUILDDIR time
[ $status = 0 ]
echo "$output"
eq "${lines[0]}" "$(debug found_task)"
# Quick hack: copy the entire output.. (TODO)
prev_output="$output"
# Maybe there's some additional output (like config)
# to suppress from this test.
CONTEXT="$output" run find_line "$(debug start_proc)"
[ $status = 0 ]
# The line number matching start_proc
i="$output"
# Get the lines again from the original forgebuild output
run echo "$prev_output"
# Now, we should receive debugstart_proc infoprocess and inforun
eq "${lines[i++]}" "$(debug start_proc)"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
}
# When multiple tasks are requested via positional arguments and they exist,
# all those tasks are run.
@test "Claims to run multiple tasks" {
run $FORGEBUILD -b $FORGEBUILDDIR time pwd
[ $status = 0 ]
echo "$output"
eq "${lines[0]}" "$(debug found_task)"
eq "${lines[1]}" "$(debug found_task)"
# Quick hack: copy the entire output.. (TODO)
prev_output="$output"
# Maybe there's some additional output (like config)
# to suppress from this test.
CONTEXT="$output" run find_line "$(debug start_proc)"
[ $status = 0 ]
# The line number matching start_proc
i="$output"
# Get the lines again from the original forgebuild output
run echo "$prev_output"
# Now, for each of the two tasks triggered we should receive
# debugstart_proc infoprocess and inforun
eq "${lines[i++]}" "$(debug start_proc)"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
eq "${lines[i++]}" "$(debug start_proc)"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
}
# In case duplicate tasks are passed as positional arguments, each
# unique task is triggered only once
@test "Doesn't reprocess duplicate tasks" {
run $FORGEBUILD -b $FORGEBUILDDIR time pwd time pwd time
[ $status = 0 ]
echo "$output"
eq "${lines[0]}" "$(debug found_task)"
eq "${lines[1]}" "$(debug found_task)"
# Quick hack: copy the entire output.. (TODO)
prev_output="$output"
# Maybe there's some additional output (like config)
# to suppress from this test.
CONTEXT="$output" run find_line "$(debug start_proc)"
[ $status = 0 ]
# The line number matching start_proc
i="$output"
# Get the lines again from the original forgebuild output
run echo "$prev_output"
# Now, for each of the two tasks triggered we should receive
# debugstart_proc infoprocess and inforun
eq "${lines[i++]}" "$(debug start_proc)"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
eq "${lines[i++]}" "$(debug start_proc)"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
}

117
tests/02-log-level.bats Executable file
View File

@ -0,0 +1,117 @@
#! /usr/bin/env bats
# 02-log-level.bats
# In these tests, we ensure that log levels are respected. Log level is passed
# to forgebuild via the `LOG` environment variable. Its possible values are:
# error, info, debug. Casing may be mixed, so LOG=ERROR is the same as LOG=error.
# When the log level is not recognized or unspecified, it defaults to LOG=info.
# LOG=error only prints errors and warnings.
# LOG=info prints errors, warnings and info.
# LOG=debug prints errors, warnings, info and debug (everything).
# LOG=debug is already covered by the test suite in 01-cli-flags.bats so
# we don't need to check again, just make sure you don't break previous tests
load helpers
function setup {
ORIGDIR="$(pwd)"
TMPDIR="$(mktemp -d)"
# If no $FORGEBUILD is supplied in ENV, use the default
# (otherwise you can't `bats ./tests/` manually)
[ -z "$FORGEBUILD" ] && FORGEBUILD="forgebuild"
# Setup forgebuild tasks
FORGEBUILDDIR="$TMPDIR"/forgebuild
mkdir $FORGEBUILDDIR
echo "date +%s%N > $TMPDIR/forgebuild-time" > $FORGEBUILDDIR/time
chmod +x $FORGEBUILDDIR/time
echo "echo \$(pwd) > $TMPDIR/forgebuild-pwd" > $FORGEBUILDDIR/pwd
chmod +x $FORGEBUILDDIR/pwd
# Disable translations
export LANG="NONE"
# Enable debug for full log
export LOG="debug"
}
function teardown {
cd $ORIGDIR
if [ -d $TMPDIR ]; then rm -rf $TMPDIR; fi
}
# When LOG=info, debug lines are not printed
@test "LOG=info doesn't print debug lines" {
LOG="info" run $FORGEBUILD -b $FORGEBUILDDIR
[ $status = 0 ]
echo "$output"
eq "${lines[0]}" "$(info no_task)"
# Maybe there's some additional output (like config)
# to suppress from this test.
prev_output="$output"
CONTEXT="$output" run find_line "$(info process)"
[ $status = 0 ]
# The line number matching process
i="$output"
run echo "$prev_output"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
}
# When LOG=info, debug lines are not printed
@test "LOG=info doesn't print debug lines" {
LOG="info" run $FORGEBUILD -b $FORGEBUILDDIR
[ $status = 0 ]
echo "$output"
eq "${lines[0]}" "$(info no_task)"
# Maybe there's some additional output (like config)
# to suppress from this test.
prev_output="$output"
CONTEXT="$output" run find_line "$(info process)"
[ $status = 0 ]
# The line number matching process
i="$output"
run echo "$prev_output"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
eq "${lines[i++]}" "$(info process)"
eq "${lines[i++]}" "$(info run)"
}
# Check that LOG variable is case-insensitive
@test "LOG variable is case-insensitive" {
# Do a first run in case sourceless tasks are supported,
# and therefore marked as done, which would invalidate
# equal outputs (first run would run tasks, unlike the second)
$FORGEBUILD -b $FORGEBUILDDIR
run1="$(LOG="info" $FORGEBUILD -b $FORGEBUILDDIR)"
run2="$(LOG="iNfo" $FORGEBUILD -b $FORGEBUILDDIR)"
run3="$(LOG="INFO" $FORGEBUILD -b $FORGEBUILDDIR)"
eq "$run1" "$run2"
eq "$run1" "$run3"
}
# Unknown LOG level is the same as info loglevel
# TODO: maybe wrong loglevel warning?
@test "Unknown LOG level is treated like LOG=info" {
# Do a first run in case sourceless tasks are supported,
# and therefore marked as done, which would invalidate
# equal outputs (first run would run tasks, unlike the second)
$FORGEBUILD -b $FORGEBUILDDIR
run1="$(LOG="info") $FORGEBUILD -b $FORGEBUILDDIR)"
run2="$(LOG="BOGUS") $FORGEBUILD -b $FORGEBUILDDIR)"
eq "$run1" "$run2"
}
# When LOG=error, only error lines are printed
@test "LOG=error only prints error lines" {
LOG=error run $FORGEBUILD -b $FORGEBUILDDIR
eq "$output" ""
}

100
tests/03-log-context.bats Executable file
View File

@ -0,0 +1,100 @@
#! /usr/bin/env bats
# 03-log-context.bats
# The log messages may contain dynamic information about the forgebuild setup
# or the task being processed. In these tests, we check that these variables
# are substituted.
# In order to ensure that, we use a special translation file containing only
# variables to be replaced. This file is passed as LANG environment variable.
load helpers
function setup {
ORIGDIR="$(pwd)"
TMPDIR="$(mktemp -d)"
# If no $FORGEBUILD is supplied in ENV, use the default
# (otherwise you can't `bats ./tests/` manually)
[ -z "$FORGEBUILD" ] && FORGEBUILD="forgebuild"
# Setup forgebuild tasks
FORGEBUILDDIR="$TMPDIR"/forgebuild
mkdir $FORGEBUILDDIR
echo "date +%s%N > $TMPDIR/forgebuild-time" > $FORGEBUILDDIR/time
chmod +x $FORGEBUILDDIR/time
echo "echo \$(pwd) > $TMPDIR/forgebuild-pwd" > $FORGEBUILDDIR/pwd
chmod +x $FORGEBUILDDIR/pwd
# Disable translations
#export LANG="$TMPDIR/translations.json"
export LANG="/tmp/LANG.json"
cat << \EOF > $LANG
{
"error": "error",
"info": "info",
"warn": "warn",
"debug": "debug",
"no_task": "no_task",
"missing_basedir": "$i18n_basedir",
"unknown_arg": "$i18n_arg",
"found_task": "$i18n_task",
"process": "$i18n_task",
"run": "$i18n_task",
"start_proc": "$i18n_task",
"config": "dummy"
}
EOF
# Enable debug for full log
export LOG="debug"
}
function teardown {
cd $ORIGDIR
if [ -d $TMPDIR ]; then rm -rf $TMPDIR; fi
}
@test "LANG=translations.json is loaded as translation file" {
run $FORGEBUILD -b /tmp/THISPATHDOESNOTEXIST
# If we find error/tmp/THISPATHDOESNOTEXIST, the file was loaded and
# the variable correctly substituted so we don't want to error
if [ "$output" != "error/tmp/THISPATHDOESNOTEXIST" ]; then
# If we find error$i18n_basedir, the file was successfully loaded
# but no variable were substituted (no error)
eq "$output" "error\$i18n_basedir"
fi
}
@test "\$i18n_basedir is substituted" {
run $FORGEBUILD -b /tmp/THISPATHDOESNOTEXIST
eq "$output" "error/tmp/THISPATHDOESNOTEXIST"
}
@test "\$i18n_task is substituted" {
run $FORGEBUILD -b $FORGEBUILDDIR
eq "${lines[0]}" "$(info no_task)"
eq "${lines[1]}" "$(debug pwd)"
eq "${lines[2]}" "$(debug time)"
# Maybe there's some additional output (like config)
# to suppress from this test.
prev_output="$output"
echo "$output"
# Found task
# We don't want to match the first debug pwd so we start line 3
CONTEXT="$output" run find_line "$(debug pwd)" 3
[ $status = 0 ]
# The line number matching process
i="$output"
run echo "$prev_output"
eq "${lines[i++]}" "$(debug pwd)"
eq "${lines[i++]}" "$(info pwd)"
eq "${lines[i++]}" "$(info pwd)"
eq "${lines[i++]}" "$(debug time)"
eq "${lines[i++]}" "$(info time)"
eq "${lines[i++]}" "$(info time)"
}

67
tests/helpers.bash Executable file
View File

@ -0,0 +1,67 @@
#! /usr/bin/env bats
# Helper function, returns successfuly when the two values
# passed as positional arguments are equal ($1 = $2), and
# fails otherwise, printing out the two mismatching values
eq() {
if [ "$1" != "$2" ]; then
echo "\"$1\" != \"$2\"" >&2
return 1
fi
}
# Returns successfully when the two values aren't equal,
# and fails otherwise printing out the two matching values
neq() {
if [ "$1" = "$2" ]; then
echo "\"$1\" = \"$2\"" >&2
return 1
fi
}
# Emulate translation, without actual escape codes (colors),
# variable substitution, or actual translations
warn() {
echo "warning${1}"
}
info() {
if [[ "$LOG" = "info" ]] || [[ "$LOG" = "debug" ]] || [[ "$LOG" = "" ]]; then
echo "info${1}"
fi
}
debug() {
[[ "$LOG" = "debug" ]] && echo "debug${1}"
}
error() {
echo "error${1}"
}
# Helper function which returns the index in an array
# of lines which matches the string given as $1
# If $2 is provided, it is the line where we start looking
# (to avoid previous matches)
# Takes the full input as $CONTEXT env variable
# Exits with 1 when the string was not found
find_line() {
if [ $# -gt 1 ]; then
i=$2
else
i=0
fi
run echo "$CONTEXT"
while [ "${lines[i]}" != "" ]; do
if [ "${lines[i]}" = "$1" ]; then
# We've found the line, break out of the loop
break;
fi
((i++))
done
# If the current line is an empty line, we haven't matched
if [ "${lines[i]}" = "" ]; then exit 1; fi
# Output the matching line number
echo $i
}