From: Andre Noll <maan@systemlinux.org>
Date: Thu, 10 Feb 2011 22:57:55 +0000 (+0100)
Subject: First draft of a test-suite.
X-Git-Tag: v0.4.6~10^2~1
X-Git-Url: http://git.tuebingen.mpg.de/?p=paraslash.git;a=commitdiff_plain;h=c7250fb279af132333266af36e19c156dd84eca1

First draft of a test-suite.

Idea and parts of the implementation taken from test-lib.sh of the
git project. At the moment there are only three basic tests with more
to follow.
---

diff --git a/Makefile.in b/Makefile.in
index 3fbbce21..a246f4c1 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -9,6 +9,7 @@ PACKAGE_VERSION := @PACKAGE_VERSION@
 PACKAGE_STRING := @PACKAGE_STRING@
 install_sh := @install_sh@
 cmdline_dir := @cmdline_dir@
+executables := @executables@
 
 build_date := $(shell date)
 uname_s := $(shell uname -s 2>/dev/null || echo "UNKNOWN_OS")
@@ -67,6 +68,7 @@ man_pages_in := $(patsubst %, web/%.man.in.html, @executables@)
 ggo_dir := ggo
 object_dir := objects
 man_dir := man/man1
+test_dir := t
 
 m4_ggos := afh audioc audiod client filter gui recv server write
 all_ggos := $(m4_ggos) dccp_recv alsa_write oss_write fade http_recv \
@@ -93,10 +95,10 @@ endif
 ifndef BUILD_VERBOSE
 	BUILD_VERBOSE = 0
 endif
-ifeq ($(BUILD_VERBOSE),1)
-	Q =
-else
+ifeq ($(BUILD_VERBOSE),0)
 	Q = @
+else
+	Q =
 endif
 
 .PHONY: all clean distclean maintainer-clean install man tarball\
@@ -277,7 +279,7 @@ clean2: clean
 	$(Q) rm -rf man $(object_dir)
 	$(Q) rm -f *_command_list.*
 
-distclean: clean2
+distclean: clean2 test-clean
 	@[ -z "$(Q)" ] || echo 'DISTCLEAN'
 	$(Q) rm -f Makefile autoscan.log config.status config.log
 	$(Q) rm -rf autom4te.cache aclocal.m4
@@ -313,3 +315,4 @@ $(tarball): $(cmdline_generated)
 %.pdf: %.ps
 	ps2pdf - - < $< > $@
 
+include $(test_dir)/makefile.test
diff --git a/t/.gitignore b/t/.gitignore
new file mode 100644
index 00000000..e61ae9c6
--- /dev/null
+++ b/t/.gitignore
@@ -0,0 +1,2 @@
+/trashes
+/test-results
diff --git a/t/audio_files/short-44100-2.ogg b/t/audio_files/short-44100-2.ogg
new file mode 100644
index 00000000..91981cd0
Binary files /dev/null and b/t/audio_files/short-44100-2.ogg differ
diff --git a/t/makefile.test b/t/makefile.test
new file mode 100644
index 00000000..d4aedf80
--- /dev/null
+++ b/t/makefile.test
@@ -0,0 +1,32 @@
+RM ?= rm -f
+
+results_dir := $(test_dir)/test-results
+trash_dir := $(test_dir)/trashes
+
+test_options := --executables-dir $(shell pwd)
+test_options += --results-dir $(results_dir)
+test_options += --trash-dir $(trash_dir)
+test_options += --executables "$(executables)"
+test_options += --objects "$(basename $(notdir $(all_objs)))"
+
+ifdef V
+	ifeq ("$(origin V)", "command line")
+		test_options += --verbose=$(V)
+	endif
+endif
+
+tests := $(wildcard $(test_dir)/t[0-9][0-9][0-9][0-9]-*.sh)
+
+test: $(tests)
+
+$(tests): all
+	$(Q) $@ $(test_options)
+
+test-help:
+	$(Q) for t in $(tests); do $$t $(test_options) -h; done
+
+test-clean:
+	$(RM) -r $(results_dir)
+	$(RM) -r $(trash_dir)
+
+.PHONY: $(tests) test-help
diff --git a/t/t0000-help-output.sh b/t/t0000-help-output.sh
new file mode 100755
index 00000000..fcc2c783
--- /dev/null
+++ b/t/t0000-help-output.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+test_description='Parse help output of all executables.
+
+Each paraslash executable supports the -h switch which instructs
+the program to print out all available options and to exit. This test
+checks whether this help output looks as expected.'
+
+. ${0%/*}/test-lib.sh
+
+grep_output()
+{
+	local regex='^  -h, --help'
+	$1 -h | grep "$regex"
+}
+
+for exe in $o_executables; do
+	test_expect_success "$exe" "grep_output $o_executables_dir/$exe"
+done
+test_done
diff --git a/t/t0001-oggdec-correctness.sh b/t/t0001-oggdec-correctness.sh
new file mode 100755
index 00000000..01260e5c
--- /dev/null
+++ b/t/t0001-oggdec-correctness.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+test_description='Check correctness of oggdec output.
+
+Executes para_filter -f oggdec on the test files provided by the test
+suite and compares the output against the output of the reference
+implementation.'
+
+. ${0%/*}/test-lib.sh
+
+test_require_objects "oggdec_filter"
+missing_objects="$result"
+
+test_require_executables "oggdec"
+missing_executables="$result"
+
+get_audio_file_paths ogg
+oggs="$result"
+
+for ogg in $oggs; do
+	if [[ -n "$missing_objects" ]]; then
+		test_skip "${ogg##*/}" "missing object(s): $missing_objects"
+		continue
+	fi
+	if [[ -n "$missing_executables" ]]; then
+		test_skip "${ogg##*/}" \
+			"missing executables(s): $missing_executables"
+		continue
+	fi
+	test_expect_success "${ogg##*/}" "
+		$PARA_FILTER -f oggdec < $ogg | sha1sum > filter.sha1 &&
+		oggdec --quiet --raw --output - -  < $ogg | sha1sum > oggdec.sha1 &&
+		diff -u filter.sha1 oggdec.sha1
+	"
+done
+test_done
diff --git a/t/t0002-oggdec-performance.sh b/t/t0002-oggdec-performance.sh
new file mode 100755
index 00000000..d496a3ec
--- /dev/null
+++ b/t/t0002-oggdec-performance.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+test_description='Measure time to decode ogg/vorbis files.
+
+Executes para_filter -f oggdec on the test files provided by the
+test suite and fails if it takes much longer than the reference
+implementation.'
+
+. ${0%/*}/test-lib.sh
+
+test_require_objects "oggdec_filter"
+missing_objects="$result"
+
+get_audio_file_paths ogg
+oggs="$result"
+
+test_require_executables "oggdec"
+missing_executables="$result"
+
+for ogg in $oggs; do
+	if [[ -n "$missing_objects" ]]; then
+		test_skip "${ogg##*/}" "missing object(s): $missing_objects"
+		continue
+	fi
+	if [[ -n "$missing_executables" ]]; then
+		test_skip "${ogg##*/}" \
+			"missing executables(s): $missing_executables"
+		continue
+	fi
+	test_expect_success "${ogg##*/}" '
+		test_duration oggdec --quiet --raw --output - - < $ogg &&
+		t1=$result &&
+		test_duration $PARA_FILTER -f oggdec < $ogg &&
+		t2=$result &&
+		echo "oggdec: $t1, para_filter: $t2"
+		(($t2 <= $t1 * 3 / 2 + 100))
+	'
+done
+test_done
diff --git a/t/test-lib.sh b/t/test-lib.sh
new file mode 100644
index 00000000..e444e578
--- /dev/null
+++ b/t/test-lib.sh
@@ -0,0 +1,292 @@
+#!/bin/bash
+
+# paraslash test suite helper functions
+# Licensed under the GPL v2. For licencing details see COPYING.
+# uses ideas and code from git's test-lib.sh, Copyright (c) 2005 Junio C Hamano
+
+
+get_audio_file_paths()
+{
+	local suffix="$1"
+
+	if (($# == 0)); then
+		result=$(find "$test_audio_file_dir" -type f)
+	else
+		result=$(find "$test_audio_file_dir" -type f -name "*.$suffix")
+	fi
+}
+
+say_color()
+{
+	if [[ "$o_nocolor" != "true" && -n "$1" ]]; then
+		export TERM=$ORIGINAL_TERM
+		case "$1" in
+			error) tput bold; tput setaf 1;;
+			skip)  tput setaf 5;;
+			ok)
+				(($o_verbose == 0)) && return
+				tput setaf 2;;
+			pass)  tput bold; tput setaf 2;;
+			info)  tput setaf 3;;
+			run)
+				(($o_verbose == 0)) && return
+				tput setaf 6;;
+		esac
+	fi
+	shift
+	printf "%s\n" "$*"
+	if [[ "$o_nocolor" != "true" && -n "$1" ]]; then
+		tput sgr0
+		export TERM=dumb
+	fi
+}
+
+die()
+{
+	local code=$?
+	[[ "$exit_ok" == "true" ]] && exit $code
+	say_color error "FATAL: Unexpected exit with code $code"
+	exit 1
+}
+
+error()
+{
+	say_color error "error: $*"
+	exit_ok="true"
+	exit 1
+}
+
+say()
+{
+	say_color info "$*"
+}
+
+_test_run()
+{
+	local f
+
+	let test_count++
+	eval >&3 2>&4 "$2"
+	if (($? == 0)); then
+		let test_success++
+		say_color ok "ok $test_count - $1"
+		return
+	fi
+	let test_failure++
+	say_color error "not ok - $test_count $1"
+	f="$o_results_dir/${0##*/}-$$.out"
+	if [[ -s "$f" ]]; then
+		sed -e 's/^/#	/' < "$f"
+	else
+		sed -e 's/^/#	/' <<< "$2"
+	fi
+	[[ "$o_immediate" != "true" ]] && return
+	exit_ok="true"
+	exit 1
+}
+
+test_skip()
+{
+	(($# != 2)) && error "bug: not 2 parameters to test_skip()"
+	let test_count++
+	let test_skipped++
+	say_color skip >&3 "skipping test $this_test.$test_count ($1): $2"
+	say_color skip "ok $test_count - $1 # skipped ($2)"
+}
+
+test_require_objects()
+{
+	local o1 o2 found
+
+	result=
+	# if no objects were given, we assume this test is run manually
+	# rather than via "make test". We won't check anything in this case
+	[[ -z "$o_objects" ]] && return
+
+	for o1 in $1; do
+		found=
+		for o2 in $o_objects; do
+			[[ "$o1" != "$o2" ]] && continue
+			found="true"
+			break
+		done
+		[[ "$found" == "true" ]] && continue
+		[[ -n "$result" ]] && result+=" "
+		result+="$o1"
+	done
+	[[ -z "$result" ]]
+}
+
+test_require_executables()
+{
+	local i
+
+	result=
+	for i in "$@"; do
+		[[ -n "$(builtin type -t "$i")" ]] && continue
+		[[ -n "$result" ]] && result+=" "
+		result+="$i"
+	done
+	[[ -z "$result" ]]
+}
+
+test_duration()
+{
+	local t=$(exec 2>&1 1>/dev/null; time -p "$@")
+	result=$(awk '{print $2 * 1000}' <<< $t)
+}
+
+test_expect_success()
+{
+	(($# != 2)) && error "bug: not 2 parameters to test_expect_success()"
+	say >&3 "expecting success: $2"
+	_test_run "$1" "$2"
+	echo >&3 ""
+}
+
+test_done()
+{
+	test_results_path="$o_results_dir/${0##*/}-$$.counts"
+	{
+		echo "total $test_count"
+		echo "success $test_success"
+		echo "failed $test_failure"
+		echo "skipped $test_skipped"
+		echo
+	} > $test_results_path
+
+	exit_ok="true"
+	msg="$test_count test(s) ($test_skipped test(s) skipped)"
+	if (($test_failure == 0)); then
+		say_color pass "# ${0##*/}: passed all $msg"
+		exit 0
+	else
+		say_color error "# ${0##*/}: failed $test_failure among $msg"
+		exit 1
+	fi
+}
+
+sanitize_environment()
+{
+	export LANG=C
+	export LC_ALL=C
+	export PAGER=cat
+	export TZ=UTC
+	export TERM=dumb
+	export EDITOR=:
+	export HOME=$(pwd)
+
+	unset VISUAL
+	unset EMAIL
+	unset CDPATH
+	unset GREP_OPTIONS
+}
+
+can_use_colors()
+{
+	result="false"
+	[[ "$TERM" == "dumb" ]] && return
+	[[ -t 1 ]] || return
+	tput bold >/dev/null 2>&1 || return
+	tput setaf 1 >/dev/null 2>&1 || return
+	tput sgr0 >/dev/null 2>&1 || return
+	result="true"
+}
+
+parse_options()
+{
+	while (($# > 0)); do
+		case "$1" in
+		-i|--immediate) o_immediate="true"; shift;;
+		-l|--long) export o_long="true"; shift;;
+		-h|--help) o_help="true"; shift;;
+		-v=0|--verbose=0) o_verbose="0"; shift;;
+		-v=1|--verbose=1) o_verbose="1"; shift;;
+		-v|--verbose|-v=2|--verbose=2) o_verbose="2"; shift;;
+		--no-color) o_nocolor="true"; shift;;
+		--results-dir) o_results_dir="$2"; shift; shift;;
+		--trash-dir) o_trash_dir="$2"; shift; shift;;
+		--executables-dir) export o_executables_dir="$2"; shift; shift;;
+		--executables) export o_executables="$2"; shift; shift;;
+		--objects) export o_objects="$2"; shift; shift;;
+		*) echo "error: unknown test option '$1'" >&2; exit 1;;
+		esac
+	done
+	[[ -z "$o_verbose" ]] && o_verbose=1
+}
+
+create_trash_dir_and_cd()
+{
+	local trash="$o_trash_dir/trash-dir.${0##*/}"
+
+	rm -rf "$trash" || error "could not remove trash dir"
+	mkdir -p "$trash" || error "could not make trash dir"
+	cd "$trash" || error "could not change to trash dir"
+}
+
+fixup_dirs()
+{
+	local wd=$(pwd)
+
+	test_dir="$wd/${0%/*}"
+	test_audio_file_dir="$test_dir/audio_files"
+
+	[[ -z "$o_results_dir" ]] && o_results_dir="$test_dir/test-results"
+	[[ -z "$o_executables_dir" ]] && o_executables_dir="$test_dir/.."
+	[[ -z "$o_trash_dir" ]] && o_trash_dir="$test_dir/trashes"
+
+	# we want alsolute paths because relative paths become invalid
+	# after changing to the trash dir
+	[[ -n "${o_results_dir##/*}" ]] && o_results_dir="$wd/$o_results_dir"
+	[[ -n "${o_executables_dir##/*}" ]] && o_executables_dir="$wd/$o_results_dir"
+	[[ -n "${o_trash_dir##/*}" ]] && o_trash_dir="$wd/$o_trash_dir"
+
+	mkdir -p "$o_results_dir"
+}
+
+parse_options "$@"
+if [[ "$o_nocolor" != "true" ]]; then
+	can_use_colors
+	[[ "$result" != "true" ]] && o_nocolor="true"
+fi
+
+# Each test must set test_description
+[[ -z "${test_description}" ]] && error "${0##*/} did not set test_description"
+if [[ "$o_help" == "true" ]]; then
+	printf "${0##*/}: "
+	sed -e '1!d' <<< "$test_description"
+	if (($o_verbose >= 2)); then
+		echo
+		sed -e '1,2d' -e 's/^/	/g' <<<"$test_description"
+		echo
+	fi
+	exit 0
+fi
+fixup_dirs
+
+[[ -z "$o_executables" ]] && o_executables="para_afh para_audioc para_audiod
+	para_client para_fade para_filter para_gui para_recv para_server
+	para_write"
+for exe in $o_executables; do
+	export $(tr 'a-z' 'A-Z' <<< $exe)="$o_executables_dir/$exe"
+done
+
+test_failure=0
+test_count=0
+test_success=0
+test_skipped=0
+
+ORIGINAL_TERM=$TERM
+sanitize_environment
+create_trash_dir_and_cd
+
+if (($o_verbose >= 2)); then
+	exec 4>&2 3>&1
+else
+	exec 4>$o_results_dir/${0##*/}-$$.out 3>&4
+fi
+
+exit_ok=
+trap 'die' EXIT
+
+say_color run "# running ${0##*/}"
diff --git a/web/manual.m4 b/web/manual.m4
index 946b9d09..5926b555 100644
--- a/web/manual.m4
+++ b/web/manual.m4
@@ -280,7 +280,11 @@ might need to tell the configure script where to find them. Try
 	./configure --help
 
 to see a list of options. If the paraslash package was compiled
-successfully, execute as root,
+successfully, execute (optionally)
+
+	make test
+
+to run the paraslash test suite. If all tests pass, execute as root
 
 	make install