#!/bin/tcsh
#
# Test the stats programs against a common set of inputs and expected outputs.
# Optional $1 is "-v" for verbose output, "-vv" for very verbose output, or
# "-vvv" for very verbose.  With no arg, this script puts the overall output
# results in output/.all, which is compared to expected-output/.all for
# overall regression purposes.
#
# The idea of "overall regression" is that some individual program tests can
# purposely fail, but the overall reqgression succeesds.  This curious,
# seemingly nonsensical notion is in fact sensible in the pedagogical world.
# This so because for pediagogical purposes, we sometimes use incorrect
# programs as instructioal examples.  The testing failure of such programs is
# therefore expected, and such failure in fact has pedagogical value.  So,
# rather than "grooming out" failed tests, we can, and should leave them in.
#
# One might characterize what we have here as a form of meta-expected results.
# That is, expect expected results not to be matched.  This means the file
# expected-output/.all is a meta-expected output, that encapsulates the
# expected failure of some individual programs.  "Expect the unexpected" might
# be an interesting tag line for this idaa, if not a paper title.
#

#
# Make the output and diffs dirs, if the don't already exist.
#
if (! -e diffs) mkdir diffs
if (! -e output) mkdir output

#
# To account for known diffs in output, e.g., the output from stats-wrong, we
# do overall regression by comparing the the overall outputs between runs.
# These results are stored in outputs/.all and expected-outputs/.all.
#
rm -f output/.all

#
# Iterate through the programs to be tested, putting the output of each in a
# prog-specific output subsubdir, and the diffs, if any, in a prog-specific
# diffs subsubdir.
#
foreach prog_name (stats-wrong stats-no-functions stats-with-functions \
	stats-further-refined stats-non-neg stats-neg-drop stats-neg-error \
	stats-neg-error-return)

    if ($1 == "") then
        run1.csh $prog_name -v >>& output/.all
    else
        run1.csh $prog_name $1
    endif
end

if ($1 == "") then
    diff output/.all expected-output/.all
endif

#
# For very very verbose output, dump all of the diffs, with a bit of file-name
# messaging.
#
if ($1 == "-vvv") then
    echo
    foreach prog_name (stats-wrong stats-no-functions stats-with-functions \
	    stats-further-refined stats-non-neg stats-neg-drop stats-neg-error \
	    stats-neg-error-return)
	if (`ls diffs/$prog_name` != "") then
	    echo '****' $prog_name '****'
	    foreach f (diffs/$prog_name/*)
	    	echo '**' $f:t '**'
	        cat $f
		echo
	    end
	    echo
	endif
    end
endif
