#!/bin/bash

# Bill Sacks
# 4-15-2015

#======================================================================
# Overview
#======================================================================

# This script generates baselines and/or compares with previous baselines, for
# all tests that were run in a single test suite. It then outputs the test
# status (PASS/FAIL/etc.)  for each comparison / generation.
#
# This is meant to be run after all tests in a given test suite are
# complete. Some typical uses are: (1) re-comparing against baselines if some
# baselines did not yet exist when the test suite was run, (2) comparing against
# a different set of baselines from the ones originally specified when you
# created the test suite, (3) generating baselines after the fact (if you ran
# the test suite without -generate).
#
# In addition to some of the options given to create_test_suite, you also need
# to provide the script with (1) the unique test ID of the test suite, and (2)
# the directory in which the various test run directories can be found.
#
# cprnc should be in your path, but if it isn't, the script tries to
# find cprnc in its yellowstone location
#
# See usage message for details on inputs.

#======================================================================
# Limitations
#======================================================================

# - Doesn't look at the PASS/FAIL status of a test. This means, for
#   example, that a -generate may copy files to the baseline directory
#   from a test that failed; these files might not be the correct,
#   end-of-run history files, and so could later result in a
#   misleading FAIL message from a -compare.
#
# - Only looks for history files in the run directory. This will fail
#   to find a history file if things have been moved to the short-term
#   archive directory. This could be fixed to look in the short-term
#   archive directory if a file can't be found in the run directory,
#   as is done in testcase_end. I haven't done this yet, partly
#   because it adds an extra directory that the user has to specify
#   (because we don't have access to the DOUT_S_ROOT variable), and
#   partly because this shouldn't matter for most tests.

#======================================================================
# Testing
#======================================================================

# There currently is no unit test script for this. Here are the tests
# that should be done on this script:
#
# - Missing arguments
#
# - generate only
#
# - compare only
#
# - compare and generate

#======================================================================
# Local functions
#======================================================================

function Usage {
    echo "SYNOPSIS"
    echo "     $progname [options]"
    echo ""
    echo "     Generates baselines and/or compares with previous baselines,"
    echo "     for all tests that were run in a single test suite."
    echo "     It then outputs the test status (PASS/FAIL/etc.) for each comparison / generation."
    echo ""
    echo "     This is meant to be run after all tests in a given test suite are complete."
    echo "     Some typical uses are:"
    echo "     (1) re-comparing against baselines if some baselines did not yet exist"
    echo "         when the test suite was run"
    echo "     (2) comparing against a different set of baselines from the ones originally"
    echo "         specified when you created the test suite"
    echo "     (3) generating baselines after the fact (if you ran the test suite without -generate)"
    echo ""
    echo "     The -baselineroot, -generate and -compare arguments will usually match the arguments"
    echo "     given to create_test_suite. The -testid argument will match the -testid argument"
    echo "     given to create_test_suite, if provided; otherwise, it will be the testid created"
    echo "     on the fly by that script."
    echo ""
    echo "     cprnc should be in your path, but if it isn't, the script tries to find cprnc"
    echo "     in its yellowstone location"
    echo ""
    echo "OPTIONS"
    echo "     -baselineroot <path>  Path to directory containing baselines (required)"
    echo ""
    echo "     -generate <tag>       Tag name to use for generation (optional)"
    echo "                           A directory with this name is created in baselineroot,"
    echo "                           if it doesn't already exist."
    echo "                           Either -generate or -compare must be given"
    echo ""
    echo "     -compare <tag>        Tag name to use for comparison (optional)"
    echo "                           Baselines are found in a directory with the name of this tag"
    echo "                           within baselineroot."
    echo "                           Either -generate or -compare must be given"
    echo ""
    echo "     -testid <id>          ID of the test suite on which to run this script (required)"
    echo "                           (e.g., 123456)"
    echo ""
    echo "     -runloc <path>        Path to directory containing test run directories (optional)"
    echo "                           A given test's run directory can be found in:"
    echo "                           \$runloc/\$CASE/run"
    echo "                           If not given, defaults to: $runloc_default"
    echo ""
    echo "     -help                 Print this help message and exit"
    echo ""
    echo "EXAMPLES"
    echo "     $progname"
    echo "       -baselineroot /glade/scratch/\$USER/cesm_baselines"
    echo "       -generate cesm1_1_beta17"
    echo "       -compare cesm1_1_beta16"
    echo "       -testid 123456"
    echo "       -runloc /glade/scratch/\$USER"
    echo "     This will find directories in /glade/scratch/\$USER whose name ends with"
    echo "     the testid, 123456. For each such test (say, SMS.f09_g16.IG.yellowstone_intel.GC.123456)"
    echo "     it will find the last history files in the run directory of that test,"
    echo "     for each possible component (cpl and clm - including h0, h1, etc.)."
    echo "     It will then copy those history files to"
    echo "     /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta17/SMS.f09_g16.IG.yellowstone_intel"
    echo "     and will compare that history file with the existing baseline in"
    echo "     /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta16/SMS.f09_g16.IG.yellowstone_intel"
    echo ""
}

#======================================================================
# Set parameters
#======================================================================
# If cprnc isn't in your path, this is where to find it
cprnc_default=/glade/p/cesm/cseg/tools/cprnc/cprnc

#======================================================================
# Begin main script
#======================================================================

progname=`basename $0`
tools_dir=`dirname $0`

#----------------------------------------------------------------------
# Define default values for command-line arguments
#----------------------------------------------------------------------
baselineroot=''
generate_tag=''
compare_tag=''
testid=''

runloc_default="/glade/scratch/$USER"
runloc=$runloc_default

#----------------------------------------------------------------------
# Process command-line arguments
#----------------------------------------------------------------------
while [ $# -gt 0 ]; do
    case $1 in
	-baselineroot )
	    baselineroot=$2
	    shift
	    ;;
	-generate )
	    generate_tag=$2
	    shift
	    ;;
	-compare )
	    compare_tag=$2
	    shift
	    ;;
	-testid )
	    testid=$2
	    shift
	    ;;
	-runloc )
	    runloc=$2
	    shift
	    ;;
	-help )
	    Usage
	    exit 0
	    ;;
	* )
	    echo "Unknown argument: $1" >&2
	    echo "Run $progname -help for usage" >&2
	    exit 1
	    ;;
    esac
    shift
done

#----------------------------------------------------------------------
# Exit if required command-line arguments weren't provided 
#----------------------------------------------------------------------
error=0  # no errors yet
if [ -z "$baselineroot" ]; then
    echo "$progname: baselineroot must be provided" >&2
    error=1
fi
# Either -generate or -compare must be provided, but not necessarily both
if [[ -z "$generate_tag" && -z "$compare_tag" ]]; then
    echo "$progname: At least one of generate_tag or compare_tag must be provided" >&2
    error=1
fi
if [ -z "$testid" ]; then
    echo "$progname: testid must be provided" >&2
    error=1
fi
if [ -z "$runloc" ]; then
    echo "$progname: runloc must be provided" >&2
    error=1
fi

if [ $error -gt 0 ]; then
    echo "" >&2
    echo "Run $progname -help for usage" >&2
    exit 1
fi

#----------------------------------------------------------------------
# Determine path to cprnc
#----------------------------------------------------------------------
cprnc_path=`command -v cprnc`
if [ $? -gt 0 ]; then
    # cprnc not found in path; use default
    cprnc_path=$cprnc_default
fi

#----------------------------------------------------------------------
# Loop over directories with the given testid
#----------------------------------------------------------------------
tests=`cd $runloc; ls -1d *.${testid}`
for testcase in $tests; do
    # testcase will look like: SMS.T31_g37.IG4804.yellowstone_intel.134426
    # optionally with .C or .G or .GC before the testid
    # Form testcase_base by stripping off the trailing testid and the optional .C, .G or .GC:
    testcase_base=`echo $testcase | perl -p -e "s/(\.G?C?)?\.\Q$testid\E\$//"`

    if [ "$testcase_base" == "sharedlibroot" ]; then
	continue
    fi

    rundir=${runloc}/${testcase}/run

    # We need to call component_compgen_baseline.sh separately for compare &
    # generate, because currently it can only do one at a time (because only one
    # baseline_dir can be provided, and the baseline_dir is assumed to contain
    # the actual tag, rather than being a directory with all tags' baselines)

    if [ -n "$compare_tag" ]; then
	echo "--- Baseline Comparison ---: "
	${tools_dir}/component_compgen_baseline.sh -baseline_dir ${baselineroot}/${compare_tag}/${testcase_base} -test_dir ${rundir} -testcase ${testcase} -testcase_base ${testcase_base} -compare_tag ${compare_tag} -cprnc_exe ${cprnc_path}
    fi

    if [ -n "$generate_tag" ]; then
	echo "--- Baseline Generation ---: "
	${tools_dir}/component_compgen_baseline.sh -baseline_dir ${baselineroot}/${generate_tag}/${testcase_base} -test_dir ${rundir} -testcase ${testcase} -testcase_base ${testcase_base} -generate_tag ${generate_tag} -cprnc_exe ${cprnc_path}
    fi

    echo ""
done

