#!/usr/bin/env perl
#
# Summarize cprnc output from all tests in a CESM test suite.
# See usage message for details.
#
# Bill Sacks
# 4-10-13

use strict;
use Getopt::Long;
use File::Basename;
use List::Util qw(max sum);

#----------------------------------------------------------------------
# Get arguments and check them
#----------------------------------------------------------------------

my %opts;
GetOptions(
           "basedir=s"       => \$opts{'basedir'},
           "testid=s"        => \$opts{'testid'},
           "narrow"          => \$opts{'narrow'},
           "h|help"          => \$opts{'help'},
)  or usage();

usage() if $opts{'help'};

if (@ARGV) {
   print "ERROR: unrecognized arguments: @ARGV\n";
   usage();
}

if (!$opts{'basedir'}) {
   print "ERROR: -basedir must be provided\n";
   usage();
}
if (!$opts{'testid'}) {
   print "ERROR: -testid must be provided\n";
   usage();
}

#----------------------------------------------------------------------
# Main script
#----------------------------------------------------------------------

# Create hash containing summary of cprnc differences. This is a reference to a hash, with:
# Keys: "Directory Filename Variable"
# Values: Reference to a hash containing:
#    Dir => directory (gives test name)
#    Filename  => cprnc filename
#    Variable  => variable
#    RMS       => normalized rms value [may or may not be present]
#    FILLDIFF  => ' '                  [may or may not be present]
my ($summary_hash) =
   process_cprnc_output($opts{'basedir'}, $opts{'testid'});

my $outbase="cprnc.summary.$opts{'testid'}";

# set widths of output strings
my $widths_hash;
if ($opts{'narrow'}) {
   $widths_hash = { Dir => 40, Filename => 40, Variable => 40 };
}
else {
   $widths_hash = max_widths($summary_hash);
}
   

print_results_by_test("${outbase}.by_test", $summary_hash, $widths_hash);
print_results_by_varname("${outbase}.by_varname", $summary_hash, $widths_hash);
print_results_by_rms("${outbase}.by_rms", $summary_hash, $widths_hash);



#----------------------------------------------------------------------
# Subroutines
#----------------------------------------------------------------------

sub usage {
   die <<EOF;
SYNOPSIS
    summarize_cprnc_diffs -basedir <basedir> -testid <testid> [OPTIONS]

    <basedir> is the base directory in which test directories can be found

    <testid> is the testid of the tests to summarize
             (can contain shell wildcards)

    This script can be used to post-process and summarize baseline comparison
    output from one or more CESM test suites.

    The script finds all directories in basedir whose name ends with the given
    testid; these are the test directories of interest. It then examines the
    'run' subdirectory of each test directory of interest, looking for files of
    the form *.base.cprnc.out. (Thus, note that it only looks at output for
    baseline comparisons - NOT output from the test itself, such as cprnc output
    files from the exact restart test.)

    Summaries of cprnc differences (normalized RMS differences and FILLDIFFs)
    are placed in three output files beginning with the name 'cprnc.summary', in
    the current directory.  These files contain the same information, but one is
    sorted by test name, one is sorted by variable name, and is one sorted from
    largest to smallest normalized RMS differences.


OPTIONS
    -narrow              Use generally-narrower output field widths to aid readability,
                         at the expense of truncated strings

    -help [or -h]        Display this help

EOF
}


# process_cprnc_output
# Read through all cprnc files, and build hashes of instances of RMS and FILLDIFF
# Inputs:
#  - basedir
#  - testid
# Output: hash reference
# Dies with an error if no cprnc output files are found
sub process_cprnc_output {
   my ($basedir, $testid) = @_;

   my %diffs;
   my $num_files = 0;

   my @test_dirs = glob "${basedir}/*${testid}";

   foreach my $test_dir (@test_dirs) {
      my $test_dir_base = basename($test_dir);

      my @cprnc_files = glob "${test_dir}/run/*.base.cprnc.out";

      foreach my $cprnc_file (@cprnc_files) {
         my $cprnc_file_base = basename($cprnc_file);
         $num_files++;

         open IN, "<", $cprnc_file or die "ERROR opening ${cprnc_file}";
         
         while (my $line = <IN>) {
            chomp $line;

            process_line($line, $test_dir_base, $cprnc_file_base, \%diffs);
         }  # while <IN>

         close IN;
      }  # foreach cprnc_file

   }  # foreach test_dir

   if ($num_files == 0) {
      die "ERROR: no base.cprnc.out files found\n";
   }

   return \%diffs;
}


# process_line: Process one line from one file
# Inputs:
#  - line
#  - test_dir
#  - cprnc_file
#  - diffs hash reference (MODIFIED)
sub process_line {
   my ($line, $test_dir, $cprnc_file, $diffs) = @_;

   my $diff_type;
   my $varname;
   my $rms;
   my $ignore;
   my $rms_normalized;
   
   if ($line =~ /^ *RMS /) {
      ($diff_type, $varname, $rms, $ignore, $rms_normalized) = split " ", $line;
   } elsif ($line =~ /^ *FILLDIFF /) {
      ($diff_type, $varname) = split " ", $line;
      $rms = "";
      $rms_normalized = "";
   } else {
      $diff_type = "";
   }

   if ($diff_type eq 'RMS' || $diff_type eq 'FILLDIFF') {
      # We have found a cprnc difference

      my $key = "$test_dir $cprnc_file $varname";

      # For RMS errors, keep the highest error found
      if ($diff_type eq "RMS") {
         if (exists $diffs->{$key} && exists $diffs->{$key}{$diff_type}) {
            if ($diffs->{$key}{$diff_type} > $rms_normalized) {
               warn "WARNING: Ignoring lower RMS value:        $key : $rms_normalized < $diffs->{$key}{$diff_type}\n";
               return;
            }
            else {
               warn "WARNING: Replacing RMS with higher value: $key : $rms_normalized > $diffs->{$key}{$diff_type}\n";
            }
         }
      }

      # If the diffs hash doesn't already contain information about this
      # directory/filename/variable combo, then we need to create a hash
      # reference with the appropriate basic metadata.
      if (!exists $diffs->{$key}) {
         $diffs->{$key} = {
                         Dir      => $test_dir,
                         Filename => $cprnc_file,
                         Variable => $varname,
                        };
      }

      # Whether or not the hash already contained the given key, we need to add the value
      # of interest -- either the normalized RMS error or the fact that there is a
      # FILLDIFF; in the latter case, note that $rms_normalized is irrelevant, but we use
      # it as the value of the hash anyway for simplicity.
      $diffs->{$key}{$diff_type} = $rms_normalized;
   } elsif ($diff_type ne '') {
      die "Unexpected diff_type: $diff_type";
   }
}


# max_widths
# Inputs:
#  - summary_hash (hash reference)
# Output: reference to a hash containing the maximum width of each of
# the following in the summary hash:
#  - Dir
#  - Filename
#  - Variable
sub max_widths {
   my $summary_hash = shift;

   my %maxes;

   foreach my $var ('Dir','Filename','Variable') {
      $maxes{$var} = max (map { length($summary_hash->{$_}{$var}) } keys %$summary_hash);
   }

   return \%maxes;
}


# print_results_by_test: Print sorted hash entries to a file, sorted by test name
# Inputs:
#  - outfile: name of output file
#  - summary_hash: hash reference containing results to print
#  - widths: hash reference giving widths of output strings
sub print_results_by_test {
   my ($outfile, $summary_hash, $widths) = @_;

   open OUT, ">", "$outfile" or die "ERROR opening $outfile";

   my @sorted_keys = sort{ $summary_hash->{$a}{'Dir'}      cmp $summary_hash->{$b}{'Dir'}
                        or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'}
                        or $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'} }
      keys %$summary_hash;

   my $last_dir;
   my $last_filename;

   my $separator_width = sum(values %$widths) + 30;
   
   for my $key (@sorted_keys) {

      # Print a separator line between different files
      if ($summary_hash->{$key}{'Dir'}      ne $last_dir || 
          $summary_hash->{$key}{'Filename'} ne $last_filename) {
         if ($last_dir && $last_filename) {
            print OUT "=" x $separator_width . "\n";
         }
         $last_dir      = $summary_hash->{$key}{'Dir'};
         $last_filename = $summary_hash->{$key}{'Filename'};
      }

      my $line = format_line($summary_hash->{$key}, $widths);

      print OUT "$line\n";
   }

   close OUT;
}


# print_results_by_varname: Print sorted hash entries to a file, sorted by variable name
# Inputs:
#  - outfile: name of output file
#  - summary_hash: hash reference containing results to print
#  - widths: hash reference giving widths of output strings
sub print_results_by_varname {
   my ($outfile, $summary_hash, $widths) = @_;

   open OUT, ">", "$outfile" or die "ERROR opening $outfile";

   my @sorted_keys = sort{ $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'}
                        or $summary_hash->{$a}{'Dir'}      cmp $summary_hash->{$b}{'Dir'}
                        or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'} }
      keys %$summary_hash;

   my $last_variable;
   
   my $separator_width = sum(values %$widths) + 30;

   for my $key (@sorted_keys) {

      # Print a separator line between different variables
      if ($summary_hash->{$key}{'Variable'} ne $last_variable) {
         if ($last_variable) {
            print OUT "=" x $separator_width . "\n";
         }
         $last_variable = $summary_hash->{$key}{'Variable'};
      }

      my $line = format_line($summary_hash->{$key}, $widths);

      print OUT "$line\n";
   }

   close OUT;
}



# print_results_by_rms: Print sorted hash entries to a file, sorted by RMS
# Inputs:
#  - outfile: name of output file
#  - summary_hash: hash reference containing results to print
#  - widths: hash reference giving widths of output strings
sub print_results_by_rms {
   my ($outfile, $summary_hash, $widths) = @_;

   open OUT, ">", "$outfile" or die "ERROR opening $outfile";

   my @sorted_keys = sort {$summary_hash->{$b}{'RMS'}      <=> $summary_hash->{$a}{'RMS'}
                        or $summary_hash->{$a}{'Dir'}      cmp $summary_hash->{$b}{'Dir'}
                        or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'}
                        or $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'} }
      keys %$summary_hash;

   for my $key (@sorted_keys) {
      my $line = format_line($summary_hash->{$key}, $widths);

      print OUT "$line\n";
   } 

   close OUT;
}


# Inputs:
# - reference to a hash containing:
#   - Dir
#   - Filename
#   - Variable
#   - RMS (optional)
#   - FILLDIFF (optional)
# - widths: hash reference giving widths of output strings
# Return a formatted line for printing
sub format_line {
   my ($hash_ref, $widths) = @_;

   my $dir = $hash_ref->{'Dir'};
   my $filename = $hash_ref->{'Filename'};
   my $variable = $hash_ref->{'Variable'};
   my $rms = "";
   my $filldiff = "";
   if (exists $hash_ref->{'RMS'}) {
      $rms = sprintf(" : RMS %-16g", $hash_ref->{'RMS'});
   }
   if (exists $hash_ref->{'FILLDIFF'}) {
      $filldiff = " : FILLDIFF";
   }

   # for width=40, the format string will contain '%-40.40s'
   my $format = '%-' . $widths->{'Dir'} . '.' . $widths->{'Dir'} . 's : ' .
                '%-' . $widths->{'Filename'} . '.' . $widths->{'Filename'} . 's : ' .
                '%-' . $widths->{'Variable'} . '.' . $widths->{'Variable'} . 's' .
                '%s%s';

   sprintf($format, $dir, $filename, $variable, $filldiff, $rms);
}

#=======================================================================
# Notes about testing: unit tests
#=======================================================================

#-----------------------------------------------------------------------
# Testing process_line
#-----------------------------------------------------------------------

# use Data::Dumper;
#
# my %diffs;
#
# # shouldn't do anything
# process_line("hello", "test_dir1", "file_a", \%diffs);
#
# # test basic filldiff
# process_line("FILLDIFF var1", "test_dir1", "file_b", \%diffs);
#
# # add an RMS to existing filldiff
# process_line("RMS var1 99 NORMALIZED 42", "test_dir1", "file_b", \%diffs);
#
# # test basic rms error
# process_line("RMS var17 99 NORMALIZED 3.14", "test_dir1", "file_b", \%diffs);
#
# # add a filldiff to existing rms error
# process_line("FILLDIFF var17", "test_dir1", "file_b", \%diffs);
#
# # add a filldiff without RMS
# process_line("FILLDIFF var42", "test_dir2", "file_c", \%diffs);
#
# # add an RMS error without filldiff
# process_line("RMS var100 99 NORMALIZED 100", "test_dir2", "file_d", \%diffs);
#
# # test a warning: should issue a warning and replace the above setting
# process_line("RMS var100 99 NORMALIZED 200", "test_dir2", "file_d", \%diffs);
#
# # test a warning: should issue a warning but NOT replace the above setting
# process_line("RMS var100 99 NORMALIZED 50", "test_dir2", "file_d", \%diffs);
#
# print Dumper(\%diffs);


# THE ABOVE SHOULD PRINT:

# WARNING: Replacing RMS with higher value: test_dir2 file_d var100 : 200 > 100
# WARNING: Ignoring lower RMS value:        test_dir2 file_d var100 : 50 < 200
# $VAR1 = {
#           'test_dir1 file_b var17' => {
#                                         'FILLDIFF' => '',
#                                         'RMS' => '3.14',
#                                         'Filename' => 'file_b',
#                                         'Variable' => 'var17',
#                                         'Dir' => 'test_dir1'
#                                       },
#           'test_dir2 file_c var42' => {
#                                         'FILLDIFF' => '',
#                                         'Filename' => 'file_c',
#                                         'Variable' => 'var42',
#                                         'Dir' => 'test_dir2'
#                                       },
#           'test_dir1 file_b var1' => {
#                                        'FILLDIFF' => '',
#                                        'RMS' => '42',
#                                        'Filename' => 'file_b',
#                                        'Variable' => 'var1',
#                                        'Dir' => 'test_dir1'
#                                      },
#           'test_dir2 file_d var100' => {
#                                          'RMS' => 200,
#                                          'Filename' => 'file_d',
#                                          'Variable' => 'var100',
#                                          'Dir' => 'test_dir2'
#                                        }
#         };



#-----------------------------------------------------------------------
# Testing the print routines
#-----------------------------------------------------------------------

# Add the following to the above test code:

# my $widths_hash = { Dir => 40, Filename => 40, Variable => 40 };
# print_results_by_test("testout.by_test", \%diffs, $widths_hash);
# print_results_by_rms("testout.by_rms", \%diffs, $widths_hash);

# This should give:

# $ cat testout.by_rms 
# test_dir2                                : file_d                                   : var100                                   : RMS 200             
# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 42              
# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 3.14            
# test_dir2                                : file_c                                   : var42                                    : FILLDIFF
# $ cat testout.by_test
# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 42              
# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 3.14            
# ======================================================================================================================================================
# test_dir2                                : file_c                                   : var42                                    : FILLDIFF
# ======================================================================================================================================================
# test_dir2                                : file_d                                   : var100                                   : RMS 200             



#=======================================================================
# Notes about testing: integration tests
#=======================================================================

# Test the following

# Note: can do these tests by running the cprnc tests and organizing
# outputs into particular directories.
#
# For each of these tests, sort the different output files and compare
# the sorted files to make sure the same info is in all output files;
# then look at one of the output files.
#
# - no RMS or FILLDIFFs at all (testid that just contains output from
#   comparing control & copy)
#
# - some RMS and some FILLDIFFs, split across 2 directories, each with
#   2 cprnc files (this can be done by comparing the control file with
#   diffs_in_fill.nc, diffs_in_vals.nc, diffs_in_vals_and_diffs_in_fill.nc
#   and diffs_in_vals_and_fill.nc)
#
# - multiple RMS errors to test RMS sorting, split across 2 directories
#   (this can be done by comparing the control file with four of the
#   vals_differ_by_* files)
