#!/usr/bin/env python

import unittest, sys, os, tempfile, shutil, re, threading, time, signal, glob, stat, traceback

SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(SCRIPT_DIR)
from acme_util import run_cmd
import acme_util, update_acme_tests, create_test_impl, wait_for_tests

DART_CONFIG = "DartConfiguration.tcl"
DART_BACKUP = "temp_dart_backup"

###############################################################################
class RunUnitTests(unittest.TestCase):
###############################################################################

    def do_unit_tests(self, script):
        stat, output, _ = run_cmd("./%s --test 2>&1" % script, ok_to_fail=True, from_dir=SCRIPT_DIR)
        self.assertEqual(stat, 0, msg=output)

    def test_acme_bisect_unit_test(self):
        self.do_unit_tests("acme_bisect")

    def test_cime_merge_helper_unit_test(self):
        self.do_unit_tests("cime_merge_helper")

    def test_compare_namelists_unit_test(self):
        self.do_unit_tests("compare_namelists")

    def test_jenkins_generic_job_unit_test(self):
        self.do_unit_tests("jenkins_generic_job")

    def test_simple_compare_unit_test(self):
        self.do_unit_tests("simple_compare")

    def test_update_acme_tests_unit_test(self):
        self.do_unit_tests("update_acme_tests")

    def test_list_acme_tests_unit_test(self):
        self.do_unit_tests("list_acme_tests")

    def test_wait_for_tests_unit_test(self):
        self.do_unit_tests("wait_for_tests")

    def test_acme_util_unit_test(self):
        # A little different since this is a library, not an executable
        stat, output, _ = run_cmd("python -m doctest acme_util.py -v 2>&1", ok_to_fail=True, from_dir=SCRIPT_DIR)
        self.assertEqual(stat, 0, msg=output)

###############################################################################
def make_fake_teststatus(path, testname, status, phase):
###############################################################################
    with open(path, "w") as fd:
        fd.write("%s %s %s\n" % (status, testname, phase))

###############################################################################
def parse_test_status(line):
###############################################################################
    regex = re.compile(r"Test '(\w+)' finished with status '(\w+)'")
    m = regex.match(line)
    return m.groups()

###############################################################################
def kill_subprocesses(name=None, sig=signal.SIGKILL, expected_num_killed=None, tester=None):
###############################################################################
    # Kill all subprocesses
    proc_ids = acme_util.find_proc_id(proc_name=name, children_only=True)
    if (expected_num_killed is not None):
        tester.assertEqual(len(proc_ids), expected_num_killed,
                           msg="Expected to find %d processes to kill, found %d" % (expected_num_killed, len(proc_ids)))
    for proc_id in proc_ids:
        try:
            os.kill(proc_id, sig)
        except OSError:
            pass

###############################################################################
def kill_python_subprocesses(sig=signal.SIGKILL, expected_num_killed=None, tester=None):
###############################################################################
    kill_subprocesses("[Pp]ython", sig, expected_num_killed, tester)

###########################################################################
def assert_dashboard_has_build(tester, build_name, expected_count=1):
###########################################################################
    time.sleep(10) # Give chance for cdash to update

    wget_file = tempfile.mktemp()

    run_cmd("wget http://my.cdash.org/index.php?project=ACME_test -O %s" % wget_file)

    raw_text = open(wget_file, "r").read()
    os.remove(wget_file)

    num_found = raw_text.count(build_name)
    tester.assertEqual(num_found, expected_count,
                       msg="Dashboard did not have expected num occurances of build name '%s'. Expected %s, found %s" % (build_name, expected_count, num_found))

###############################################################################
def setup_proxy():
###############################################################################
    if ("http_proxy" not in os.environ):
        machine = acme_util.probe_machine_name()
        if (machine is not None):
            proxy = acme_util.get_machine_info("PROXY")
            if (proxy is not None):
                os.environ["http_proxy"] = proxy
                return True

    return False

###############################################################################
class TestWaitForTests(unittest.TestCase):
###############################################################################

    ###########################################################################
    def setUp(self):
    ###########################################################################
        self._testdir_all_pass     = tempfile.mkdtemp()
        self._testdir_with_fail    = tempfile.mkdtemp()
        self._testdir_unfinished   = tempfile.mkdtemp()

        for r in range(10):
            make_fake_teststatus(os.path.join(self._testdir_all_pass, "TestStatus_%d" % r), "Test_%d" % r, "PASS", "PHASE_%s" % r)

        for r in range(10):
            make_fake_teststatus(os.path.join(self._testdir_with_fail, "TestStatus_%d" % r), "Test_%d" % r, "PASS" if r % 2 == 0 else "FAIL", "PHASE_%s" % r)

        for r in range(10):
            make_fake_teststatus(os.path.join(self._testdir_unfinished, "TestStatus_%d" % r), "Test_%d" % r, "PEND" if r == 5 else "PASS", "PHASE_%s" % r)

        # Set up proxy if possible
        self._unset_proxy = setup_proxy()

        self._thread_error = None

        # wait_for_tests could blow away our dart config, need to back it up
        if (os.path.exists(DART_CONFIG)):
            shutil.move(DART_CONFIG, DART_BACKUP)

    ###########################################################################
    def tearDown(self):
    ###########################################################################
        shutil.rmtree(self._testdir_all_pass)
        shutil.rmtree(self._testdir_with_fail)
        shutil.rmtree(self._testdir_unfinished)

        kill_subprocesses()

        if (self._unset_proxy):
            del os.environ["http_proxy"]

        if (os.path.exists(DART_BACKUP)):
            shutil.move(DART_BACKUP, DART_CONFIG)

    ###########################################################################
    def simple_test(self, testdir, expected_results, extra_args=""):
    ###########################################################################
        stat, output, errput = run_cmd("%s/wait_for_tests -p ACME_test TestStatus* %s" % (SCRIPT_DIR, extra_args), ok_to_fail=True, from_dir=testdir)
        if (expected_results == ["PASS"]*len(expected_results)):
            self.assertEqual(stat, 0, msg="COMMAND SHOULD HAVE WORKED\nwait_for_tests output:\n%s\n\nerrput:\n%s" % (output, errput))
        else:
            self.assertEqual(stat, acme_util.TESTS_FAILED_ERR_CODE,
                             msg="COMMAND SHOULD HAVE DETECTED FAILED TESTS\nwait_for_tests output:\n%s\n\nerrput:\n%s" % (output, errput))

        lines = [line for line in output.splitlines() if line.startswith("Test '")]
        self.assertEqual(len(lines), 10)
        for idx, line in enumerate(lines):
            testname, status = parse_test_status(line)
            self.assertEqual(status, expected_results[idx])
            self.assertEqual(testname, "Test_%d" % idx)

    ###########################################################################
    def threaded_test(self, testdir, expected_results, extra_args=""):
    ###########################################################################
        try:
            self.simple_test(testdir, expected_results, extra_args)
        except AssertionError as e:
            self._thread_error = str(e)

    ###########################################################################
    def test_wait_for_test_all_pass(self):
    ###########################################################################
        self.simple_test(self._testdir_all_pass, ["PASS"] * 10)

    ###########################################################################
    def test_wait_for_test_with_fail(self):
    ###########################################################################
        expected_results = ["PASS" if item % 2 == 0 else "FAIL" for item in range(10)]
        self.simple_test(self._testdir_with_fail, expected_results)

    ###########################################################################
    def test_wait_for_test_no_wait(self):
    ###########################################################################
        expected_results = ["PEND" if item == 5 else "PASS" for item in range(10)]
        self.simple_test(self._testdir_unfinished, expected_results, "-n")

    ###########################################################################
    def test_wait_for_test_wait(self):
    ###########################################################################
        run_thread = threading.Thread(target=self.threaded_test, args=(self._testdir_unfinished, ["PASS"] * 10))
        run_thread.daemon = True
        run_thread.start()

        time.sleep(5) # Kinda hacky

        self.assertTrue(run_thread.isAlive(), msg="wait_for_tests should have waited")

        make_fake_teststatus(os.path.join(self._testdir_unfinished, "TestStatus_5"), "Test_5", "PASS", "PHASE_5")

        run_thread.join(timeout=10)

        self.assertFalse(run_thread.isAlive(), msg="wait_for_tests should have finished")

        self.assertTrue(self._thread_error is None, msg="Thread had failure: %s" % self._thread_error)

    ###########################################################################
    def test_wait_for_test_wait_kill(self):
    ###########################################################################
        expected_results = ["PEND" if item == 5 else "PASS" for item in range(10)]
        run_thread = threading.Thread(target=self.threaded_test, args=(self._testdir_unfinished, expected_results))
        run_thread.daemon = True
        run_thread.start()

        time.sleep(5)

        self.assertTrue(run_thread.isAlive(), msg="wait_for_tests should have waited")

        kill_python_subprocesses(signal.SIGTERM, expected_num_killed=1, tester=self)

        run_thread.join(timeout=10)

        self.assertFalse(run_thread.isAlive(), msg="wait_for_tests should have finished")

        self.assertTrue(self._thread_error is None, msg="Thread had failure: %s" % self._thread_error)

    ###########################################################################
    def test_wait_for_test_cdash_pass(self):
    ###########################################################################
        expected_results = ["PASS"] * 10
        run_thread = threading.Thread(target=self.threaded_test, args=(self._testdir_all_pass, expected_results, "-d regression_test_pass"))
        run_thread.daemon = True
        run_thread.start()

        run_thread.join(timeout=10)

        self.assertFalse(run_thread.isAlive(), msg="wait_for_tests should have finished")

        self.assertTrue(self._thread_error is None, msg="Thread had failure: %s" % self._thread_error)

        assert_dashboard_has_build(self, "regression_test_pass")

    ###########################################################################
    def test_wait_for_test_cdash_kill(self):
    ###########################################################################
        expected_results = ["PEND" if item == 5 else "PASS" for item in range(10)]
        run_thread = threading.Thread(target=self.threaded_test, args=(self._testdir_unfinished, expected_results, "-d regression_test_kill"))
        run_thread.daemon = True
        run_thread.start()

        time.sleep(5)

        self.assertTrue(run_thread.isAlive(), msg="wait_for_tests should have waited")

        kill_python_subprocesses(signal.SIGTERM, expected_num_killed=1, tester=self)

        run_thread.join(timeout=10)

        self.assertFalse(run_thread.isAlive(), msg="wait_for_tests should have finished")
        self.assertTrue(self._thread_error is None, msg="Thread had failure: %s" % self._thread_error)

        assert_dashboard_has_build(self, "regression_test_kill")

        cdash_result_dir = os.path.join(self._testdir_unfinished, "Testing")
        tag_file         = os.path.join(cdash_result_dir, "TAG")
        self.assertTrue(os.path.isdir(cdash_result_dir))
        self.assertTrue(os.path.isfile(tag_file))

        tag = open(tag_file, "r").readlines()[0].strip()
        xml_file = os.path.join(cdash_result_dir, tag, "Test.xml")
        self.assertTrue(os.path.isfile(xml_file))

        xml_contents = open(xml_file, "r").read()
        self.assertTrue(r'<TestList><Test>Test_0</Test><Test>Test_1</Test><Test>Test_2</Test><Test>Test_3</Test><Test>Test_4</Test><Test>Test_5</Test><Test>Test_6</Test><Test>Test_7</Test><Test>Test_8</Test><Test>Test_9</Test></TestList>'
                        in xml_contents)
        self.assertTrue(r'<Test Status="failed"><Name>Test_5</Name>' in xml_contents)

        # TODO: Any further checking of xml output worth doing?

###############################################################################
class TestCreateTestCommon(unittest.TestCase):
###############################################################################

    ###########################################################################
    def setUp(self):
    ###########################################################################
        self._thread_error      = None
        self._unset_proxy       = setup_proxy()
        self._baseline_name     = "fake_testing_only_%s" % acme_util.get_utc_timestamp()
        self._baseline_area      = acme_util.get_machine_info("CCSM_BASELINE")
        self._testroot          = acme_util.get_machine_info("CESMSCRATCHROOT")
        self._jenkins_root      = os.path.join(self._testroot, "jenkins")
        self._machine           = acme_util.probe_machine_name()
        self._compiler          = acme_util.get_machine_info("COMPILERS")[0]

        # wait_for_tests could blow away our dart config, need to back it up
        if (os.path.exists(DART_CONFIG)):
            shutil.move(DART_CONFIG, DART_BACKUP)

    ###########################################################################
    def tearDown(self):
    ###########################################################################
        kill_subprocesses()

        if (self._unset_proxy):
            del os.environ["http_proxy"]

        baselines = os.path.join(self._baseline_area, self._compiler, self._baseline_name)
        if (os.path.isdir(baselines)):
            shutil.rmtree(baselines)

        for root in [self._testroot, self._jenkins_root]:
            for test_id in ["master", self._baseline_name]:
                for leftover in glob.glob(os.path.join(root, "*%s*/" % test_id)):
                    shutil.rmtree(leftover)
                for leftover in glob.glob(os.path.join(root, "*%s*" % test_id)):
                    os.remove(leftover)

        if (os.path.exists(DART_BACKUP)):
            shutil.move(DART_BACKUP, DART_CONFIG)

###############################################################################
class TestCreateTest(TestCreateTestCommon):
###############################################################################

    ###########################################################################
    def simple_test(self, expect_works, extra_args):
    ###########################################################################
        stat, output, errput = run_cmd("%s/create_test acme_test_only_pass %s" % (SCRIPT_DIR, extra_args), ok_to_fail=True)
        if (expect_works):
            self.assertEqual(stat, 0, msg="COMMAND SHOULD HAVE WORKED\ncreate_test output:\n%s\n\nerrput:\n%s\n\ncode: %d" % (output, errput, stat))
        else:
            self.assertEqual(stat, acme_util.TESTS_FAILED_ERR_CODE, msg="COMMAND SHOULD HAVE DETECTED FAILED TESTS\ncreate_test output:\n%s\n\nerrput:\n%ss\n\ncode: %d" % (output, errput, stat))

    ###############################################################################
    def test_create_test_rebless_namelist(self):
    ###############################################################################
        # Generate some namelist baselines
        self.simple_test(True, "-g -n -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, acme_util.get_utc_timestamp()))

        # Basic namelist compare
        self.simple_test(True, "-c -n -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, acme_util.get_utc_timestamp()))

        # Modify namelist
        fake_nl = """
 &fake_nml
   fake_item = 'fake'
   fake = .true.
/"""
        baseline_area = acme_util.get_machine_info("CCSM_BASELINE")
        compiler      = acme_util.get_machine_info("COMPILERS")[0]
        baseline_glob = glob.glob(os.path.join(baseline_area, compiler, self._baseline_name, "TEST*"))
        self.assertEqual(len(baseline_glob), 3, msg="Expected three matches, got:\n%s" % "\n".join(baseline_glob))

        baseline_dir = baseline_glob[0]
        nl_path = os.path.join(baseline_dir, "CaseDocs", "datm_in")
        self.assertTrue(os.path.isfile(nl_path), msg="Missing file %s" % nl_path)

        import stat
        os.chmod(nl_path, stat.S_IRUSR | stat.S_IWUSR)
        with open(nl_path, "a") as nl_file:
            nl_file.write(fake_nl)

        # Basic namelist compare should now fail
        self.simple_test(False, "-c -n -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, acme_util.get_utc_timestamp()))

        # Regen
        self.simple_test(True, "-g -n -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, acme_util.get_utc_timestamp()))

        # Basic namelist compare should now pass again
        self.simple_test(True, "-c -n -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, acme_util.get_utc_timestamp()))

###############################################################################
class TestCreateTestImpl(TestCreateTestCommon):
###############################################################################

    ###########################################################################
    def test_phases(self):
    ###########################################################################
        tests = update_acme_tests.get_full_test_names(["acme_test_only"], self._machine, self._compiler)
        ct = create_test_impl.CreateTest(tests)

        build_fail_test = tests[0]
        run_fail_test   = tests[1]
        pass_test       = tests[2]

        self.assertTrue("BUILDFAIL" in build_fail_test, msg="Wrong test '%s'" % build_fail_test)
        self.assertTrue("RUNFAIL" in run_fail_test, msg="Wrong test '%s'" % run_fail_test)
        self.assertTrue("RUNPASS" in pass_test, msg="Wrong test '%s'" % pass_test)

        for idx, phase in enumerate(ct._phases):
            for test in tests:
                if (phase == create_test_impl.INITIAL_PHASE):
                    continue

                elif (phase == create_test_impl.BUILD_PHASE):
                    ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS)

                    if (test == build_fail_test):
                        ct._update_test_status(test, phase, wait_for_tests.TEST_FAIL_STATUS)
                        self.assertTrue(ct._is_broken(test))
                        self.assertFalse(ct._work_remains(test))
                    else:
                        ct._update_test_status(test, phase, wait_for_tests.TEST_PASS_STATUS)
                        self.assertFalse(ct._is_broken(test))
                        self.assertTrue(ct._work_remains(test))

                elif (phase == create_test_impl.RUN_PHASE):
                    if (test == build_fail_test):
                        with self.assertRaises(SystemExit):
                            ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS)
                    else:
                        ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS)
                        self.assertFalse(ct._work_remains(test))

                        if (test == run_fail_test):
                            ct._update_test_status(test, phase, wait_for_tests.TEST_FAIL_STATUS)
                            self.assertTrue(ct._is_broken(test))
                        else:
                            ct._update_test_status(test, phase, wait_for_tests.TEST_PASS_STATUS)
                            self.assertFalse(ct._is_broken(test))

                    self.assertFalse(ct._work_remains(test))

                else:
                    with self.assertRaises(SystemExit):
                        ct._update_test_status(test, ct._phases[idx+1], wait_for_tests.TEST_PENDING_STATUS)

                    with self.assertRaises(SystemExit):
                        ct._update_test_status(test, phase, wait_for_tests.TEST_PASS_STATUS)

                    ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS)
                    self.assertFalse(ct._is_broken(test))
                    self.assertTrue(ct._work_remains(test))

                    with self.assertRaises(SystemExit):
                        ct._update_test_status(test, phase, wait_for_tests.TEST_PENDING_STATUS)

                    ct._update_test_status(test, phase, wait_for_tests.TEST_PASS_STATUS)

                    with self.assertRaises(SystemExit):
                        ct._update_test_status(test, phase, wait_for_tests.TEST_FAIL_STATUS)

                    self.assertFalse(ct._is_broken(test))
                    self.assertTrue(ct._work_remains(test))

    ###########################################################################
    def test_full(self):
    ###########################################################################
        tests = update_acme_tests.get_full_test_names(["acme_test_only"], self._machine, self._compiler)
        test_id="%s-%s" % (self._baseline_name, acme_util.get_utc_timestamp())
        ct = create_test_impl.CreateTest(tests, test_id=test_id)

        build_fail_test = tests[0]
        run_fail_test   = tests[1]
        pass_test       = tests[2]

        ct.create_test()
        if (acme_util.does_machine_have_batch()):
            run_cmd("%s/wait_for_tests *%s*/TestStatus*" % (SCRIPT_DIR, test_id), ok_to_fail=True, from_dir=self._testroot)

        test_statuses = glob.glob("%s/*%s*/TestStatus" % (self._testroot, test_id))
        self.assertEqual(len(tests), len(test_statuses))

        for test_status in test_statuses:
            status, test_name = wait_for_tests.parse_test_status_file(test_status)
            if (test_name == build_fail_test):
                self.assertEqual(status[create_test_impl.BUILD_PHASE], wait_for_tests.TEST_FAIL_STATUS)
            elif (test_name == run_fail_test):
                self.assertEqual(status[create_test_impl.RUN_PHASE], wait_for_tests.TEST_FAIL_STATUS)
            else:
                self.assertEqual(test_name, pass_test)
                self.assertEqual(status[create_test_impl.RUN_PHASE], wait_for_tests.TEST_PASS_STATUS)

###############################################################################
class TestJenkinsGenericJob(TestCreateTestCommon):
###############################################################################

    ###########################################################################
    def simple_test(self, expect_works, extra_args):
    ###########################################################################
        stat, output, errput = run_cmd("%s/jenkins_generic_job %s" % (SCRIPT_DIR, extra_args), ok_to_fail=True)
        if (expect_works):
            self.assertEqual(stat, 0, msg="COMMAND SHOULD HAVE WORKED\njenkins_generic_job output:\n%s\n\nerrput:\n%s" % (output, errput))
        else:
            self.assertEqual(stat, acme_util.TESTS_FAILED_ERR_CODE, msg="COMMAND SHOULD HAVE DETECTED FAILED TESTS\njenkins_generic_job output:\n%s\n\nerrput:\n%s" % (output, errput))

    ###########################################################################
    def threaded_test(self, expect_works, extra_args):
    ###########################################################################
        try:
            self.simple_test(expect_works, extra_args)
        except AssertionError as e:
            self._thread_error = str(e)

    ###########################################################################
    def assert_no_sentinel(self):
    ###########################################################################
        self.assertFalse(os.path.isfile(os.path.join(self._testroot, "ONGOING_TEST")),
                         "job did not cleanup successfully")

    ###########################################################################
    def assert_num_leftovers(self, test_id=None):
    ###########################################################################
        # There should only be two directories matching the test_id in both
        # the testroot (bld/run dump area) and jenkins root
        if (test_id is None):
            test_id = self._baseline_name
        num_tests_in_tiny = len(update_acme_tests.get_test_suite("acme_test_only_pass"))

        jenkins_dirs = glob.glob("%s/*%s*/" % (self._jenkins_root, test_id))
        scratch_dirs = glob.glob("%s/*%s*/" % (self._testroot, test_id))

        self.assertEqual(num_tests_in_tiny, len(jenkins_dirs),
                         msg="Wrong number of leftover directories in %s, expected %d, see %s" % \
                             (self._jenkins_root, num_tests_in_tiny, jenkins_dirs))

        self.assertEqual(num_tests_in_tiny, len(scratch_dirs),
                         msg="Wrong number of leftover directories in %s, expected %d, see %s" % \
                             (self._testroot, num_tests_in_tiny, scratch_dirs))

    ###########################################################################
    def test_jenkins_generic_job(self):
    ###########################################################################
        # Unfortunately, this test is very long-running

        # Generate fresh baselines so that this test is not impacted by
        # unresolved diffs
        self.simple_test(True, "-t acme_test_only_pass -g -b %s" % self._baseline_name)
        self.assert_num_leftovers()

        build_name = "jenkins_generic_job_pass_%s" % acme_util.get_utc_timestamp()
        self.simple_test(True, "-t acme_test_only_pass -p ACME_test -b %s -d -c %s --cdash-build-group=Nightly" % (self._baseline_name, build_name))
        self.assert_num_leftovers() # jenkins_generic_job should have automatically cleaned up leftovers from prior run
        self.assert_no_sentinel()
        assert_dashboard_has_build(self, build_name)

    ###########################################################################
    def test_jenkins_generic_job_kill(self):
    ###########################################################################
        build_name = "jenkins_generic_job_kill_%s" % acme_util.get_utc_timestamp()
        run_thread = threading.Thread(target=self.threaded_test, args=(False, " -t acme_test_only_slow_pass -p ACME_test -b master --baseline-compare=no -d -c %s --cdash-build-group=Nightly" % build_name))
        run_thread.daemon = True
        run_thread.start()

        time.sleep(60)

        kill_subprocesses(sig=signal.SIGTERM)

        run_thread.join(timeout=10)

        self.assertFalse(run_thread.isAlive(), msg="jenkins_generic_job should have finished")
        self.assertTrue(self._thread_error is None, msg="Thread had failure: %s" % self._thread_error)
        self.assert_no_sentinel()
        assert_dashboard_has_build(self, build_name)

###############################################################################
class TestBlessTestResults(TestCreateTestCommon):
###############################################################################

    _test_name = "TESTRUNDIFF_P1.f19_g16_rx1.A"

    ###########################################################################
    def simple_test(self, expect_works, extra_args):
    ###########################################################################
        stat, output, errput = run_cmd("%s/create_test %s %s" % (SCRIPT_DIR, self._test_name, extra_args), ok_to_fail=True)
        if (expect_works):
            self.assertEqual(stat, 0, msg="COMMAND SHOULD HAVE WORKED\ncreate_test output:\n%s\n\nerrput:\n%s\n\ncode: %d" % (output, errput, stat))
        else:
            self.assertEqual(stat, acme_util.TESTS_FAILED_ERR_CODE, msg="COMMAND SHOULD HAVE DETECTED FAILED TESTS\ncreate_test output:\n%s\n\nerrput:\n%ss\n\ncode: %d" % (output, errput, stat))

    ###############################################################################
    def test_create_test_rebless_namelist(self):
    ###############################################################################
        # Generate some namelist baselines
        self.simple_test(True, "-g -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, acme_util.get_utc_timestamp()))

        # Change baseline
        baseline_area = acme_util.get_machine_info("CCSM_BASELINE")
        compiler      = acme_util.get_machine_info("COMPILERS")[0]
        baseline_glob = glob.glob(os.path.join(baseline_area, compiler, self._baseline_name, "%s*" % self._test_name))
        self.assertEqual(len(baseline_glob), 1, msg="Expected one match, got:\n%s" % "\n".join(baseline_glob))

        baseline_dir = baseline_glob[0]
        os.remove(os.path.join(baseline_dir, "cpl.hi.nc"))
        shutil.copy(os.path.join(SCRIPT_DIR, "tests", "cpl.hi2.nc.test"), os.path.join(baseline_dir, "cpl.hi.nc"))

        # Hist compare should now fail
        test_id = "%s-%s" % (self._baseline_name, acme_util.get_utc_timestamp())
        self.simple_test(False, "-c -b %s -t %s" % (self._baseline_name, test_id))

        # Bless
        run_cmd("%s/bless_test_results --hist-only --force -b %s -t %s" % (SCRIPT_DIR, self._baseline_name, test_id))

        # Hist compare should now pass again
        self.simple_test(True, "-c -b %s -t %s-%s" % (self._baseline_name, self._baseline_name, acme_util.get_utc_timestamp()))

###############################################################################
class TestUpdateACMETests(unittest.TestCase):
###############################################################################

    ###########################################################################
    def setUp(self):
    ###########################################################################
        # Grab all active tests
        self._testlist_allactive = os.path.join(acme_util.get_cime_root(), "scripts", "Testing", "Testlistxml", "testlist_allactive.xml")
        shutil.copy2(self._testlist_allactive, ".")

    ###########################################################################
    def tearDown(self):
    ###########################################################################
        shutil.copy2("testlist_allactive.xml", self._testlist_allactive)

    ###########################################################################
    def test_update_acme_tests(self):
    ###########################################################################
        # Add some testable stuff to acme tests
        update_acme_tests._TEST_SUITES["acme_tiny"] = \
            (None, (("ERS.f19_g16_rx1.A", "jgftestmodtest/test_mod"),
                    ("NCK.f19_g16_rx1.A", "jgftestmodtest/test_mod"))
             )

        try:
            update_acme_tests.update_acme_tests(self._testlist_allactive, update_acme_tests.get_test_suites())
        except:
            traceback.print_tb(sys.exc_info()[2])
            self.assertTrue(False, str(sys.exc_info()[1]))

        stat = run_cmd("grep 'jgftestmodtest/test_mod' %s" % self._testlist_allactive, ok_to_fail=True)[0]
        self.assertEqual(stat, 0, msg="update_acme_tests did not update XML")

    ###########################################################################
    def test_update_acme_tests_test_mods(self):
    ###########################################################################
        machine = "melvin"
        not_my_machine = "%s_jgftest" % machine

        # Add some testable stuff to acme tests
        update_acme_tests._TEST_SUITES["acme_tiny"] = \
            (None, (("ERS.f19_g16_rx1.A", "test_mod"),
                    ("ERS.f19_g16_rx1.B", "test_mod", machine),
                    ("ERS.f19_g16_rx1.C", "test_mod", (machine, not_my_machine)),
                    ("ERS.f19_g16_rx1.D", "test_mod", not_my_machine),
                    "ERS.f19_g16_rx1.E")
             )

        tests = update_acme_tests.get_test_suite("acme_tiny", compiler="gnu")
        self.assertEqual(5, len(tests))
        self.assertTrue("ERS.f19_g16_rx1.A.melvin_gnu.test_mod" in tests)
        self.assertTrue("ERS.f19_g16_rx1.B.melvin_gnu.test_mod" in tests)
        self.assertTrue("ERS.f19_g16_rx1.C.melvin_gnu.test_mod" in tests)
        self.assertTrue("ERS.f19_g16_rx1.D.melvin_gnu" in tests)
        self.assertTrue("ERS.f19_g16_rx1.E.melvin_gnu" in tests)

###########################################################################

if (__name__ == "__main__"):
    unittest.main()
