diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index 71d357cedf6..98bb170436d 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -428,9 +428,11 @@ def read_baseline_file(baseline_file): Value stored in baseline file without comments. """ with open(baseline_file) as fd: - lines = [x.strip().split(" ")[-1] for x in fd.readlines() if not x.startswith("#")] + lines = [ + x.strip().split(" ")[-1] for x in fd.readlines() if not x.startswith("#") + ] - return "\n".join(lines) + return lines[-1] def _perf_compare_throughput_baseline(case, baseline, tolerance): @@ -474,7 +476,7 @@ def _perf_compare_throughput_baseline(case, baseline, tolerance): below_tolerance = diff < tolerance info = "Throughput changed by {:.2f}%: baseline={:.3f} sypd, tolerance={:d}%, current={:.3f} sypd".format( - diff * 100, baseline, int(tolerance * 100), current + diff * 100, baseline, int(tolerance * 100), current ) if below_tolerance: comment = "TPUTCOMP: " + info @@ -531,8 +533,8 @@ def _perf_compare_memory_baseline(case, baseline, tolerance): if diff is not None: below_tolerance = diff < tolerance - info = "Memory usage highwater changed by {:.2f}%: baseline={:.3f} sypd, tolerance={:d}%, current={:.3f} sypd".format( - diff * 100, baseline, int(tolerance * 100), current + info = "Memory usage highwater changed by {:.2f}%: baseline={:.3f} MB, tolerance={:d}%, current={:.3f} MB".format( + diff * 100, baseline, int(tolerance * 100), current ) if below_tolerance: comment = "MEMCOMP: " + info diff --git a/CIME/tests/test_unit_baselines_performance.py b/CIME/tests/test_unit_baselines_performance.py index 1422f3412b8..02ed39dc688 100644 --- a/CIME/tests/test_unit_baselines_performance.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -155,35 +155,29 @@ def test_get_cpl_mem_usage(self, isfile): def test_read_baseline_file_multi_line(self): with mock.patch( "builtins.open", - mock.mock_open(read_data="#comment about data\n1000.0\n2000.0\n"), + mock.mock_open( + read_data="sha:1df0 date:2023 1000.0\nsha:3b05 date:2023 2000.0" + ), ) as mock_file: baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == "1000.0\n2000.0" + assert baseline == "2000.0" def test_read_baseline_file_content(self): with mock.patch( - "builtins.open", mock.mock_open(read_data="1000.0") + "builtins.open", mock.mock_open(read_data="sha:1df0 date:2023 1000.0") ) as mock_file: baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") assert baseline == "1000.0" - def test_read_baseline_file(self): - with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: - baseline = performance.read_baseline_file("/tmp/cpl-mem.log") - - mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == "" - def test_write_baseline_file(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: performance.write_baseline_file("/tmp/cpl-tput.log", "1000") - mock_file.assert_called_with("/tmp/cpl-tput.log", "w") - mock_file.return_value.write.assert_called_with("1000") + mock_file.assert_called_with("/tmp/cpl-tput.log", "a") @mock.patch("CIME.baselines.performance.get_cpl_throughput") @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") @@ -368,7 +362,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( assert below_tolerance assert ( comment - == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" + == "TPUTCOMP: Throughput changed by -0.80%: baseline=500.000 sypd, tolerance=10%, current=504.000 sypd" ) @mock.patch("CIME.baselines.performance._perf_get_throughput") @@ -399,7 +393,8 @@ def test_perf_compare_throughput_baseline_above_threshold( assert not below_tolerance assert ( - comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" + comment + == "Error: TPUTCOMP: Throughput changed by 49.60%: baseline=1000.000 sypd, tolerance=5%, current=504.000 sypd" ) @mock.patch("CIME.baselines.performance._perf_get_throughput") @@ -431,7 +426,7 @@ def test_perf_compare_throughput_baseline( assert below_tolerance assert ( comment - == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" + == "TPUTCOMP: Throughput changed by -0.80%: baseline=500.000 sypd, tolerance=5%, current=504.000 sypd" ) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @@ -466,7 +461,7 @@ def test_perf_compare_memory_baseline_no_baseline( assert below_tolerance assert ( comment - == "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline" + == "MEMCOMP: Memory usage highwater changed by 0.00%: baseline=0.000 MB, tolerance=5%, current=1003.000 MB" ) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @@ -557,7 +552,7 @@ def test_perf_compare_memory_baseline_no_tolerance( assert below_tolerance assert ( comment - == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" + == "MEMCOMP: Memory usage highwater changed by 0.30%: baseline=1000.000 MB, tolerance=10%, current=1003.000 MB" ) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @@ -592,7 +587,7 @@ def test_perf_compare_memory_baseline_above_threshold( assert not below_tolerance assert ( comment - == "Error: Memory usage increase >5% from baseline's 1000.000000 to 2003.000000" + == "Error: MEMCOMP: Memory usage highwater changed by 100.30%: baseline=1000.000 MB, tolerance=5%, current=2003.000 MB" ) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @@ -627,7 +622,7 @@ def test_perf_compare_memory_baseline( assert below_tolerance assert ( comment - == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" + == "MEMCOMP: Memory usage highwater changed by 0.30%: baseline=1000.000 MB, tolerance=5%, current=1003.000 MB" ) def test_get_latest_cpl_logs_found_multiple(self): diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 609460fe9c0..22825fd5559 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -509,16 +509,14 @@ def test_generate_baseline(self): with open(baseline_dir / "cpl-tput.log") as fd: lines = fd.readlines() - assert len(lines) == 2 - assert re.match("# sha:.* date:.*", lines[0]) - assert lines[1] == "719.635" + assert len(lines) == 1 + assert re.match("sha:.* date:.* (\d+\.\d+)", lines[0]) with open(baseline_dir / "cpl-mem.log") as fd: lines = fd.readlines() - assert len(lines) == 2 - assert re.match("# sha:.* date:.*", lines[0]) - assert lines[1] == "1673.89" + assert len(lines) == 1 + assert re.match("sha:.* date:.* (\d+\.\d+)", lines[0]) def test_kwargs(self): case = mock.MagicMock()