diff --git a/scripts/geos_ats_package/geos_ats/command_line_parsers.py b/scripts/geos_ats_package/geos_ats/command_line_parsers.py index 95332d455..893f65486 100644 --- a/scripts/geos_ats_package/geos_ats/command_line_parsers.py +++ b/scripts/geos_ats_package/geos_ats/command_line_parsers.py @@ -25,6 +25,7 @@ "stopcheck": "check the stop time and stop cycle", "curvecheck": "check the ultra curves", "restartcheck": "check the restart file", + "performancecheck": "check nonlinear and linear solvers performance" } verbose_options = { diff --git a/scripts/geos_ats_package/geos_ats/helpers/solver_statistics_check.py b/scripts/geos_ats_package/geos_ats/helpers/solver_statistics_check.py new file mode 100644 index 000000000..aa5f3178b --- /dev/null +++ b/scripts/geos_ats_package/geos_ats/helpers/solver_statistics_check.py @@ -0,0 +1,216 @@ + +import os +import importlib.util +import sys +import re +import argparse +import numpy as np +from scipy.interpolate import interp1d +import matplotlib.pyplot as plt +import hdf5_wrapper +import h5py + +def parse_log_file( fname ): + """ + Parses the log file and creates an hdf5 with number of linear and nonlinear iterations per time-step + + Args: fname (str): name of the log file to parse + + Returns: output_fileName (str): + errors: + """ + # Define regular expressions + cycle_pattern = r"\d+\s*:\s*Time: [\d.e+-]+ s, dt: [\d.e+-]+ s, Cycle: (\d+)" + config_and_nnlinear_iter_pattern = r"\d+\s*:\s*Attempt:\s*(\d+),\s*ConfigurationIter:\s*(\d+),\s*NewtonIter:\s*(\d+)" + linear_iter_pattern = r"\d+\s*:\s*Last LinSolve\(iter,res\) = \(\s*(\d+),\s*([\d.e+-]+)\s*\) ;" + + # Initialize variables to store the extracted data + data = {} + + with open(fname, 'r') as file: + for line in file: + # Match Cycle number + cycle_match = re.match(cycle_pattern, line) + if cycle_match: + cycle_number = cycle_match.group(1) + data[cycle_number] = { + 'Attempts': {} + } + + # Match ConfigurationIter data + config_iter_match = re.match(config_and_nnlinear_iter_pattern, line) + if config_iter_match and cycle_number: + attempt, config_iter, newton_iter = config_iter_match.groups() + if int(newton_iter) > 0: + attempt_data = data[cycle_number]['Attempts'].get(attempt, {}) + config_data = attempt_data.get('ConfigurationIters', []) + config_data.append({ + 'ConfigurationIter': config_iter, + 'NewtonIters': {} + }) + attempt_data['ConfigurationIters'] = config_data + data[cycle_number]['Attempts'][attempt] = attempt_data + + # Match Iteration data + iteration_match = re.match(linear_iter_pattern, line) + if iteration_match and cycle_number and attempt and config_iter: + num_iterations = int(iteration_match.group(1)) + attempt_data = data[cycle_number]['Attempts'][attempt] + config_data = attempt_data['ConfigurationIters'] + config_iter_data = config_data[-1] + config_iter_data['NewtonIters'][newton_iter] = num_iterations + + # Create an HDF5 file for storing the data + output_fileName = os.path.join(os.path.dirname(fname), 'extracted_solverStat_data.h5') + with h5py.File(output_fileName, 'w') as hdf5_file: + for cycle, cycle_data in data.items(): + cycle_group = hdf5_file.create_group(f'Cycle_{cycle}') + for attempt, attempt_data in cycle_data['Attempts'].items(): + attempt_group = cycle_group.create_group(f'Attempt_{attempt}') + for config_iter_data in attempt_data['ConfigurationIters']: + config_iter_group = attempt_group.create_group(f'ConfigIter_{config_iter_data["ConfigurationIter"]}') + newton_iter_list = [] + linear_iter_list = [] + for newton_iter, num_iterations in config_iter_data['NewtonIters'].items(): + newton_iter_list.append(int(newton_iter)) + linear_iter_list.append(num_iterations) + + matrix_data = np.column_stack((newton_iter_list, linear_iter_list)) + config_iter_group.create_dataset('NewtonAndLinearIterations', data=matrix_data) + + print(f'Data has been saved to {output_fileName}') + + errors = [] + + return output_fileName, errors + +def load_data(fname): + """ + Args: + fname (str): + errors (list): + + Returns: + tuple: data, errors + """ + data = {} + if os.path.isfile(fname): + data = hdf5_wrapper.hdf5_wrapper(fname).get_copy() + else: + raise Exception(f'file {fname} not found. If baselines do not exist you may simply need to rebaseline this case.') + return data + +# def plot_performance_curves(): +# """ +# """ + +def compare_performance_curves( fname, baseline, tolerances, output ): + """ + Compute time history curves + + Args: + fname (str): Target curve file name + baseline (str): Baseline curve file name + tolerances (list): Tolerance for nonlinear and linear iterations + output (str): Path to place output figures + Returns: + tuple: warnings, errors + """ + # Setup + warnings = [] + errors = [] + + newton_iterations_tolerance, linear_iterations_tolerance = tolerances + + # Load data + target_data = load_data( fname ) + baseline_data = load_data( baseline ) + + # Check if the number of cycles is the same + target_cycles = set(target_data.keys()) + baseline_cycles = set(baseline_data.keys()) + if target_cycles != baseline_cycles: + errors.append(f'Number of cycles is different.') + + # Loop over each cycle + for cycle in target_cycles: + target_num_attempts = set(target_data[cycle].keys()) + baseline_num_attempts = set(baseline_data[cycle].keys()) + + # Check if the number of attempts is the same for this cycle + if target_num_attempts != baseline_num_attempts: + errors.append(f'Number of attempts for Cycle {cycle} is different.') + + # Loop over each attempt + for attempt in target_num_attempts: + target_config_iters = set(target_data[cycle][attempt].keys()) + baeline_config_iters = set(baseline_data[cycle][attempt].keys()) + + # Check if the number of ConfigurationIters is the same for this Attempt + if target_config_iters != baeline_config_iters: + errors.append(f'Number of ConfigurationIters for Cycle {cycle}, Attempt {attempt} is different.') + + # Loop over each ConfigurationIter + for config_iter in target_config_iters: + # Check if the NewtonAndLinearIterations are within tolerance + target_iterations = np.array(target_data[cycle][attempt][config_iter]['NewtonAndLinearIterations']) + baseline_iterations = np.array(baseline_data[cycle][attempt][config_iter]['NewtonAndLinearIterations']) + + newton_diff = np.abs(target_iterations[:, 0] - baseline_iterations[:, 0]) + linear_diff = np.abs(target_iterations[:, 1] - baseline_iterations[:, 1]) + + if (np.any(newton_diff > newton_iterations_tolerance * target_iterations[:, 0]) or + np.any(linear_diff > linear_iterations_tolerance * target_iterations[:, 1])): + errors.append(f'Differences found in NewtonAndLinearIterations for Cycle {cycle}, Attempt {attempt}, ConfigurationIter {config_iter}.') + + return warnings, errors + +def solver_statistics_check_parser(): + """ + Build the curve check parser + + Returns: + argparse.parser: The performance check parser + """ + parser = argparse.ArgumentParser() + parser.add_argument("filename", help="Path to the log file") + parser.add_argument("baseline", help="Path to the baseline file") + parser.add_argument("-t", + "--tolerance", + nargs='+', + action='append', + help=f"The tolerance for nonlinear and linear iterations", + default=[]) + parser.add_argument("-o", + "--output", + help="Output figures to this directory", + default='./solver_statistics_check_figures') + return parser + + +def main(): + """ + Entry point for the performance check script + """ + parser = solver_statistics_check_parser() + args = parser.parse_args() + fname, parsingErrors = parse_log_file( args.filename ) + + # We raise immediately if there is any issue while parsing + if len(parsingErrors): + print('\n'.join(parsingErrors)) + raise Exception(f'Performance check error while parsing log file.') + + warnings, errors = compare_performance_curves( fname, args.baseline, args.tolerance, args.output ) + + if len(warnings): + print('Performance check warnings:') + print('\n'.join(warnings)) + + if len(errors): + print('Performance check errors:') + print('\n'.join(errors)) + raise Exception(f'Performance check produced {len(errors)} errors!') + +if __name__ == '__main__': + main() diff --git a/scripts/geos_ats_package/geos_ats/test_builder.py b/scripts/geos_ats_package/geos_ats/test_builder.py index 6144e4728..6520a0533 100644 --- a/scripts/geos_ats_package/geos_ats/test_builder.py +++ b/scripts/geos_ats_package/geos_ats/test_builder.py @@ -28,6 +28,13 @@ def as_dict(self): return asdict(self) +@dataclass(frozen=True) +class SolverstatisticscheckParameters: + tolerance: tuple[float, float] + + def as_dict(self): + return asdict(self) + @dataclass(frozen=True) class TestDeck: name: str @@ -37,6 +44,7 @@ class TestDeck: check_step: int restartcheck_params: RestartcheckParameters = None curvecheck_params: CurveCheckParameters = None + performancecheck_params: SolverstatisticscheckParameters= None def collect_block_names(fname): @@ -80,8 +88,9 @@ def generate_geos_tests(decks: Iterable[TestDeck]): """ for ii, deck in enumerate(decks): - restartcheck_params = None - curvecheck_params = None + restartcheck_params=None + curvecheck_params=None + performancecheck_params=None if deck.restartcheck_params is not None: restartcheck_params = deck.restartcheck_params.as_dict() @@ -89,6 +98,9 @@ def generate_geos_tests(decks: Iterable[TestDeck]): if deck.curvecheck_params is not None: curvecheck_params = deck.curvecheck_params.as_dict() + if deck.performancecheck_params is not None: + performancecheck_params = deck.performancecheck_params.as_dict() + for partition in deck.partitions: nx, ny, nz = partition N = nx * ny * nz @@ -111,7 +123,8 @@ def generate_geos_tests(decks: Iterable[TestDeck]): y_partitions=ny, z_partitions=nz, restartcheck_params=restartcheck_params, - curvecheck_params=curvecheck_params) + curvecheck_params=curvecheck_params, + performancecheck_params=performancecheck_params) ] if deck.restart_step > 0: diff --git a/scripts/geos_ats_package/geos_ats/test_steps.py b/scripts/geos_ats_package/geos_ats/test_steps.py index 129e162ed..3bc8fc182 100644 --- a/scripts/geos_ats_package/geos_ats/test_steps.py +++ b/scripts/geos_ats_package/geos_ats/test_steps.py @@ -13,6 +13,7 @@ logger = logging.getLogger('geos_ats') +EXTRACTED_SOLVERSTATISTICSCHECK_DATAFILE = 'extracted_solverStat_data.h5' def getGeosProblemName(deck, name): """ @@ -397,12 +398,13 @@ class geos(TestStepBase): checkstepnames = ["restartcheck"] - def __init__(self, restartcheck_params=None, curvecheck_params=None, **kw): + def __init__(self, restartcheck_params=None, curvecheck_params=None, performancecheck_params=None, **kw): """ Initializes the parameters of this test step, and creates the appropriate check steps. RESTARTCHECK_PARAMS [in]: Dictionary that gets passed on to the restartcheck step. CURVECHECK_PARAMS [in]: Dictionary that gets passed on to the curvecheck step. + PERFORMANCE_PARAMS [in]: Dictionary that gets passed on to the performancecheck step. KEYWORDS [in]: Dictionary that is used to set the parameters of this step and also all check steps. """ @@ -419,6 +421,10 @@ def __init__(self, restartcheck_params=None, curvecheck_params=None, **kw): if restartcheck_params is not None: self.checksteps.append(restartcheck(restartcheck_params, **kw)) + if checkOption in ["all", "perfomrancecheck"]: + if performancecheck_params is not None: + self.checksteps.append(performancecheck(performancecheck_params, **kw)) + def label(self): return "geos" @@ -763,6 +769,106 @@ def resultPaths(self): def clean(self): self._clean(self.resultPaths()) +class performancecheck(CheckTestStepBase): + """ + Class for the performance check test step. + """ + + doc = """CheckTestStep to compare a curve file against a baseline.""" + + command = """solver_statistics_check.py filename baseline""" + + params = TestStepBase.defaultParams + CheckTestStepBase.checkParams + ( + TestStepBase.commonParams["deck"], TestStepBase.commonParams["name"], TestStepBase.commonParams["np"], + TestStepBase.commonParams["allow_rebaseline"], TestStepBase.commonParams["baseline_dir"], + TestStepBase.commonParams["output_directory"], + TestParam("tolerance", "tolerances for newton and linear iterations"), + TestParam("filename", "Name of GEOS.") ) + + def __init__(self, permormancecheck_params, **kw): + """ + Set parameters with PERFORMANCECHECK_PARAMS and then with KEYWORDS. + """ + CheckTestStepBase.__init__(self) + self.p.warnings_are_errors = True + + if permormancecheck_params is not None: + c = permormancecheck_params.copy() + + # Check whether tolerance was specified as a single float, list + # and then convert into a comma-delimited string + tol = c.get('tolerance', 0.0) + if isinstance(tol, (float, int)): + tol = [tol] * 2 + c['tolerance'] = ','.join([str(x) for x in tol]) + + self.setParams(c, self.params) + self.setParams(kw, self.params) + + def label(self): + return "performancecheck" + + def makeArgs(self): + cur_dir = os.path.dirname(os.path.realpath(__file__)) + script_location = os.path.join(cur_dir, "helpers", "solver_statistics_check.py") + args = [script_location] + + if self.p.tolerance is not None: + for t in self.p.tolerance.split(','): + args += ["-t", t] + + args += ['-o', self.figure_root] + args += [self.target_file, self.baseline_file] + return list(map(str, args)) + + def executable(self): + if self.getTestMode(): + return "python" + else: + return sys.executable + + def rebaseline(self): + if not self.p.allow_rebaseline: + Log("Rebaseline not allowed for performancecheck of %s." % self.p.name) + return + + baseline_dir = os.path.split(self.baseline_file)[0] + os.makedirs(baseline_dir, exist_ok=True) + file = os.path.join(self.p.output_directory, EXTRACTED_SOLVERSTATISTICSCHECK_DATAFILE) + shutil.copyfile(file, self.baseline_file) + + def update(self, dictionary): + self.setParams(dictionary, self.params) + self.handleCommonParams() + + self.requireParam("deck") + self.requireParam("baseline_dir") + self.requireParam("output_directory") + + self.baseline_file = os.path.join(self.p.baseline_dir, EXTRACTED_SOLVERSTATISTICSCHECK_DATAFILE) + output_directory = self.p.output_directory + + # Search for all files with a .data extension within the output_directory + data_files = glob.glob(os.path.join(output_directory, "*.data")) + if len(data_files) == 1: + self.target_file = data_files[0] + elif len(data_files) > 1: + raise Exception(f'More than 1 .data file was found') + else: + raise Exception(f'No .data file was found in {output_directory}') + + self.figure_root = os.path.join(self.p.output_directory, 'solver_statistics_check') + + if self.p.allow_rebaseline is None: + self.p.allow_rebaseline = True + + def resultPaths(self): + figure_pattern = os.path.join(self.figure_root, '*.png') + figure_list = sorted(glob.glob(figure_pattern)) + return [self.target_file] + figure_list + + def clean(self): + self._clean(self.resultPaths()) def infoTestStepParams(params, maxwidth=None): if maxwidth is None: diff --git a/tests/allTests/embeddedFractures/baselines/Sneddon_embeddedFrac_benchmark_01/extracted_solverStat_data.h5 b/tests/allTests/embeddedFractures/baselines/Sneddon_embeddedFrac_benchmark_01/extracted_solverStat_data.h5 new file mode 100644 index 000000000..8c2e85e1e Binary files /dev/null and b/tests/allTests/embeddedFractures/baselines/Sneddon_embeddedFrac_benchmark_01/extracted_solverStat_data.h5 differ diff --git a/tests/allTests/embeddedFractures/embeddedFractures_mechanics.ats b/tests/allTests/embeddedFractures/embeddedFractures_mechanics.ats index f746d2464..dc6923ec9 100644 --- a/tests/allTests/embeddedFractures/embeddedFractures_mechanics.ats +++ b/tests/allTests/embeddedFractures/embeddedFractures_mechanics.ats @@ -1,6 +1,6 @@ import os import geos_ats -from geos_ats.test_builder import TestDeck, CurveCheckParameters, RestartcheckParameters, generate_geos_tests +from geos_ats.test_builder import TestDeck, CurveCheckParameters, RestartcheckParameters, SolverstatisticscheckParameters, generate_geos_tests restartcheck_params = {'atol': 1e-08, 'rtol': 4e-07} @@ -12,37 +12,36 @@ curvecheck_params["script_instructions"] = [[ ]] curvecheck_params["curves"] = "displacementJump" -decks = [ - TestDeck(name='Sneddon_embeddedFrac_smoke', - description="Smoke test for Sneddon's problem with horizontal fracture", - partitions=((1, 1, 1), (2, 2, 1)), - restart_step=0, - check_step=1, - restartcheck_params=RestartcheckParameters(**restartcheck_params)), - TestDeck(name='Sneddon_embeddedFrac_benchmark', - description="Sneddon's problem with horizontal fracture (uses MGR)", - partitions=((1, 1, 1), ), - restart_step=0, - check_step=1, - curvecheck_params=CurveCheckParameters(**curvecheck_params)), - TestDeck(name='Sneddon_embeddedFrac_staticCondensation_smoke', - description="Sneddon with horizontal fracture usic static condensation", - partitions=((1, 1, 1), (2, 2, 1)), - restart_step=0, - check_step=1, - restartcheck_params=RestartcheckParameters(**restartcheck_params)), - TestDeck(name='Sneddon_embeddedFrac_staticCondensation_benchmark', - description="Sneddon with horizontal fracture usic static condensation", - partitions=((1, 1, 1), ), - restart_step=0, - check_step=1, - curvecheck_params=CurveCheckParameters(**curvecheck_params)), - TestDeck(name='SneddonRotated_smoke', - description='Sneddon with inclined fracture', - partitions=((1, 1, 1), (2, 2, 1)), - restart_step=0, - check_step=1, - restartcheck_params=RestartcheckParameters(**restartcheck_params)) -] +decks = [TestDeck(name='Sneddon_embeddedFrac_smoke', + description="Smoke test for Sneddon's problem with horizontal fracture", + partitions=((1, 1, 1), (2, 2, 1)), + restart_step=0, + check_step=1, + restartcheck_params=RestartcheckParameters(**restartcheck_params)), + TestDeck(name='Sneddon_embeddedFrac_benchmark', + description="Sneddon's problem with horizontal fracture (uses MGR)", + partitions=((1, 1, 1),), + restart_step=0, + check_step=1, + curvecheck_params=CurveCheckParameters(**curvecheck_params), + performancecheck_params=SolverstatisticscheckParameters(tolerance=[0.1, 0.2])), + TestDeck(name='Sneddon_embeddedFrac_staticCondensation_smoke', + description="Sneddon with horizontal fracture using static condensation", + partitions=((1, 1, 1), (2, 2, 1)), + restart_step=0, + check_step=1, + restartcheck_params=RestartcheckParameters(**restartcheck_params)), + TestDeck(name='Sneddon_embeddedFrac_staticCondensation_benchmark', + description="Sneddon with horizontal fracture using static condensation", + partitions=((1, 1, 1),), + restart_step=0, + check_step=1, + curvecheck_params=CurveCheckParameters(**curvecheck_params)), + TestDeck(name='SneddonRotated_smoke', + description='Sneddon with inclined fracture', + partitions=((1, 1, 1), (2, 2, 1)), + restart_step=0, + check_step=1, + restartcheck_params=RestartcheckParameters(**restartcheck_params))] generate_geos_tests(decks) diff --git a/tests/allTests/poroElasticCoupling/PoroElastic_Mandel_smoke_fim_mgr.xml b/tests/allTests/poroElasticCoupling/PoroElastic_Mandel_smoke_fim_mgr.xml new file mode 120000 index 000000000..cbac92253 --- /dev/null +++ b/tests/allTests/poroElasticCoupling/PoroElastic_Mandel_smoke_fim_mgr.xml @@ -0,0 +1 @@ +../../../../inputFiles/poromechanics/PoroElastic_Mandel_smoke_fim.xml \ No newline at end of file diff --git a/tests/allTests/poroElasticCoupling/PoroElastic_staircase_co2_3d_mgr.xml b/tests/allTests/poroElasticCoupling/PoroElastic_staircase_co2_3d_mgr.xml new file mode 120000 index 000000000..47c124b4c --- /dev/null +++ b/tests/allTests/poroElasticCoupling/PoroElastic_staircase_co2_3d_mgr.xml @@ -0,0 +1 @@ +../../../../inputFiles/poromechanics/PoroElastic_staircase_co2_3d.xml \ No newline at end of file diff --git a/tests/allTests/poroElasticCoupling/poroElasticCoupling.ats b/tests/allTests/poroElasticCoupling/poroElasticCoupling.ats index 5786270eb..ce8e7ea51 100644 --- a/tests/allTests/poroElasticCoupling/poroElasticCoupling.ats +++ b/tests/allTests/poroElasticCoupling/poroElasticCoupling.ats @@ -1,5 +1,6 @@ -import geos_ats -from geos_ats.test_builder import TestDeck, RestartcheckParameters, generate_geos_tests + +import geos_ats +from geos_ats.test_builder import TestDeck, RestartcheckParameters, SolverstatisticscheckParameters, generate_geos_tests class Description(object): @@ -196,27 +197,37 @@ def _build_PoroElasticGravity_cases(): def test_poro_elastic_coupling_cases(): - deck_instances = [ - _build_Terzaghi_cases(), - _build_Mandel_fim_cases(), - _build_Mandel_sequential_cases(), - _build_Mandel_prism6_cases(), - _build_Deadoil_fim_cases(), - _build_Deadoil_sequential_cases(), - _build_PoroElasticWell_cases(), - _build_PoroDruckerPragerWell_cases(), - _build_PoroDelftEggWell_cases(), - _build_PoroModifiedCamClayWell_cases(), - _build_PoroImpermeableFault_cases(), - _build_PoroPermeableFault_cases(), - _build_PoroStaircaseSinglePhasePeacemanWell_cases(), - _build_PoroStaircaseCO2PeacemanWell_cases(), - _build_PoroElasticPEBICO2FIM_cases(), - _build_PoroElasticPEBICO2Sequential_cases(), - _build_PoroElasticGravity_cases() - ] - - generate_geos_tests(deck_instances) - + deck_instances = [_build_Terzaghi_cases(), + _build_Mandel_fim_cases(), + _build_Mandel_sequential_cases(), + _build_Mandel_prism6_cases(), + _build_Deadoil_fim_cases(), + _build_Deadoil_sequential_cases(), + _build_PoroElasticWell_cases(), + _build_PoroDruckerPragerWell_cases(), + _build_PoroDelftEggWell_cases(), + _build_PoroModifiedCamClayWell_cases(), + _build_PoroImpermeableFault_cases(), + _build_PoroPermeableFault_cases(), + _build_PoroStaircaseSinglePhasePeacemanWell_cases(), + _build_PoroStaircaseCO2PeacemanWell_cases(), + _build_PoroElasticPEBICO2FIM_cases(), + _build_PoroElasticPEBICO2Sequential_cases(), + _build_PoroElasticGravity_cases(), + TestDeck( name="PoroElastic_Mandel_smoke_fim_mgr", + description="Mandel fim case using mgr strategy", + partitions=[ (1, 1, 1), (3, 1, 2) ], + restart_step=0, + check_step=0, + performancecheck_params=SolverstatisticscheckParameters(tolerance=[0.1, 0.2])), + TestDeck( name="PoroElastic_staircase_co2_3d_mgr", + description="Staircase CO2 poroelastic problem with Peaceman wells using mgr strategy", + partitions=[ (1, 1, 1), (2, 2, 1) ], + restart_step=0, + check_step=0, + performancecheck_params=SolverstatisticscheckParameters(tolerance=[0.1, 0.2])) + ] + + generate_geos_tests( deck_instances ) test_poro_elastic_coupling_cases()