diff --git a/setup.cfg b/setup.cfg index 846f8d6aa..36b2dad84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 6.44.2 +current_version = 6.46.1 commit = True tag = True diff --git a/setup.py b/setup.py index 74323b429..ac8efc026 100644 --- a/setup.py +++ b/setup.py @@ -117,6 +117,6 @@ test_suite="tests", tests_require=test_requirements, url="https://github.com/esm-tools/esm_tools", - version="6.44.2", + version="6.46.1", zip_safe=False, ) diff --git a/src/esm_archiving/__init__.py b/src/esm_archiving/__init__.py index 38ae4c54b..757f804a7 100644 --- a/src/esm_archiving/__init__.py +++ b/src/esm_archiving/__init__.py @@ -4,7 +4,7 @@ __author__ = """Paul Gierz""" __email__ = "pgierz@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .esm_archiving import (archive_mistral, check_tar_lists, delete_original_data, determine_datestamp_location, diff --git a/src/esm_calendar/__init__.py b/src/esm_calendar/__init__.py index 0ae75400c..ca54fc803 100644 --- a/src/esm_calendar/__init__.py +++ b/src/esm_calendar/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .esm_calendar import * diff --git a/src/esm_cleanup/__init__.py b/src/esm_cleanup/__init__.py index f618c7043..a8255847b 100644 --- a/src/esm_cleanup/__init__.py +++ b/src/esm_cleanup/__init__.py @@ -2,4 +2,4 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" diff --git a/src/esm_database/__init__.py b/src/esm_database/__init__.py index f1424ce00..f89566d90 100644 --- a/src/esm_database/__init__.py +++ b/src/esm_database/__init__.py @@ -2,4 +2,4 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" diff --git a/src/esm_environment/__init__.py b/src/esm_environment/__init__.py index 9a99650cf..7cf1d5cb3 100644 --- a/src/esm_environment/__init__.py +++ b/src/esm_environment/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .esm_environment import * diff --git a/src/esm_master/__init__.py b/src/esm_master/__init__.py index 71f0e6584..9eae47c6a 100644 --- a/src/esm_master/__init__.py +++ b/src/esm_master/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from . import database diff --git a/src/esm_master/task.py b/src/esm_master/task.py index 86f5b9ddc..85eea1a3a 100644 --- a/src/esm_master/task.py +++ b/src/esm_master/task.py @@ -1,18 +1,16 @@ import os -import sys -import subprocess -import shlex # contains shlex.split that respects quoted strings - # deniz: it is better to use more pathlib in the future so that dir/path # operations will be more portable (supported since Python 3.4, 2014) import pathlib - -from .software_package import software_package -from esm_parser import user_error +import shlex # contains shlex.split that respects quoted strings +import subprocess +import sys import esm_environment import esm_plugin_manager +from esm_tools import user_error +from .software_package import software_package ###################################################################################### ################################# class "task" ####################################### diff --git a/src/esm_motd/__init__.py b/src/esm_motd/__init__.py index 01c56cb87..32c2a93c2 100644 --- a/src/esm_motd/__init__.py +++ b/src/esm_motd/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .esm_motd import * diff --git a/src/esm_motd/esm_motd.py b/src/esm_motd/esm_motd.py index e6798900e..7d9170746 100644 --- a/src/esm_motd/esm_motd.py +++ b/src/esm_motd/esm_motd.py @@ -3,11 +3,12 @@ import urllib.request from time import sleep -import esm_parser -import esm_utilities import yaml +import esm_parser import esm_tools +import esm_utilities +from esm_tools import user_error class MessageOfTheDayError(Exception): @@ -83,7 +84,7 @@ def action_handler(self, action, time, package, version): if action == "sleep": sleep(time) elif action == "error": - esm_parser.user_error( + user_error( "Version", ( f"Version {version} of '{package}' package has been tagged as " diff --git a/src/esm_parser/__init__.py b/src/esm_parser/__init__.py index ebf370a86..254eceac8 100644 --- a/src/esm_parser/__init__.py +++ b/src/esm_parser/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .dict_to_yaml import * diff --git a/src/esm_parser/esm_parser.py b/src/esm_parser/esm_parser.py index 5c94c813d..c16bb4cba 100644 --- a/src/esm_parser/esm_parser.py +++ b/src/esm_parser/esm_parser.py @@ -53,19 +53,15 @@ Specific documentation for classes and functions are given below: """ # Python 2 and 3 version agnostic compatiability: -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import - -import pdb +from __future__ import (absolute_import, division, print_function, + unicode_literals) # Python Standard Library imports import collections import copy import logging import os -import re +import pdb import shutil import socket import subprocess @@ -79,21 +75,20 @@ # Always import externals before any non standard library imports +import coloredlogs # Third-Party Imports import numpy -import coloredlogs -import colorama import yaml -# functions reading in dict from file -from .yaml_to_dict import * -from .provenance import * - +# Loader for package yamls +import esm_tools # Date class from esm_calendar import Date +from esm_tools import user_error, user_note -# Loader for package yamls -import esm_tools +from .provenance import * +# functions reading in dict from file +from .yaml_to_dict import * # Logger and related constants logger = logging.getLogger("root") @@ -2325,12 +2320,14 @@ def do_math_in_entry(tree, rhs, config): if "${" in str(entry): return entry entry = " " + str(entry) + " " + date_operation = False while "$((" in entry: math, after_math = entry.split("))", 1) math, before_math = math[::-1].split("(($", 1) math = math[::-1] before_math = before_math[::-1] if DATE_MARKER in math: + date_operation = True all_dates = [] steps = math.split(" ") steps = [step for step in steps if step] @@ -2408,12 +2405,18 @@ def do_math_in_entry(tree, rhs, config): math = math + "all_dates[" + str(index) + "]" index += 1 result = eval(math) - if isinstance(result, list): + if isinstance(result, list) and date_operation: result = result[ -1 ] # should be extended in the future - here: if list (= if diff between dates) than result in seconds + elif isinstance(result, list): + entry = ListWithProvenance(result, None) + entry.set_provenance(rhs.provenance) + return entry + result = str(result) entry = before_math + result + after_math + # TODO MA: this is a provisional dirty fix for release. Get rid of this once a more # general solution is worked out # ORIGINAL LINE: return convert(entry.strip()) @@ -2789,51 +2792,6 @@ def find_key(d_search, k_search, exc_strings="", level="", paths2finds=[], sep=" return paths2finds -def user_note(note_heading, note_text, color=colorama.Fore.YELLOW, dsymbols=["``"]): - """ - Notify the user about something. In the future this should also write in the log. - - Parameters - ---------- - note_heading : str - Note type used for the heading. - text : str - Text clarifying the note. - """ - reset_s = colorama.Style.RESET_ALL - - if isinstance(note_text, list): - new_note_text = "" - for item in note_text: - new_note_text = f"{new_note_text}- {item}\n" - note_text = new_note_text - - for dsymbol in dsymbols: - note_text = re.sub( - f"{dsymbol}([^{dsymbol}]*){dsymbol}", f"{color}\\1{reset_s}", str(note_text) - ) - print(f"\n{color}{note_heading}\n{'-' * len(note_heading)}{reset_s}") - print(f"{note_text}\n") - - -def user_error(error_type, error_text, exit_code=1, dsymbols=["``"]): - """ - User-friendly error using ``sys.exit()`` instead of an ``Exception``. - - Parameters - ---------- - error_type : str - Error type used for the error heading. - text : str - Text clarifying the error. - exit_code : int - The exit code to send back to the parent process (default to 1) - """ - error_title = "ERROR: " + error_type - user_note(error_title, error_text, color=colorama.Fore.RED, dsymbols=dsymbols) - sys.exit(exit_code) - - class GeneralConfig(dict): # pragma: no cover """All configs do this!""" diff --git a/src/esm_parser/provenance.py b/src/esm_parser/provenance.py index c083f42f2..36990f5dd 100644 --- a/src/esm_parser/provenance.py +++ b/src/esm_parser/provenance.py @@ -523,6 +523,23 @@ def get_provenance(self, index=-1): return provenance_dict + def extract_first_nested_values_provenance(self): + """ + Recursively loops through the dictionary keys and returns the first provenance + found in the nested values. + + Returns + ------- + first_provenance : esm_parser.provenance.Provenance + The first provenance found in the nested values + """ + first_provenance = None + for key, val in self.items(): + if isinstance(val, PROVENANCE_MAPPINGS): + return val.extract_first_nested_values_provenance() + elif hasattr(val, "provenance"): + return val.provenance[-1] + def __setitem__(self, key, val): """ Any time an item in a DictWithProvenance is set, extend the old provenance of @@ -773,6 +790,23 @@ def get_provenance(self, index=-1): return provenance_list + def extract_first_nested_values_provenance(self): + """ + Recursively loops through the list elements and returns the first provenance + found in the nested values. + + Returns + ------- + first_provenance : esm_parser.provenance.Provenance + The first provenance found in the nested values + """ + first_provenance = None + for elem in self: + if isinstance(elem, PROVENANCE_MAPPINGS): + return elem.extract_first_nested_values_provenance() + elif hasattr(elem, "provenance"): + return elem.provenance[-1] + def __setitem__(self, indx, val): """ Any time an item in a ListWithProvenance is set, extend the old provenance of diff --git a/src/esm_parser/yaml_to_dict.py b/src/esm_parser/yaml_to_dict.py index d97336f3a..44e3f335a 100644 --- a/src/esm_parser/yaml_to_dict.py +++ b/src/esm_parser/yaml_to_dict.py @@ -11,6 +11,7 @@ import esm_parser import esm_tools +from esm_tools import user_error from .provenance import * @@ -135,7 +136,7 @@ def constructor_env_variables(loader, node): for env_var in envvar_matches: # first check if the variable exists in the shell environment if not os.getenv(env_var): - esm_parser.user_error( + user_error( f"{env_var} is not defined", f"{env_var} is not an environment variable. Exiting", ) @@ -227,10 +228,10 @@ def yaml_file_to_dict(filepath): except yaml.scanner.ScannerError as yaml_error: logger.debug(f"Your file {filepath + extension} has syntax issues!") error = EsmConfigFileError(filepath + extension, yaml_error) - esm_parser.user_error("Yaml syntax", f"{error}") + user_error("Yaml syntax", f"{error}") except Exception as error: logger.exception(error) - esm_parser.user_error( + user_error( "Yaml syntax", f"Syntax error in ``{filepath}``\n\n``Details:\n``{error}", ) @@ -331,7 +332,7 @@ def check_changes_duplicates(yamldict_all, fpath): # If more than one ``_changes`` without ``choose_`` return error if len(changes_no_choose) > 1: changes_no_choose = [x.replace(",", ".") for x in changes_no_choose] - esm_parser.user_error( + user_error( "YAML syntax", "More than one ``_changes`` out of a ``choose_`` in " + fpath @@ -347,7 +348,7 @@ def check_changes_duplicates(yamldict_all, fpath): changes_group.remove(changes_no_choose[0]) if len(changes_group) > 0: changes_group = [x.replace(",", ".") for x in changes_group] - esm_parser.user_error( + user_error( "YAML syntax", "The general ``" + changes_no_choose[0] @@ -394,7 +395,7 @@ def check_changes_duplicates(yamldict_all, fpath): "," )[0] if case == sub_case: - esm_parser.user_error( + user_error( "YAML syntax", "The following ``_changes`` can be accessed " + "simultaneously in " @@ -413,7 +414,7 @@ def check_changes_duplicates(yamldict_all, fpath): else: # If these ``choose_`` are different they can be accessed # simultaneously, then it returns an error - esm_parser.user_error( + user_error( "YAML syntax", "The following ``_changes`` can be accessed " + "simultaneously in " @@ -452,7 +453,7 @@ def check_changes_duplicates(yamldict_all, fpath): add_group.remove(add_no_choose[0]) if len(add_group) > 0: add_group = [x.replace(",", ".") for x in add_group] - esm_parser.user_error( + user_error( "YAML syntax", "The general ``" + add_no_choose[0] @@ -469,7 +470,7 @@ def check_changes_duplicates(yamldict_all, fpath): def check_for_empty_components(yaml_load, fpath): for key, value in yaml_load.items(): if not value: - esm_parser.user_error( + user_error( "YAML syntax", f"The component ``{key}`` is empty in the file ``{fpath}``. ESM-Tools does" + " not support empty components, either add some variables to the " @@ -540,7 +541,7 @@ def map_constructor(loader, node, deep=False): value = loader.construct_object(value_node, deep=deep) if key in mapping: - esm_parser.user_error( + user_error( "Duplicated variables", "Key ``{0}`` is duplicated {1}\n\n".format( key, str(key_node.start_mark).replace(" ", "").split(",")[0] @@ -623,7 +624,7 @@ def construct_scalar(self, node): self.env_variables.append((env_var, rval)) return rval else: - esm_parser.user_error( + user_error( f"{env_var} is not defined", f"{env_var} is not an environment variable. Exiting", ) @@ -868,4 +869,4 @@ def dump(self, data, stream=None, **kw): if not self.add_comments: return super().dump(data, stream, **kw) self._add_origin_comments(data) - return super().dump(data, stream, **kw) \ No newline at end of file + return super().dump(data, stream, **kw) diff --git a/src/esm_plugin_manager/__init__.py b/src/esm_plugin_manager/__init__.py index f03aa4dd1..f935be8f3 100644 --- a/src/esm_plugin_manager/__init__.py +++ b/src/esm_plugin_manager/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi, Paul Gierz, Sebastian Wahl""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .esm_plugin_manager import * diff --git a/src/esm_profile/__init__.py b/src/esm_profile/__init__.py index 6790f2a2f..d986d5e0f 100644 --- a/src/esm_profile/__init__.py +++ b/src/esm_profile/__init__.py @@ -2,6 +2,6 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .esm_profile import * diff --git a/src/esm_runscripts/__init__.py b/src/esm_runscripts/__init__.py index 0ce5ebf4e..dff8c2d2e 100644 --- a/src/esm_runscripts/__init__.py +++ b/src/esm_runscripts/__init__.py @@ -2,7 +2,7 @@ __author__ = """Dirk Barbi""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .batch_system import * from .chunky_parts import * diff --git a/src/esm_runscripts/batch_system.py b/src/esm_runscripts/batch_system.py index f0cd44a58..94bb03a28 100644 --- a/src/esm_runscripts/batch_system.py +++ b/src/esm_runscripts/batch_system.py @@ -4,10 +4,12 @@ import sys import textwrap -import esm_environment -from esm_parser import find_variable, user_error, user_note from loguru import logger +import esm_environment +from esm_parser import find_variable +from esm_tools import user_error, user_note + from . import dataprocess, helpers, prepare from .pbs import Pbs from .slurm import Slurm @@ -814,7 +816,7 @@ def calc_launcher_flags(config, model, cluster): cpus_per_proc = config[model].get("cpus_per_proc", omp_num_threads) # Check for CPUs and OpenMP threads if omp_num_threads > cpus_per_proc: - esm_parser.user_error( + user_error( "OpenMP configuration", ( "The number of OpenMP threads cannot be larger than the number" @@ -826,7 +828,7 @@ def calc_launcher_flags(config, model, cluster): elif "nproca" in config[model] and "nprocb" in config[model]: # ``nproca``/``nprocb`` not compatible with ``omp_num_threads`` if omp_num_threads > 1: - esm_parser.user_note( + user_note( "nproc", "``nproca``/``nprocb`` not compatible with ``omp_num_threads``", ) diff --git a/src/esm_runscripts/cli.py b/src/esm_runscripts/cli.py index 69ea10154..afab6e1fb 100644 --- a/src/esm_runscripts/cli.py +++ b/src/esm_runscripts/cli.py @@ -15,7 +15,7 @@ from loguru import logger from esm_motd import check_all_esm_packages -from esm_parser import user_error +from esm_tools import user_error from .helpers import SmartSink from .sim_objects import * diff --git a/src/esm_runscripts/compute.py b/src/esm_runscripts/compute.py index 4a0211659..ef93c1120 100644 --- a/src/esm_runscripts/compute.py +++ b/src/esm_runscripts/compute.py @@ -9,12 +9,13 @@ import questionary import yaml from colorama import Back, Fore, Style, init +from loguru import logger import esm_calendar import esm_parser import esm_runscripts import esm_tools -from loguru import logger +from esm_tools import user_error, user_note from .batch_system import batch_system from .filelists import copy_files, log_used_files @@ -444,7 +445,7 @@ def update_runscript(fromdir, scriptsdir, tfile, gconfig, file_type): # If the --update flag is used, notify that the target script will # be updated and do update it if gconfig["update"]: - esm_parser.user_note( + user_note( f"Original {file_type} different from target", differences + "\n" + f"{scriptsdir + '/' + tfile} will be updated!", ) @@ -454,7 +455,7 @@ def update_runscript(fromdir, scriptsdir, tfile, gconfig, file_type): # If the --update flag is not called, exit with an error showing the # user how to proceed else: - esm_parser.user_note( + user_note( f"Original {file_type} different from target", differences + "\n" @@ -546,7 +547,7 @@ def copy_tools_to_thisrun(config): # exit right away to prevent further recursion. There might still be # running instances of esmr_runscripts and something like # `killall esm_runscripts` might be required - esm_parser.user_error(error_type, error_text) + user_error(error_type, error_text) # If ``fromdir`` and ``scriptsdir`` are the same, this is already a computing # simulation which means we want to use the script in the experiment folder, diff --git a/src/esm_runscripts/config_initialization.py b/src/esm_runscripts/config_initialization.py index e12cfed5e..354a681ad 100644 --- a/src/esm_runscripts/config_initialization.py +++ b/src/esm_runscripts/config_initialization.py @@ -4,6 +4,7 @@ import esm_parser import esm_tools +from esm_tools import user_error from . import chunky_parts @@ -161,7 +162,7 @@ def get_user_config_from_command_line(command_line_config): except SystemExit as sysexit: sys.exit(sysexit) except: - esm_parser.user_error( + user_error( "Syntax error", f"An error occurred while reading the config file " f"``{command_line_config['runscript_abspath']}`` from the command line.", @@ -284,7 +285,7 @@ def check_account(config): # Check if the 'account' variable is needed and missing if config["computer"].get("accounting", False): if "account" not in config["general"]: - esm_parser.user_error( + user_error( "Missing account info", f"You cannot run simulations in '{config['computer']['name']}' " "without providing an 'account' variable in the 'general' section, whose " diff --git a/src/esm_runscripts/coupler.py b/src/esm_runscripts/coupler.py index 68d729255..0986783bd 100644 --- a/src/esm_runscripts/coupler.py +++ b/src/esm_runscripts/coupler.py @@ -1,8 +1,9 @@ import sys -import esm_parser from loguru import logger +from esm_tools import user_error + known_couplers = ["oasis3mct", "yac"] @@ -159,14 +160,26 @@ def add_files(self, full_config): sys.exit(0) direction_info = None - if "coupling_directions" in full_config[self.name]: - if ( - right_grid + "->" + left_grid - in full_config[self.name]["coupling_directions"] - ): - direction_info = full_config[self.name][ - "coupling_directions" - ][right_grid + "->" + left_grid] + coupling_directions = full_config[self.name].get( + "coupling_directions" + ) + if coupling_directions: + direction = f"{right_grid}->{left_grid}" + direction_info = coupling_directions.get(direction) + if not direction_info: + user_error( + "Missing coupling direction", + f"The ``{direction}`` does not exist in " + f"``{self.name}.coupling_directions``. You can solve " + f"this by defining it there@HINT_0@.", + hints=[ + { + "type": "prov", + "object": coupling_directions, + "text": " (for example near @HINT@)", + }, + ], + ) transf_info = None if "coupling_methods" in full_config[self.name]: if interpolation in full_config[self.name]["coupling_methods"]: @@ -174,7 +187,7 @@ def add_files(self, full_config): interpolation ] else: - esm_parser.user_error( + user_error( "Missing coupling method", f"The coupling method ``{interpolation}`` defined in " f"the ``{self.name}.coupling_target_fields`` is not " diff --git a/src/esm_runscripts/filelists.py b/src/esm_runscripts/filelists.py index 4da58712a..2bd95f497 100644 --- a/src/esm_runscripts/filelists.py +++ b/src/esm_runscripts/filelists.py @@ -11,12 +11,12 @@ import f90nml import yaml +from loguru import logger import esm_parser -from loguru import logger +from esm_tools import user_error -from . import helpers -from . import jinja +from . import helpers, jinja def rename_sources_to_targets(config): @@ -146,7 +146,7 @@ def complete_targets(config): f"The input file variable {category} of {filetype}_sources can not be fully resolved:\n\n" + yaml.dump(file_source, indent=4) ) - esm_parser.user_error(error_type, error_text) + user_error(error_type, error_text) else: config[model][filetype + "_targets"][ category @@ -306,7 +306,7 @@ def target_subfolders(config): # * only in targets if denotes subfolder # TODO: change with user_error() if not descr in config[model][filetype + "_sources"]: - esm_parser.user_error( + user_error( "Filelists", f"No source found for target ``{name}`` in model " f"``{model}``\n", @@ -390,7 +390,7 @@ def get_target_name_from_wildcard(config, model, filename, filetype, descr): Raises ------ - user_error : esm_parser.user_error + user_error : user_error If source and target wildcard patterns do not match (different number of ``*`` in the patterns), raises a user friendly error """ @@ -407,7 +407,7 @@ def get_target_name_from_wildcard(config, model, filename, filetype, descr): wild_card_target = target_filename.split("*") # Check for syntax mistakes if len(wild_card_target) != len(wild_card_source): - esm_parser.user_error( + user_error( "Wild card", ( "The wild card pattern of the source " @@ -502,7 +502,7 @@ def find_valid_year(config, year): error_type = "Year Error" error_text = f"Sorry, no entry found for year {year} in config {config}" - esm_parser.user_error(error_type, error_text) + user_error(error_type, error_text) def replace_year_placeholder(config): @@ -1271,7 +1271,7 @@ def avoid_overwriting(config, source, target): date_stamped_target = f"{target}_{config['general']['run_datestamp']}" if os.path.isfile(date_stamped_target): - esm_parser.user_error( + user_error( "File movement conflict", f"The file ``{date_stamped_target}`` already exists. Skipping movement:\n" f"{source} -> {date_stamped_target}", @@ -1287,7 +1287,7 @@ def avoid_overwriting(config, source, target): target = date_stamped_target elif os.path.isdir(target): - esm_parser.user_error( + user_error( "File operation not supported", f"The target ``{target}`` is a folder, and this should not be happening " "here. Please, open an issue in www.github.com/esm-tools/esm_tools", @@ -1531,7 +1531,7 @@ def complete_all_file_movements(config): if file_in_fm in mconfig.get( "restart_in_files", {} ) or file_in_fm in mconfig.get("restart_out_files", {}): - esm_parser.user_error( + user_error( "Movement direction not specified", f"'{model}.file_movements.{file_in_fm}' refers to a " + "restart file which can be moved/copied/link in two " diff --git a/src/esm_runscripts/helpers.py b/src/esm_runscripts/helpers.py index 293cc5a7c..664f00320 100644 --- a/src/esm_runscripts/helpers.py +++ b/src/esm_runscripts/helpers.py @@ -8,8 +8,8 @@ import esm_parser import esm_plugin_manager import esm_tools -from loguru import logger from esm_profile import print_profile_summary +from esm_tools import user_error def vprint(message, config): @@ -206,7 +206,7 @@ def update_reusable_filetypes(config, reusable_filetypes=None): for update_filetype in update_filetypes: # Check if that file type exists/makes sense. Otherwise, through an error if update_filetype not in potentially_reusable_filetypes: - esm_parser.user_error( + user_error( "update-filetypes", f"``{update_filetype}`` specified by you in ``--update-filetypes`` is " + "not a ESM-Tools file type. Please, select one (or more) of the " diff --git a/src/esm_runscripts/jinja.py b/src/esm_runscripts/jinja.py index e0d92adcf..c7fd04d11 100644 --- a/src/esm_runscripts/jinja.py +++ b/src/esm_runscripts/jinja.py @@ -3,7 +3,7 @@ from jinja2 import StrictUndefined, Template, UndefinedError from loguru import logger -import esm_parser +from esm_tools import user_error def render_template(config, source, target): @@ -40,7 +40,7 @@ def render_template(config, source, target): content = template.render(config) except UndefinedError as e: missing_variable = e.message.split("'")[3] - esm_parser.user_error( + user_error( "Jinja", f"Error rendering template from ``{source}`` to ``{target}``. Variable " f"``{missing_variable}`` is not defined in any configuration file.", diff --git a/src/esm_runscripts/namelists.py b/src/esm_runscripts/namelists.py index 8921dd340..6a7629df1 100644 --- a/src/esm_runscripts/namelists.py +++ b/src/esm_runscripts/namelists.py @@ -15,7 +15,7 @@ import f90nml from loguru import logger -from esm_parser import user_error +from esm_tools import user_error class Namelist: diff --git a/src/esm_runscripts/oasis.py b/src/esm_runscripts/oasis.py index 7a4eafaa0..fb78609c7 100644 --- a/src/esm_runscripts/oasis.py +++ b/src/esm_runscripts/oasis.py @@ -4,10 +4,9 @@ import sys import questionary - -from esm_parser import user_error from loguru import logger +from esm_tools import user_error class oasis: @@ -665,8 +664,15 @@ def prepare_restarts(self, restart_file, all_fields, models, config): # before (i.e. when using LOCTRANS) if os.path.isfile(restart_file): logger.debug(f"{restart_file} already exits, overwriting") - logger.info("cdo -O merge " + filelist + " " + restart_file) - os.system("cdo -O merge " + filelist + " " + restart_file) # + enddate) + + cdo_merge_command = f"cdo -O -f nc4c merge {filelist} {restart_file}" # {enddate}" + logger.info(cdo_merge_command) + exit_code = os.system(cdo_merge_command) + if exit_code != 0: + cdo_merge_command = f"cdo -O merge {filelist} {restart_file}" # {enddate}" + logger.warning("nc4c merge failed, trying without it...") + logger.info(cdo_merge_command) + os.system(cdo_merge_command) rmlist = glob.glob("notimestep*") rmlist.append("onlyonetimestep.nc") for rmfile in rmlist: diff --git a/src/esm_runscripts/prepare.py b/src/esm_runscripts/prepare.py index 9615091f1..10477d388 100644 --- a/src/esm_runscripts/prepare.py +++ b/src/esm_runscripts/prepare.py @@ -5,12 +5,13 @@ import questionary import yaml +from loguru import logger import esm_parser import esm_utilities from esm_calendar import Calendar, Date from esm_plugin_manager import install_missing_plugins -from loguru import logger +from esm_tools import user_error, user_note from . import batch_system, helpers @@ -222,7 +223,7 @@ def model_env_into_computer(config): model0 = env_vars[key][1] while True: # Warn the user about the overwriting of the variable - esm_parser.user_note("Environment conflict", f"In '{model0}':") + user_note("Environment conflict", f"In '{model0}':") esm_parser.pprint_config({key: env_vars[key][0]}) logging.info("\nIn '" + model + "':") esm_parser.pprint_config({key: value}) @@ -248,7 +249,7 @@ def model_env_into_computer(config): # If the user selects ``n`` raise a user error with recommendations elif user_answer == "n": config[model]["env_overwrite"] = False - esm_parser.user_error( + user_error( "Environment conflict", "You were not happy with the environment variable " + f"'{key}' in '{model0}' being overwritten by the same " @@ -719,7 +720,7 @@ def add_vcs_info(config): vcs_versions["esm_tools"] = helpers.get_all_git_info(f"{esm_tools_repo}/../") else: # FIXME(PG): This should absolutely never happen. The error message could use a better wording though... - esm_parser.user_error( + user_error( "esm_tools doesn't know where it's own install location is. Something is very seriously wrong." ) with open(exp_vcs_info_file, "w") as f: @@ -775,7 +776,7 @@ def check_vcs_info_against_last_run(config): not config["general"].get("allow_vcs_differences", False) and current_vcs_info != last_vcs_info ): - esm_parser.user_error( + user_error( "VCS Differences", """ You have differences in either the model code or in the esm-tools between two runs! @@ -871,7 +872,7 @@ def check_config_for_warnings_errors(config): """ # Initialize the trigger variables (i.e. ``error`` and ``warning``)) - triggers = {"error": {"note_function": esm_parser.user_error}} + triggers = {"error": {"note_function": user_error}} # Find conditions to warn (avoid warning more than once) last_jobtype = config["general"].get("last_jobtype", "") @@ -880,7 +881,7 @@ def check_config_for_warnings_errors(config): # Only warn if it is an interactive session or while submitted if not isresubmitted or isinteractive: - triggers["warning"] = {"note_function": esm_parser.user_note} + triggers["warning"] = {"note_function": user_note} # Loop through the triggers for trigger, trigger_info in triggers.items(): diff --git a/src/esm_runscripts/prepexp.py b/src/esm_runscripts/prepexp.py index b0fbe44f5..31b7b1f9f 100644 --- a/src/esm_runscripts/prepexp.py +++ b/src/esm_runscripts/prepexp.py @@ -6,10 +6,11 @@ import questionary from colorama import Fore +from loguru import logger import esm_parser import esm_tools -from loguru import logger +from esm_tools import user_error, user_note from . import filelists from .helpers import end_it_all, evaluate, write_to_log @@ -121,7 +122,7 @@ def copy_tools_to_thisrun(config): # exit right away to prevent further recursion. There might still be # running instances of esmr_runscripts and something like # `killall esm_runscripts` might be required - esm_parser.user_error(error_type, error_text) + user_error(error_type, error_text) # If ``fromdir`` and ``scriptsdir`` are different, we are not in the experiment. # In this case, update the runscript if necessary. @@ -194,7 +195,7 @@ def _call_esm_runscripts_internally(config, command, exedir): else: error_type = "runtime error in function ``_call_esm_runscripts_internally``" error_text = f"{exedir} does not exists. Aborting." - esm_parser.user_error(error_type, error_text) + user_error(error_type, error_text) logger.debug(command) @@ -447,7 +448,7 @@ def update_runscript(fromdir, scriptsdir, tfile, gconfig, file_type): # If the --update flag is used, notify that the target script will # be updated and do update it if gconfig["update"]: - esm_parser.user_note( + user_note( f"Original {file_type} different from target", f"{differences}\n{scriptsdir}/{tfile} will be updated!", ) @@ -457,7 +458,7 @@ def update_runscript(fromdir, scriptsdir, tfile, gconfig, file_type): # If the --update flag is not called, exit with an error showing the # user how to proceed else: - esm_parser.user_note( + user_note( f"Original {file_type} different from target", differences + "\n" diff --git a/src/esm_runscripts/prev_run.py b/src/esm_runscripts/prev_run.py index 7583872e9..3451289b2 100644 --- a/src/esm_runscripts/prev_run.py +++ b/src/esm_runscripts/prev_run.py @@ -3,10 +3,11 @@ import questionary import yaml +from loguru import logger import esm_parser from esm_calendar import Calendar, Date -from loguru import logger +from esm_tools import user_error, user_note class PrevRunInfo(dict): @@ -241,7 +242,7 @@ def prev_run_config(self): ) # Dates don't match if calc_prev_date_stamp != prev_date_stamp and self.warn: - esm_parser.user_note( + user_note( f"End date of the previous configuration file for '{component}'" + " not coinciding:", ( @@ -330,7 +331,7 @@ def find_config(self, component): "prev_run_config_file" ) if not user_prev_run_config_full_path: - esm_parser.user_error( + user_error( "'prev_run_config_file' not defined", "You are trying to run a branchoff experiment that uses the " + f"'prev_run' functionality for '{component}' without " @@ -368,7 +369,7 @@ def find_config(self, component): # Check for errors if not os.path.isdir(config_dir): - esm_parser.user_error( + user_error( "Config folder not existing", ( f"The config folder {config_dir} does not exist. " @@ -430,7 +431,7 @@ def find_config(self, component): ) else: # Error - esm_parser.user_error( + user_error( "Too many possible config files", "There is more than one config file with the same final date " + "as the one required for the continuation of this experiment." @@ -450,7 +451,7 @@ def find_config(self, component): prev_run_config_file = user_prev_run_config_file prev_run_config_path = f"{config_dir}/{prev_run_config_file}" if not os.path.isfile(prev_run_config_path): - esm_parser.user_error( + user_error( "'prev_run_config_file' incorrectly defined", f"The file defined in the '{component}.prev_run_config_path' " + f"({prev_run_config_path}) does not exist.", diff --git a/src/esm_runscripts/slurm.py b/src/esm_runscripts/slurm.py index aeb3f9ef8..48a744e60 100644 --- a/src/esm_runscripts/slurm.py +++ b/src/esm_runscripts/slurm.py @@ -7,9 +7,11 @@ import subprocess import sys -import esm_parser from loguru import logger +import esm_parser +from esm_tools import user_note + class Slurm: """ @@ -234,7 +236,7 @@ def write_het_par_wrappers(config): elif not config[model].get("execution_command") and not config[ model ].get("executable"): - esm_parser.user_note( + user_note( "Execution command", f"Execution command for ``{model}`` not found. This is okay " "if this component has no binary to be called, but if it does" diff --git a/src/esm_tests/__init__.py b/src/esm_tests/__init__.py index 078c53976..389a51863 100644 --- a/src/esm_tests/__init__.py +++ b/src/esm_tests/__init__.py @@ -2,7 +2,7 @@ __author__ = """Miguel Andres-Martinez""" __email__ = "miguel.andres-martinez@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .initialization import * from .read_shipped_data import * diff --git a/src/esm_tests/repos.py b/src/esm_tests/repos.py index 0b8cc992b..484658ee1 100644 --- a/src/esm_tests/repos.py +++ b/src/esm_tests/repos.py @@ -1,10 +1,11 @@ import os -import questionary +import questionary from loguru import logger +from esm_tools import user_error + from .test_utilities import sh -from esm_parser import user_error def update_resources_submodule(info, verbose=True): diff --git a/src/esm_tools/__init__.py b/src/esm_tools/__init__.py index 66c45d9d7..f317aa7eb 100644 --- a/src/esm_tools/__init__.py +++ b/src/esm_tools/__init__.py @@ -23,7 +23,7 @@ __author__ = """Dirk Barbi, Paul Gierz""" __email__ = "dirk.barbi@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" import functools import inspect @@ -38,6 +38,8 @@ import yaml from loguru import logger +from .error_handling import * + # Setup Loguru for the following cases: # A) If user sets if os.environ.get("DEBUG_ESM_TOOLS"): diff --git a/src/esm_tools/error_handling.py b/src/esm_tools/error_handling.py new file mode 100644 index 000000000..c1457fa88 --- /dev/null +++ b/src/esm_tools/error_handling.py @@ -0,0 +1,128 @@ +import re +import sys + +import colorama + + +def user_note( + note_heading, note_text, color=colorama.Fore.YELLOW, dsymbols=["``"], hints=[] +): + """ + Notify the user about something. In the future this should also write in the log. + + Parameters + ---------- + note_heading : str + Note type used for the heading. + text : str + Text clarifying the note. + """ + reset_s = colorama.Style.RESET_ALL + + # Convert a list of strings to a single string + if isinstance(note_text, list): + new_note_text = "" + for item in note_text: + new_note_text = f"{new_note_text}- {item}\n" + note_text = new_note_text + + # Add hints to the note_text + note_text = user_note_hints(note_text, hints) + + # Add color to the note_text + for dsymbol in dsymbols: + note_text = re.sub( + f"{dsymbol}([^{dsymbol}]*){dsymbol}", f"{color}\\1{reset_s}", str(note_text) + ) + print(f"\n{color}{note_heading}\n{'-' * len(note_heading)}{reset_s}") + print(f"{note_text}\n") + + +def user_error(error_type, error_text, exit_code=1, dsymbols=["``"], hints=[]): + """ + User-friendly error using ``sys.exit()`` instead of an ``Exception``. + + Parameters + ---------- + error_type : str + Error type used for the error heading. + text : str + Text clarifying the error. + exit_code : int + The exit code to send back to the parent process (default to 1) + """ + error_title = "ERROR: " + error_type + user_note( + error_title, error_text, color=colorama.Fore.RED, dsymbols=dsymbols, hints=hints + ) + sys.exit(exit_code) + + +def user_note_hints(note_text, hints): + """ + Add hints to the note text. The hints are added to the note text by replacing + the placeholders "@HINT_#@" with the actual hints from the hints list. + + Parameters + ---------- + note_text : str + The note text with placeholders for the hints. The placeholders are in the + form "@HINT_#@", where # is the index of the hint in the hints list. + hints : list + A list of hints to be added to the note text. Each hint is a dictionary with + the following keys: + - type: The type of the hint (e.g., "prov" for provenance) + - text: The text of the hint. This text can contain a placeholder "@HINT@" + which will be replaced with the actual hint corresponding to its index. + - object: The object to which the hint applies + + Returns + ------- + note_text : str + The note text with the placeholders replaced with the hints. + """ + + # Find all hints matching r"@HINT_(\d+)@" in the note_text + pattern = r"@HINT_(\d+)@" + hint_indexes = [int(x) for x in list(set(re.findall(pattern, note_text)))] + hint_indexes.sort() + + # Check whether all hint indexes are represented in the hints list + if hint_indexes != list(range(len(hints))): + raise ValueError( + "The hint indexes in the note_text do not match the hints list." + ) + + # Loop through the hints and replace the placeholders in the note_text + for indx, hint in enumerate(hints): + # Hint type provenance + if hint["type"] == "prov": + hint_text = hint["text"] + mapping_with_provenance = hint["object"] + + # Get the provenance + provenance = None + if hasattr(mapping_with_provenance, "extract_first_nested_values_provenance"): + provenance = mapping_with_provenance.extract_first_nested_values_provenance() + else: + logging.debug("No provenance found for %s", mapping_with_provenance) + # If the provenance is found, replace the placeholder with the provenance, + # otherwise remove the placeholder (provenance might not always exist) + if provenance: + prov_string = ( + f"``{provenance['yaml_file']}``," + f"line:``{provenance['line']}``," + f"col:``{provenance['col']}``" + ) + # Replace the HINT placeholder of the hint with the provenance string + hint_text = hint["text"].replace("@HINT@", prov_string) + # Replace the HINT placeholder on the note message with the final hint + note_text = note_text.replace(f"@HINT_{indx}@", hint_text) + else: + note_text = note_text.replace(f"@HINT_{indx}@", "") + else: + raise NotImplementedError( + f"Hint type {hint['type']} is not implemented yet." + ) + + return note_text diff --git a/src/esm_utilities/__init__.py b/src/esm_utilities/__init__.py index 05a64bacf..c3e9bbeb9 100644 --- a/src/esm_utilities/__init__.py +++ b/src/esm_utilities/__init__.py @@ -2,6 +2,6 @@ __author__ = """Paul Gierz""" __email__ = "pgierz@awi.de" -__version__ = "6.44.2" +__version__ = "6.46.1" from .utils import *