Merge branch 'release' into feat/nml_transit_for_tracer

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 6.44.2
+current_version = 6.46.1
 commit = True
 tag = True
 

setup.py

@@ -117,6 +117,6 @@
     test_suite="tests",
     tests_require=test_requirements,
     url="https://github.com/esm-tools/esm_tools",
-    version="6.44.2",
+    version="6.46.1",
     zip_safe=False,
 )

src/esm_archiving/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = """Paul Gierz"""
 __email__ = "[email protected]"
-__version__ = "6.44.2"
+__version__ = "6.46.1"
 
 from .esm_archiving import (archive_mistral, check_tar_lists,
                             delete_original_data, determine_datestamp_location,

src/esm_calendar/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.44.2"
+__version__ = "6.46.1"
 
 from .esm_calendar import *

src/esm_cleanup/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.44.2"
+__version__ = "6.46.1"

src/esm_database/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.44.2"
+__version__ = "6.46.1"

src/esm_environment/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.44.2"
+__version__ = "6.46.1"
 
 from .esm_environment import *

src/esm_master/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.44.2"
+__version__ = "6.46.1"
 
 
 from . import database

src/esm_master/task.py

@@ -1,18 +1,16 @@
 import os
-import sys
-import subprocess
-import shlex  # contains shlex.split that respects quoted strings
-
 # deniz: it is better to use more pathlib in the future so that dir/path
 # operations will be more portable (supported since Python 3.4, 2014)
 import pathlib
-
-from .software_package import software_package
-from esm_parser import user_error
+import shlex  # contains shlex.split that respects quoted strings
+import subprocess
+import sys
 
 import esm_environment
 import esm_plugin_manager
+from esm_tools import user_error
 
+from .software_package import software_package
 
 ######################################################################################
 ################################# class "task" #######################################

src/esm_motd/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.44.2"
+__version__ = "6.46.1"
 
 from .esm_motd import *

src/esm_motd/esm_motd.py

@@ -3,11 +3,12 @@
 import urllib.request
 from time import sleep
 
-import esm_parser
-import esm_utilities
 import yaml
 
+import esm_parser
 import esm_tools
+import esm_utilities
+from esm_tools import user_error
 
 
 class MessageOfTheDayError(Exception):
@@ -83,7 +84,7 @@ def action_handler(self, action, time, package, version):
             if action == "sleep":
                 sleep(time)
             elif action == "error":
-                esm_parser.user_error(
+                user_error(
                     "Version",
                     (
                         f"Version {version} of '{package}' package has been tagged as "

src/esm_parser/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.44.2"
+__version__ = "6.46.1"
 
 
 from .dict_to_yaml import *

src/esm_parser/esm_parser.py

@@ -53,19 +53,15 @@
 Specific documentation for classes and functions are given below:
 """
 # Python 2 and 3 version agnostic compatiability:
-from __future__ import print_function
-from __future__ import unicode_literals
-from __future__ import division
-from __future__ import absolute_import
-
-import pdb
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
 
 # Python Standard Library imports
 import collections
 import copy
 import logging
 import os
-import re
+import pdb
 import shutil
 import socket
 import subprocess
@@ -79,21 +75,20 @@
 
 # Always import externals before any non standard library imports
 
+import coloredlogs
 # Third-Party Imports
 import numpy
-import coloredlogs
-import colorama
 import yaml
 
-# functions reading in dict from file
-from .yaml_to_dict import *
-from .provenance import *
-
+# Loader for package yamls
+import esm_tools
 # Date class
 from esm_calendar import Date
+from esm_tools import user_error, user_note
 
-# Loader for package yamls
-import esm_tools
+from .provenance import *
+# functions reading in dict from file
+from .yaml_to_dict import *
 
 # Logger and related constants
 logger = logging.getLogger("root")
@@ -2325,12 +2320,14 @@ def do_math_in_entry(tree, rhs, config):
     if "${" in str(entry):
         return entry
     entry = " " + str(entry) + " "
+    date_operation = False
     while "$((" in entry:
         math, after_math = entry.split("))", 1)
         math, before_math = math[::-1].split("(($", 1)
         math = math[::-1]
         before_math = before_math[::-1]
         if DATE_MARKER in math:
+            date_operation = True
             all_dates = []
             steps = math.split(" ")
             steps = [step for step in steps if step]
@@ -2408,12 +2405,18 @@ def do_math_in_entry(tree, rhs, config):
                     math = math + "all_dates[" + str(index) + "]"
                     index += 1
         result = eval(math)
-        if isinstance(result, list):
+        if isinstance(result, list) and date_operation:
             result = result[
                 -1
             ]  # should be extended in the future - here: if list (= if diff between dates) than result in seconds
+        elif isinstance(result, list):
+            entry = ListWithProvenance(result, None)
+            entry.set_provenance(rhs.provenance)
+            return entry
+
         result = str(result)
         entry = before_math + result + after_math
+
     # TODO MA: this is a provisional dirty fix for release. Get rid of this once a more
     # general solution is worked out
     # ORIGINAL LINE: return convert(entry.strip())
@@ -2789,51 +2792,6 @@ def find_key(d_search, k_search, exc_strings="", level="", paths2finds=[], sep="
     return paths2finds
 
 
-def user_note(note_heading, note_text, color=colorama.Fore.YELLOW, dsymbols=["``"]):
-    """
-    Notify the user about something. In the future this should also write in the log.
-
-    Parameters
-    ----------
-    note_heading : str
-        Note type used for the heading.
-    text : str
-        Text clarifying the note.
-    """
-    reset_s = colorama.Style.RESET_ALL
-
-    if isinstance(note_text, list):
-        new_note_text = ""
-        for item in note_text:
-            new_note_text = f"{new_note_text}- {item}\n"
-        note_text = new_note_text
-
-    for dsymbol in dsymbols:
-        note_text = re.sub(
-            f"{dsymbol}([^{dsymbol}]*){dsymbol}", f"{color}\\1{reset_s}", str(note_text)
-        )
-    print(f"\n{color}{note_heading}\n{'-' * len(note_heading)}{reset_s}")
-    print(f"{note_text}\n")
-
-
-def user_error(error_type, error_text, exit_code=1, dsymbols=["``"]):
-    """
-    User-friendly error using ``sys.exit()`` instead of an ``Exception``.
-
-    Parameters
-    ----------
-    error_type : str
-        Error type used for the error heading.
-    text : str
-        Text clarifying the error.
-    exit_code : int
-        The exit code to send back to the parent process (default to 1)
-    """
-    error_title = "ERROR: " + error_type
-    user_note(error_title, error_text, color=colorama.Fore.RED, dsymbols=dsymbols)
-    sys.exit(exit_code)
-
-
 class GeneralConfig(dict):  # pragma: no cover
     """All configs do this!"""
 

src/esm_parser/provenance.py

@@ -523,6 +523,23 @@ def get_provenance(self, index=-1):
 
         return provenance_dict
 
+    def extract_first_nested_values_provenance(self):
+        """
+        Recursively loops through the dictionary keys and returns the first provenance
+        found in the nested values.
+
+        Returns
+        -------
+        first_provenance : esm_parser.provenance.Provenance
+            The first provenance found in the nested values
+        """
+        first_provenance = None
+        for key, val in self.items():
+            if isinstance(val, PROVENANCE_MAPPINGS):
+                return val.extract_first_nested_values_provenance()
+            elif hasattr(val, "provenance"):
+                return val.provenance[-1]
+
     def __setitem__(self, key, val):
         """
         Any time an item in a DictWithProvenance is set, extend the old provenance of
@@ -773,6 +790,23 @@ def get_provenance(self, index=-1):
 
         return provenance_list
 
+    def extract_first_nested_values_provenance(self):
+        """
+        Recursively loops through the list elements and returns the first provenance
+        found in the nested values.
+
+        Returns
+        -------
+        first_provenance : esm_parser.provenance.Provenance
+            The first provenance found in the nested values
+        """
+        first_provenance = None
+        for elem in self:
+            if isinstance(elem, PROVENANCE_MAPPINGS):
+                return elem.extract_first_nested_values_provenance()
+            elif hasattr(elem, "provenance"):
+                return elem.provenance[-1]
+
     def __setitem__(self, indx, val):
         """
         Any time an item in a ListWithProvenance is set, extend the old provenance of