Merge pull request #1288 from esm-tools/fix/force_overwrite_in_file_m…

…ovements

fix/force overwrite in file movements

configs/defaults/general.yaml

@@ -6,3 +6,4 @@ intermediate_movements:
 - "input"
 compute_file_checksums: False
 parallel_file_movements: False
+force_overwrite_in_file_movements: False

docs/esm_variables.rst

@@ -69,6 +69,7 @@ Runtime variables
    execution_command,   <component>,            "Command for executing the component, including ``${executable}`` and the necessary flags."
    expid,               general,                "ID of the experiment. This variable can also be defined when calling ``esm_runscripts`` with the ``-e`` flag."
    ":ref:`yaml:File Dictionaries`",     <component>,    "`YAML` dictionaries used to handle input, output, forcing, logging, binary and restart files (see :ref:`yaml:File Dictionaries`)."
+   force_overwrite_in_file_movements, general   "A boolean to indicate whether the file movements should overwrite existing files or not. If ``False`` (default), the file movements will not overwrite existing files. Only set to ``True`` if you know why you would want to do that (e.g to overwrite files in a failed tidy task)."
    heterogeneous_parallelization,   computer,    "A boolean that controls whether the simulation needs to be run with or without heterogeneous parallelization. When ``false`` OpenMP is not used for any component, independently of the value of ``omp_num_threads`` defined in the components. When ``true``, ``open_num_threads`` needs to be specified for each component using OpenMP. ``heterogeneous_parallelization`` variable **needs to be defined inside the** ``computer`` section of the runscript. See :ref:`cookbook:Heterogeneous Parallelization Run (MPI/OpenMP)` for examples."
    ini_restart_dir,     <component>,            "Path of the restarted experiment in case the current experiment runs in a different directory. For this variable to have an effect ``lresume`` needs to be ``true`` (e.g. the experiment is a restart)."
    ini_restart_exp_id,  <component>,            "ID of the restarted experiment in case the current experiment has a different ``expid``. For this variable to have an effect ``lresume`` needs to be ``true`` (e.g. the experiment is a restart)."

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 6.46.1
+current_version = 6.46.2
 commit = True
 tag = True
 

setup.py

@@ -117,6 +117,6 @@
     test_suite="tests",
     tests_require=test_requirements,
     url="https://github.com/esm-tools/esm_tools",
-    version="6.46.1",
+    version="6.46.2",
     zip_safe=False,
 )

src/esm_archiving/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = """Paul Gierz"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 from .esm_archiving import (archive_mistral, check_tar_lists,
                             delete_original_data, determine_datestamp_location,

src/esm_calendar/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 from .esm_calendar import *

src/esm_cleanup/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"

src/esm_database/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"

src/esm_environment/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 from .esm_environment import *

src/esm_master/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 
 from . import database

src/esm_motd/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 from .esm_motd import *

src/esm_parser/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 
 from .dict_to_yaml import *

src/esm_plugin_manager/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi, Paul Gierz, Sebastian Wahl"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 from .esm_plugin_manager import *

src/esm_profile/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 from .esm_profile import *

src/esm_runscripts/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dirk Barbi"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 from .batch_system import *
 from .chunky_parts import *

src/esm_runscripts/filelists.py

@@ -1250,7 +1250,9 @@ def avoid_overwriting(config, source, target):
     Function that appends the date stamp to ``target`` if the target already exists.
     Additionally, if the target exists, it renames it with the previous run time stamp,
     and creates a link named ``target`` that points at the target with the current time
-    stamp.
+    stamp. If the target exists and is not identical to the source then it will raise
+    a user friendly error, unless the ``force_overwrite_in_file_movements`` is set to
+    ``True`` in the general configuration (which will enforce the overwrite).
 
     Note
     ----
@@ -1264,19 +1266,35 @@ def avoid_overwriting(config, source, target):
         Path of the source of the file that will be copied/moved/linked
     target : src
         Path of the target of the file that will be copied/moved/linked
+
+    Raises
+    ------
+    user_error : esm_tools.helpers.user_error
+        If the target file exists and is not identical to the source file
     """
     if os.path.isfile(target):
         if filecmp.cmp(source, target):
             return target
 
         date_stamped_target = f"{target}_{config['general']['run_datestamp']}"
         if os.path.isfile(date_stamped_target):
-            user_error(
+            if filecmp.cmp(source, date_stamped_target):
+                logger.info(
+                    f"File {date_stamped_target} already exists and is identical to "
+                    f"the source ({source}). Skipping copying"
+                )
+            elif config["general"]["force_overwrite_in_file_movements"]:
+                os.remove(date_stamped_target)
+                warning_function = user_note
+            else:
+                # This will exit(1) (default in configs/defaults/general.yaml)
+                warning_function = user_error
+
+            warning_function(
                 "File movement conflict",
                 f"The file ``{date_stamped_target}`` already exists. Skipping movement:\n"
                 f"{source} -> {date_stamped_target}",
             )
-            return target
 
         if os.path.islink(target):
             os.remove(target)

src/esm_tests/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Miguel Andres-Martinez"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 from .initialization import *
 from .read_shipped_data import *

src/esm_tools/__init__.py

@@ -23,7 +23,7 @@
 
 __author__ = """Dirk Barbi, Paul Gierz"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 import functools
 import inspect

src/esm_utilities/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """Paul Gierz"""
 __email__ = "[email protected]"
-__version__ = "6.46.1"
+__version__ = "6.46.2"
 
 from .utils import *