From 751cfce573470b65f0d5e8cdf0b1d2592b2226bc Mon Sep 17 00:00:00 2001 From: "lukasz.debek" Date: Wed, 18 Oct 2023 16:11:01 +0200 Subject: [PATCH 1/9] Moved handling of the 3Di working directory structure to threedi_mi_utils module. --- CHANGES.rst | 1 + dependencies.py | 1 + external-dependencies/populate.sh | 1 + gui/threedi_plugin_grid_result_dialog.py | 2 +- utils/workingdir.py | 227 ----------------------- 5 files changed, 4 insertions(+), 228 deletions(-) delete mode 100644 utils/workingdir.py diff --git a/CHANGES.rst b/CHANGES.rst index dc79d774..4816cb2b 100755 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,7 @@ - Cross-sectional discharge: minor bugfix to correctly set the attributes of the intersected flowlines - Sideview: fix for pure 1D models (#931) - Statistics: removed water_on_street preset +- Added threedi-mi-utils dependency to handle 3Di working directory structure 3.1.11 (2023-10-02) diff --git a/dependencies.py b/dependencies.py index 3b5f04d8..ea03001f 100644 --- a/dependencies.py +++ b/dependencies.py @@ -75,6 +75,7 @@ Dependency("hydxlib", "hydxlib", "==1.5.1", False), Dependency("h5netcdf", "h5netcdf", "", False), Dependency("greenlet", "greenlet", "!=0.4.17", False), + Dependency("threedi-mi-utils", "threedi_mi_utils", ">=0.0.1", False), ] # On Windows, the hdf5 binary and thus h5py version depends on the QGis version diff --git a/external-dependencies/populate.sh b/external-dependencies/populate.sh index 931e2255..c1251c5e 100755 --- a/external-dependencies/populate.sh +++ b/external-dependencies/populate.sh @@ -27,6 +27,7 @@ networkx \ packaging \ pyqtgraph \ python-editor \ +threedi-mi-utils \ threedi-modelchecker \ threedi-schema \ threedidepth \ diff --git a/gui/threedi_plugin_grid_result_dialog.py b/gui/threedi_plugin_grid_result_dialog.py index 3e390a99..ed8138ac 100644 --- a/gui/threedi_plugin_grid_result_dialog.py +++ b/gui/threedi_plugin_grid_result_dialog.py @@ -9,7 +9,7 @@ from qgis.PyQt.QtWidgets import QAbstractItemView from qgis.core import QgsSettings from qgis.PyQt.QtGui import QStandardItemModel, QStandardItem -from threedi_results_analysis.utils.workingdir import list_local_schematisations +from threedi_mi_utils import list_local_schematisations logger = logging.getLogger(__name__) diff --git a/utils/workingdir.py b/utils/workingdir.py deleted file mode 100644 index 9ce429f9..00000000 --- a/utils/workingdir.py +++ /dev/null @@ -1,227 +0,0 @@ -from collections import OrderedDict -from itertools import chain -import json -import re -import os -from threedi_results_analysis.utils.utils import listdirs - - -class LocalRevision: - """Local revision directory structure representation.""" - - def __init__(self, local_schematisation, revision_number): - self.local_schematisation = local_schematisation - self.number = revision_number - - def structure_is_valid(self): - """Check if all revision subpaths are present.""" - is_valid = all(os.path.exists(p) if p else False for p in self.subpaths) - return is_valid - - @property - def sub_dir(self): - """Get schematisation revision subdirectory name.""" - subdirectory = f"revision {self.number}" - return subdirectory - - @property - def main_dir(self): - """Get schematisation revision main directory path.""" - schematisation_dir_path = self.local_schematisation.main_dir - schematisation_revision_dir_path = os.path.join(schematisation_dir_path, self.sub_dir) - return schematisation_revision_dir_path - - @property - def admin_dir(self): - """Get schematisation revision admin directory path.""" - admin_dir_path = os.path.join(self.main_dir, "admin") - return admin_dir_path - - @property - def grid_dir(self): - """Get schematisation revision grid directory path.""" - grid_dir_path = os.path.join(self.main_dir, "grid") - return grid_dir_path - - @property - def results_dir(self): - """Get schematisation revision results directory path.""" - grid_dir_path = os.path.join(self.main_dir, "results") - return grid_dir_path - - @property - def results_dirs(self): - """Get all (full) result folders""" - if not os.path.isdir(self.results_dir): - return [] - return listdirs(self.results_dir) - - @property - def schematisation_dir(self): - """Get schematisation revision schematisation directory path.""" - grid_dir_path = os.path.join(self.main_dir, "schematisation") - return grid_dir_path - - @property - def raster_dir(self): - """Get schematisation revision raster directory path.""" - rasters_dir_path = os.path.join(self.main_dir, "schematisation", "rasters") - return rasters_dir_path - - @property - def subpaths(self): - """Revision directory sub-paths.""" - paths = [ - self.admin_dir, - self.grid_dir, - self.results_dir, - self.schematisation_dir, - self.raster_dir, - ] - return paths - - -class WIPRevision(LocalRevision): - """Local Work In Progress directory structure representation.""" - - @property - def sub_dir(self): - """Get schematisation revision subdirectory name.""" - subdirectory = "work in progress" - return subdirectory - - @property - def main_dir(self): - """Get schematisation revision main directory path.""" - schematisation_dir_path = self.local_schematisation.main_dir - schematisation_revision_dir_path = os.path.join(schematisation_dir_path, self.sub_dir) - return schematisation_revision_dir_path - - @property - def admin_dir(self): - """Get schematisation revision admin directory path.""" - admin_dir_path = os.path.join(self.main_dir, "admin") - return admin_dir_path - - @property - def grid_dir(self): - """Get schematisation revision grid directory path.""" - grid_dir_path = os.path.join(self.main_dir, "grid") - return grid_dir_path - - @property - def results_dir(self): - """Get schematisation revision results directory path.""" - grid_dir_path = os.path.join(self.main_dir, "results") - return grid_dir_path - - @property - def schematisation_dir(self): - """Get schematisation revision schematisation directory path.""" - grid_dir_path = os.path.join(self.main_dir, "schematisation") - return grid_dir_path - - @property - def raster_dir(self): - """Get schematisation revision raster directory path.""" - rasters_dir_path = os.path.join(self.main_dir, "schematisation", "rasters") - return rasters_dir_path - - -class LocalSchematisation: - """Local revision directory structure representation.""" - - def __init__(self, working_dir, schematisation_pk, schematisation_name, parent_revision_number=None): - self.working_directory = working_dir - self.id = schematisation_pk - self.name = schematisation_name - self.revisions = OrderedDict() - self.wip_revision = WIPRevision(self, parent_revision_number) if parent_revision_number is not None else None - - @classmethod - def initialize_from_location(cls, schematisation_dir, use_config_for_revisions=True): - """ - Initialize local schematisation structure from the root schematisation dir. - In case use_config_for_revisions is True, the revisions are derived from the json file, - otherwise the schematisation dir is scanned for "revision" folders. - """ - working_dir = os.path.dirname(schematisation_dir) - if not os.path.isdir(schematisation_dir): - return None - config_path = os.path.join(schematisation_dir, "admin", "schematisation.json") - schema_metadata = cls.read_schematisation_metadata(config_path) - fallback_id = fallback_name = os.path.basename(schematisation_dir) - schematisation_pk = schema_metadata.get("id", fallback_id) - schematisation_name = schema_metadata.get("name", fallback_name) - local_schematisation = cls(working_dir, schematisation_pk, schematisation_name) - - if use_config_for_revisions: - revision_numbers = schema_metadata.get("revisions", []) - else: - folders = [ - os.path.basename(d) - for d in listdirs(schematisation_dir) - if os.path.basename(d).startswith("revision") - ] - revision_numbers = [int(re.findall(r'^revision (\d+)', folder)[0]) for folder in folders] - - for revision_number in revision_numbers: - local_revision = LocalRevision(local_schematisation, revision_number) - local_schematisation.revisions[revision_number] = local_revision - - wip_parent_revision_number = schema_metadata.get("wip_parent_revision") - if wip_parent_revision_number is not None: - local_schematisation.wip_revision = WIPRevision(local_schematisation, wip_parent_revision_number) - - return local_schematisation - - @staticmethod - def read_schematisation_metadata(schematisation_config_path): - """Read schematisation metadata from the JSON file.""" - if not os.path.exists(schematisation_config_path): - return {} - with open(schematisation_config_path, "r+") as config_file: - return json.load(config_file) - - def structure_is_valid(self): - """Check if all schematisation subpaths are present.""" - subpaths_collections = [self.subpaths] - subpaths_collections += [local_revision.subpaths for local_revision in self.revisions.values()] - subpaths_collections.append(self.wip_revision.subpaths) - is_valid = all(os.path.exists(p) if p else False for p in chain.from_iterable(subpaths_collections)) - return is_valid - - @property - def main_dir(self): - """Get schematisation main directory.""" - schematisation_dir_path = os.path.normpath(os.path.join(self.working_directory, self.name)) - return schematisation_dir_path - - @property - def admin_dir(self): - """Get schematisation admin directory path.""" - admin_dir_path = os.path.join(self.main_dir, "admin") - return admin_dir_path - - @property - def subpaths(self): - """Get schematisation directory sub-paths.""" - paths = [self.admin_dir] - return paths - - @property - def schematisation_config_path(self): - """Get schematisation configuration filepath.""" - config_path = os.path.join(self.admin_dir, "schematisation.json") - return config_path - - -def list_local_schematisations(working_dir): - """Get local schematisations present in the given directory.""" - local_schematisations = OrderedDict() - for basename in os.listdir(working_dir): - full_path = os.path.join(working_dir, basename) - local_schematisation = LocalSchematisation.initialize_from_location(full_path, False) - if local_schematisation is not None: - local_schematisations[local_schematisation.id] = local_schematisation - return local_schematisations From 0385e21af60ba615a07d3124412e1e2839591b0c Mon Sep 17 00:00:00 2001 From: "lukasz.debek" Date: Thu, 19 Oct 2023 10:44:15 +0200 Subject: [PATCH 2/9] Moved handling of the 3Di working directory structure to threedi_mi_utils module. --- dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.py b/dependencies.py index ea03001f..4786b912 100644 --- a/dependencies.py +++ b/dependencies.py @@ -75,7 +75,7 @@ Dependency("hydxlib", "hydxlib", "==1.5.1", False), Dependency("h5netcdf", "h5netcdf", "", False), Dependency("greenlet", "greenlet", "!=0.4.17", False), - Dependency("threedi-mi-utils", "threedi_mi_utils", ">=0.0.1", False), + Dependency("threedi-mi-utils", "threedi_mi_utils", "==0.1.0", False), ] # On Windows, the hdf5 binary and thus h5py version depends on the QGis version From a18e78373511065cc52dafc52f4f1fe433826b70 Mon Sep 17 00:00:00 2001 From: "lukasz.debek" Date: Mon, 23 Oct 2023 14:02:57 +0200 Subject: [PATCH 3/9] Updated CHANGES.rst --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 4816cb2b..060da933 100755 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,7 @@ - Statistics: Add two water on street presets. - General: added some layer loading feedback. +- Passed handling of the 3Di working directory structure to `threedi_mi_utils` package. 3.1.12 (2023-10-19) From 536f050abc81497f9619136a1016a54f499712e6 Mon Sep 17 00:00:00 2001 From: "lukasz.debek" Date: Tue, 31 Oct 2023 13:38:23 +0100 Subject: [PATCH 4/9] After review fixes. --- gui/threedi_plugin_grid_result_dialog.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gui/threedi_plugin_grid_result_dialog.py b/gui/threedi_plugin_grid_result_dialog.py index ed8138ac..82edc20f 100644 --- a/gui/threedi_plugin_grid_result_dialog.py +++ b/gui/threedi_plugin_grid_result_dialog.py @@ -4,6 +4,7 @@ import os from threedi_results_analysis.utils.constants import TOOLBOX_QGIS_SETTINGS_GROUP +from threedi_results_analysis.utils.utils import listdirs from qgis.PyQt import QtWidgets, uic from qgis.PyQt.QtCore import pyqtSignal, pyqtSlot, QModelIndex from qgis.PyQt.QtWidgets import QAbstractItemView @@ -204,7 +205,8 @@ def refresh(self): # Iterate over revisions for revision_number, local_revision in local_schematisation.revisions.items(): # Iterate over results - for result_dir in local_revision.results_dirs: + local_revision_results_dirs = listdirs(local_revision.results_dir) if os.path.isdir(local_revision.results_dir) else [] + for result_dir in local_revision_results_dirs: schema_item = QStandardItem(local_schematisation.name) schema_item.setEditable(False) revision_item = QStandardItem(str(revision_number)) @@ -218,7 +220,7 @@ def refresh(self): self.model.appendRow([schema_item, revision_item, result_item]) # In case no results are present, but a gridadmin is present, we still add the grid, but without result item - if len(local_revision.results_dirs) == 0 and os.path.exists(os.path.join(local_revision.grid_dir, "gridadmin.h5")): + if len(local_revision_results_dirs) == 0 and os.path.exists(os.path.join(local_revision.grid_dir, "gridadmin.h5")): schema_item = QStandardItem(local_schematisation.name) schema_item.setEditable(False) revision_item = QStandardItem(str(revision_number)) From ede5059dfe11dc7af302bc5f44ce9d3c8fdafcfc Mon Sep 17 00:00:00 2001 From: "lukasz.debek" Date: Tue, 31 Oct 2023 13:42:17 +0100 Subject: [PATCH 5/9] After review fixes. --- CHANGES.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 060da933..d8277016 100755 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -15,7 +15,6 @@ - Cross-sectional discharge: minor bugfix to correctly set the attributes of the intersected flowlines - Sideview: fix for pure 1D models (#931) - Statistics: removed water_on_street preset -- Added threedi-mi-utils dependency to handle 3Di working directory structure 3.1.11 (2023-10-02) From 80bc1b2872af062017656d9febee924ac7721c97 Mon Sep 17 00:00:00 2001 From: "lukasz.debek" Date: Mon, 6 Nov 2023 16:09:26 +0100 Subject: [PATCH 6/9] After review fixes 2. --- dependencies.py | 2 +- gui/threedi_plugin_grid_result_dialog.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dependencies.py b/dependencies.py index 4786b912..1ea70275 100644 --- a/dependencies.py +++ b/dependencies.py @@ -75,7 +75,7 @@ Dependency("hydxlib", "hydxlib", "==1.5.1", False), Dependency("h5netcdf", "h5netcdf", "", False), Dependency("greenlet", "greenlet", "!=0.4.17", False), - Dependency("threedi-mi-utils", "threedi_mi_utils", "==0.1.0", False), + Dependency("threedi-mi-utils", "threedi_mi_utils", "==0.1.1", False), ] # On Windows, the hdf5 binary and thus h5py version depends on the QGis version diff --git a/gui/threedi_plugin_grid_result_dialog.py b/gui/threedi_plugin_grid_result_dialog.py index 82edc20f..a7eff44c 100644 --- a/gui/threedi_plugin_grid_result_dialog.py +++ b/gui/threedi_plugin_grid_result_dialog.py @@ -205,8 +205,7 @@ def refresh(self): # Iterate over revisions for revision_number, local_revision in local_schematisation.revisions.items(): # Iterate over results - local_revision_results_dirs = listdirs(local_revision.results_dir) if os.path.isdir(local_revision.results_dir) else [] - for result_dir in local_revision_results_dirs: + for result_dir in local_revision.results_dirs: schema_item = QStandardItem(local_schematisation.name) schema_item.setEditable(False) revision_item = QStandardItem(str(revision_number)) @@ -220,7 +219,7 @@ def refresh(self): self.model.appendRow([schema_item, revision_item, result_item]) # In case no results are present, but a gridadmin is present, we still add the grid, but without result item - if len(local_revision_results_dirs) == 0 and os.path.exists(os.path.join(local_revision.grid_dir, "gridadmin.h5")): + if len(local_revision.results_dirs) == 0 and os.path.exists(os.path.join(local_revision.grid_dir, "gridadmin.h5")): schema_item = QStandardItem(local_schematisation.name) schema_item.setEditable(False) revision_item = QStandardItem(str(revision_number)) From 5600e0c4c73aeb33d7013505d721ab743d1d5409 Mon Sep 17 00:00:00 2001 From: "lukasz.debek" Date: Mon, 6 Nov 2023 16:15:13 +0100 Subject: [PATCH 7/9] After review fixes 2. --- gui/threedi_plugin_grid_result_dialog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gui/threedi_plugin_grid_result_dialog.py b/gui/threedi_plugin_grid_result_dialog.py index a7eff44c..ed8138ac 100644 --- a/gui/threedi_plugin_grid_result_dialog.py +++ b/gui/threedi_plugin_grid_result_dialog.py @@ -4,7 +4,6 @@ import os from threedi_results_analysis.utils.constants import TOOLBOX_QGIS_SETTINGS_GROUP -from threedi_results_analysis.utils.utils import listdirs from qgis.PyQt import QtWidgets, uic from qgis.PyQt.QtCore import pyqtSignal, pyqtSlot, QModelIndex from qgis.PyQt.QtWidgets import QAbstractItemView From fba18136ee484f4d3a741fe843ddfd98ff70072b Mon Sep 17 00:00:00 2001 From: "lukasz.debek" Date: Mon, 6 Nov 2023 16:24:19 +0100 Subject: [PATCH 8/9] Minor code change after review fixes. --- gui/threedi_plugin_grid_result_dialog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gui/threedi_plugin_grid_result_dialog.py b/gui/threedi_plugin_grid_result_dialog.py index ed8138ac..8efb09eb 100644 --- a/gui/threedi_plugin_grid_result_dialog.py +++ b/gui/threedi_plugin_grid_result_dialog.py @@ -203,6 +203,7 @@ def refresh(self): for schematisation_id, local_schematisation in local_schematisations.items(): # Iterate over revisions for revision_number, local_revision in local_schematisation.revisions.items(): + num_of_results = len(local_revision.results_dirs) # Iterate over results for result_dir in local_revision.results_dirs: schema_item = QStandardItem(local_schematisation.name) @@ -218,7 +219,7 @@ def refresh(self): self.model.appendRow([schema_item, revision_item, result_item]) # In case no results are present, but a gridadmin is present, we still add the grid, but without result item - if len(local_revision.results_dirs) == 0 and os.path.exists(os.path.join(local_revision.grid_dir, "gridadmin.h5")): + if num_of_results == 0 and os.path.exists(os.path.join(local_revision.grid_dir, "gridadmin.h5")): schema_item = QStandardItem(local_schematisation.name) schema_item.setEditable(False) revision_item = QStandardItem(str(revision_number)) From d3d9b99843629f8168a6832fda5b2815d05769bd Mon Sep 17 00:00:00 2001 From: "lukasz.debek" Date: Mon, 6 Nov 2023 16:53:22 +0100 Subject: [PATCH 9/9] Added `use_config_for_revisions` flag to the `list_local_schematisations` function. --- dependencies.py | 2 +- gui/threedi_plugin_grid_result_dialog.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies.py b/dependencies.py index 1ea70275..cc6cf1ed 100644 --- a/dependencies.py +++ b/dependencies.py @@ -75,7 +75,7 @@ Dependency("hydxlib", "hydxlib", "==1.5.1", False), Dependency("h5netcdf", "h5netcdf", "", False), Dependency("greenlet", "greenlet", "!=0.4.17", False), - Dependency("threedi-mi-utils", "threedi_mi_utils", "==0.1.1", False), + Dependency("threedi-mi-utils", "threedi_mi_utils", "==0.1.2", False), ] # On Windows, the hdf5 binary and thus h5py version depends on the QGis version diff --git a/gui/threedi_plugin_grid_result_dialog.py b/gui/threedi_plugin_grid_result_dialog.py index 8efb09eb..08ea4c9b 100644 --- a/gui/threedi_plugin_grid_result_dialog.py +++ b/gui/threedi_plugin_grid_result_dialog.py @@ -199,7 +199,7 @@ def refresh(self): self.messageLabel.setText("Please set your 3Di working directory in the 3Di Models & Simulations settings to be able to load computational grids and results from your 3Di working directory.") return - local_schematisations = list_local_schematisations(threedi_working_dir) + local_schematisations = list_local_schematisations(threedi_working_dir, use_config_for_revisions=False) for schematisation_id, local_schematisation in local_schematisations.items(): # Iterate over revisions for revision_number, local_revision in local_schematisation.revisions.items():