From 76d39bb55801562e1154a6b42c6d1adf31d8aa54 Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Thu, 21 Nov 2024 13:45:07 -0500 Subject: [PATCH 01/15] add config dictionary argument --- analysis-scripts/README.md | 7 ++++--- analysis-scripts/analysis_scripts/__init__.py | 3 ++- freanalysis/README.md | 4 ++-- freanalysis/freanalysis/plugins.py | 5 +++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/analysis-scripts/README.md b/analysis-scripts/README.md index ccf6b17..0884e27 100644 --- a/analysis-scripts/README.md +++ b/analysis-scripts/README.md @@ -1,10 +1,10 @@ # analysis-scripts -A framework for analyzing GFDL model output +A framework for analyzing GFDL model output. ### Motivation The goal of this project is to provide a simple API to guide the development of scripts that produce figures and tables from GFDL model output. This work will be used -by a simple web application to provide users an easy interface to interact with model +by a simple web application to provide users with an easy interface to interact with model output data. ### Requirements @@ -67,12 +67,13 @@ class NewAnalysisScript(AnalysisScript): } }) - def run_analysis(self, catalog, png_dir, reference_catalog=None): + def run_analysis(self, catalog, png_dir, config=None, reference_catalog=None): """Runs the analysis and generates all plots and associated datasets. Args: catalog: Path to a model output catalog. png_dir: Directory to store output png figures in. + config: Dictionary of configuration options. reference_catalog: Path to a catalog of reference data. Returns: diff --git a/analysis-scripts/analysis_scripts/__init__.py b/analysis-scripts/analysis_scripts/__init__.py index df1eb04..266dd09 100644 --- a/analysis-scripts/analysis_scripts/__init__.py +++ b/analysis-scripts/analysis_scripts/__init__.py @@ -24,12 +24,13 @@ def requires(self): raise NotImplementedError("you must override this function.") return json.dumps("{json of metadata MDTF format.}") - def run_analysis(self, catalog, png_dir, reference_catalog=None): + def run_analysis(self, catalog, png_dir, config=None, reference_catalog=None): """Runs the analysis and generates all plots and associated datasets. Args: catalog: Path to a model output catalog. png_dir: Directory to store output png figures in. + config: Dictionary of configuration options. reference_catalog: Path to a catalog of reference data. Returns: diff --git a/freanalysis/README.md b/freanalysis/README.md index be530d1..74d50fb 100644 --- a/freanalysis/README.md +++ b/freanalysis/README.md @@ -2,7 +2,7 @@ Package that can run GFDL model analysis plugins. ### Motivation -This crates a simple way for FRE to discover and run user generated analysis packages +This creates a simple way for FRE to discover and run user generated analysis packages to create figures to analyze the GFDL models. ### Requirements @@ -36,5 +36,5 @@ for name in plugins: # Run the plugins. You need to pass in a path to a data catalog, and a directory where # you want the figures to be created. for name in plugins: - figures = run_plugin(name, catalog, png_dir, reference_catalog=None) + figures = run_plugin(name, catalog, png_dir, config=None, reference_catalog=None) ``` diff --git a/freanalysis/freanalysis/plugins.py b/freanalysis/freanalysis/plugins.py index ceefd8c..cebf16a 100644 --- a/freanalysis/freanalysis/plugins.py +++ b/freanalysis/freanalysis/plugins.py @@ -58,16 +58,17 @@ def plugin_requirements(name): return _plugin_object(name).requires() -def run_plugin(name, catalog, png_dir, reference_catalog=None): +def run_plugin(name, catalog, png_dir, config=None, reference_catalog=None): """Runs the plugin's analysis. Args: name: Name of the plugin. catalog: Path to the data catalog. png_dir: Directory where the output figures will be stored. + config: Dictionary of configuration values. catalog: Path to the catalog of reference data. Returns: A list of png figure files that were created by the analysis. """ - return _plugin_object(name).run_analysis(catalog, png_dir, reference_catalog) + return _plugin_object(name).run_analysis(catalog, png_dir, config, reference_catalog) From b4ad6cbbf4ae0eb32c0260bde220002c97347beb Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Thu, 21 Nov 2024 14:03:48 -0500 Subject: [PATCH 02/15] update documentation --- figure_tools/README.md | 18 +++++++++++++++--- figure_tools/pyproject.toml | 1 - 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/figure_tools/README.md b/figure_tools/README.md index 8e8b399..b4efcc3 100644 --- a/figure_tools/README.md +++ b/figure_tools/README.md @@ -14,9 +14,21 @@ The only software packages that are required are: - numpy - xarray -### How to -Maps, heatmaps, and line plots can be made from the provided objects. These objects -can be instantiated directly from `xarray` datasets. For example: +### How to install this package +For now I'd recommend creating and installing this package in a virtual enviroment: + +```bash +$ python3 -m venv env +$ source env/bin/activate +$ pip install --upgrade pip +$ git clone https://github.com/NOAA-GFDL/analysis-scripts +$ cd analysis/scripts/figure_tools +$ pip install . +``` + +### Creating plots +Longitude-latitude maps, heatmaps, and line plots can be made from the provided +objects. These objects can be instantiated directly from `xarray` datasets. For example: ```python3 # Longitude-latitude map. diff --git a/figure_tools/pyproject.toml b/figure_tools/pyproject.toml index 3dc447d..5d674f2 100644 --- a/figure_tools/pyproject.toml +++ b/figure_tools/pyproject.toml @@ -11,7 +11,6 @@ dependencies = [ "cartopy", "matplotlib", "numpy", - "scipy", "xarray", ] requires-python = ">= 3.6" From ddfc5d710f7c9710ad156e590b2ea24a1cd22a3e Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Thu, 21 Nov 2024 14:05:17 -0500 Subject: [PATCH 03/15] fix typos --- figure_tools/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/figure_tools/README.md b/figure_tools/README.md index b4efcc3..5d015d6 100644 --- a/figure_tools/README.md +++ b/figure_tools/README.md @@ -21,8 +21,8 @@ For now I'd recommend creating and installing this package in a virtual envirome $ python3 -m venv env $ source env/bin/activate $ pip install --upgrade pip -$ git clone https://github.com/NOAA-GFDL/analysis-scripts -$ cd analysis/scripts/figure_tools +$ git clone https://github.com/NOAA-GFDL/analysis-scripts.git +$ cd analysis-scripts/figure_tools $ pip install . ``` From 73a088680c27b4ef320e5b8aea9d2ec41373307b Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Wed, 4 Dec 2024 11:48:28 -0500 Subject: [PATCH 04/15] update documentation --- README.md | 10 +++++----- freanalysis/freanalysis/plugins.py | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 260c8d3..f9cff0e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ scripts that produce figures and tables from GFDL model output. This work will by a simple web application to provide users an easy interface to interact with model output data. -### Requirements +### Repository Structure The code in this repository is broken up into components: - analysis-scripts - A very simple package that just defines an abstract base class that @@ -16,11 +16,11 @@ The code in this repository is broken up into components: for making common plots. - freanalysis - A package that is designed to be used by whatever workflow is responsible for running the analysis. -- freanalysis_aerosol - A plugin that creates aerosl mass figures. -- freanalysis_clouds - A plugin that creates cloud amount figures. -- freanalysis_radiation - A plugin that creates radiative flux figures. +- freanalysis_aerosol - An example plugin that creates aerosl mass figures. +- freanalysis_clouds - An example plugin that creates cloud amount figures. +- freanalysis_radiation - An example plugin that creates radiative flux figures. -### How to install everything +### Installing the basic components For now I'd recommend creating a virtual enviroment, and then installing each of the packages listed above: diff --git a/freanalysis/freanalysis/plugins.py b/freanalysis/freanalysis/plugins.py index cebf16a..2c21edc 100644 --- a/freanalysis/freanalysis/plugins.py +++ b/freanalysis/freanalysis/plugins.py @@ -26,8 +26,11 @@ def _plugin_object(name): ValueError if no object that inhertis from AnalysisScript is found in the plugin module. """ + # Loop through all attributes in the plugin package with the input name. for attribute in vars(discovered_plugins[name]).values(): + # Try to find a class that inherits from the AnalysisScript class. if inspect.isclass(attribute) and AnalysisScript in attribute.__bases__: + # Instantiate an object of this class. return attribute() raise ValueError(f"could not find compatible object in {name}.") From 5d2dd9a1835a56737ce982d678e0b048730b76ea Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Wed, 4 Dec 2024 12:16:33 -0500 Subject: [PATCH 05/15] removed plugins to get down to just the core stuff --- figure_tools/README.md | 43 --- figure_tools/figure_tools/__init__.py | 7 - .../figure_tools/anomaly_timeseries.py | 46 --- figure_tools/figure_tools/common_plots.py | 85 ----- figure_tools/figure_tools/figure.py | 130 -------- .../figure_tools/global_mean_timeseries.py | 48 --- figure_tools/figure_tools/lon_lat_map.py | 141 -------- figure_tools/figure_tools/time_subsets.py | 72 ---- figure_tools/figure_tools/zonal_mean_map.py | 86 ----- figure_tools/pyproject.toml | 30 -- freanalysis_aerosol/README.md | 16 - .../freanalysis_aerosol/__init__.py | 209 ------------ freanalysis_aerosol/pyproject.toml | 28 -- freanalysis_clouds/README.md | 16 - .../freanalysis_clouds/__init__.py | 122 ------- freanalysis_clouds/pyproject.toml | 28 -- freanalysis_land/README.md | 1 - freanalysis_land/freanalysis_land/__init__.py | 0 freanalysis_land/freanalysis_land/land.py | 243 -------------- freanalysis_land/pyproject.toml | 35 -- freanalysis_radiation/README.md | 16 - .../freanalysis_radiation/__init__.py | 309 ------------------ freanalysis_radiation/pyproject.toml | 28 -- 23 files changed, 1739 deletions(-) delete mode 100644 figure_tools/README.md delete mode 100644 figure_tools/figure_tools/__init__.py delete mode 100644 figure_tools/figure_tools/anomaly_timeseries.py delete mode 100644 figure_tools/figure_tools/common_plots.py delete mode 100644 figure_tools/figure_tools/figure.py delete mode 100644 figure_tools/figure_tools/global_mean_timeseries.py delete mode 100644 figure_tools/figure_tools/lon_lat_map.py delete mode 100644 figure_tools/figure_tools/time_subsets.py delete mode 100644 figure_tools/figure_tools/zonal_mean_map.py delete mode 100644 figure_tools/pyproject.toml delete mode 100644 freanalysis_aerosol/README.md delete mode 100644 freanalysis_aerosol/freanalysis_aerosol/__init__.py delete mode 100644 freanalysis_aerosol/pyproject.toml delete mode 100644 freanalysis_clouds/README.md delete mode 100644 freanalysis_clouds/freanalysis_clouds/__init__.py delete mode 100644 freanalysis_clouds/pyproject.toml delete mode 100644 freanalysis_land/README.md delete mode 100644 freanalysis_land/freanalysis_land/__init__.py delete mode 100644 freanalysis_land/freanalysis_land/land.py delete mode 100644 freanalysis_land/pyproject.toml delete mode 100644 freanalysis_radiation/README.md delete mode 100644 freanalysis_radiation/freanalysis_radiation/__init__.py delete mode 100644 freanalysis_radiation/pyproject.toml diff --git a/figure_tools/README.md b/figure_tools/README.md deleted file mode 100644 index 5d015d6..0000000 --- a/figure_tools/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# figure_tools -Tools to make common analysis figures. - -### Motivation -The goal of this project is to provide a simple API to guide the development of -scripts that produce figures from xarry datasets (such as those produced from climate -models). - -### Requirements -The only software packages that are required are: - -- cartopy -- matplotlib -- numpy -- xarray - -### How to install this package -For now I'd recommend creating and installing this package in a virtual enviroment: - -```bash -$ python3 -m venv env -$ source env/bin/activate -$ pip install --upgrade pip -$ git clone https://github.com/NOAA-GFDL/analysis-scripts.git -$ cd analysis-scripts/figure_tools -$ pip install . -``` - -### Creating plots -Longitude-latitude maps, heatmaps, and line plots can be made from the provided -objects. These objects can be instantiated directly from `xarray` datasets. For example: - -```python3 -# Longitude-latitude map. -from figure_tools import Figure, LonLatMap - - -map = LonLatMap.from_xarray_dataset(, , - time_method="annual mean", year=2010) -figure = Figure(title=) -figure.add_map(map_) -figure.save() -``` diff --git a/figure_tools/figure_tools/__init__.py b/figure_tools/figure_tools/__init__.py deleted file mode 100644 index 34f376b..0000000 --- a/figure_tools/figure_tools/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .anomaly_timeseries import AnomalyTimeSeries -from .common_plots import observation_vs_model_maps, radiation_decomposition, \ - timeseries_and_anomalies, zonal_mean_vertical_and_column_integrated_map -from .figure import Figure -from .global_mean_timeseries import GlobalMeanTimeSeries -from .lon_lat_map import LonLatMap -from .zonal_mean_map import ZonalMeanMap diff --git a/figure_tools/figure_tools/anomaly_timeseries.py b/figure_tools/figure_tools/anomaly_timeseries.py deleted file mode 100644 index 7422d48..0000000 --- a/figure_tools/figure_tools/anomaly_timeseries.py +++ /dev/null @@ -1,46 +0,0 @@ -from numpy import array, mean, transpose - -from .time_subsets import TimeSubset - - -class AnomalyTimeSeries(object): - def __init__(self, data, time_data, latitude, units): - self.data = data[...] - self.x_data = time_data[...] - self.y_data = latitude[...] - self.x_label = "Time" - self.y_label = "Latitude" - self.data_label = units - - @classmethod - def from_xarray_dataset(cls, dataset, variable): - """Instantiates an AnomalyTimeSeries object from an xarray dataset.""" - v = dataset.data_vars[variable] - axis_attrs = _dimension_order(dataset, v) - - time = TimeSubset(array(dataset.coords[v.dims[0]].data)) - latitude = array(dataset.coords[v.dims[-2]].data) - data = mean(array(v.data), axis=-1) # Average over longitude. - - time, data = time.annual_means(data) - average = mean(data, axis=0) # Average over longitude and time. - anomaly = data - for i in range(time.size): - anomaly[i, :] -= average[:] - anomaly = transpose(anomaly) - - return cls(anomaly, time, latitude, v.attrs["units"]) - - -def _dimension_order(dataset, variable): - """Raises a ValueError if the variable's dimensions are not in an expected order. - - Returns: - A list of the dimension axis attribute strings. - """ - axis_attrs = [dataset.coords[x].attrs["axis"].lower() - if "axis" in dataset.coords[x].attrs else None - for x in variable.dims] - if axis_attrs == ["t", "y", "x"]: - return axis_attrs - raise ValueError(f"variable {variable} contains unexpected axes ordering {axis_attrs}.") diff --git a/figure_tools/figure_tools/common_plots.py b/figure_tools/figure_tools/common_plots.py deleted file mode 100644 index b5e27ee..0000000 --- a/figure_tools/figure_tools/common_plots.py +++ /dev/null @@ -1,85 +0,0 @@ -from math import ceil, floor - -from numpy import abs, max, min, percentile - -from .figure import Figure - - -def observation_vs_model_maps(reference, model, title): - figure = Figure(num_rows=2, num_columns=2, title=title, size=(14, 12)) - - # Create common color bar for reference and model. - reference_range = [floor(min(reference.data)), ceil(max(reference.data))] - model_range = [floor(min(model.data)), ceil(max(model.data))] - colorbar_range = [None, None] - colorbar_range[0] = reference_range[0] if reference_range[0] < model_range[0] \ - else model_range[0] - colorbar_range[1] = reference_range[1] if reference_range[1] > model_range[1] \ - else model_range[1] - - # Reference data. - global_mean = reference.global_mean() - figure.add_map(reference, f"Observations [Mean: {global_mean:.2f}]", 1, - colorbar_range=colorbar_range) - - # Model data. - global_mean = model.global_mean() - figure.add_map(model, f"Model [Mean: {global_mean:.2f}]", 2, - colorbar_range=colorbar_range) - - # Difference between the reference and model. - difference = reference - model - color_range = _symmetric_colorbar_range(difference.data) - global_mean = difference.global_mean() - figure.add_map(difference, f"Obs - Model [Mean: {global_mean:.2f}]", 3, - colorbar_range=color_range, - normalize_colors=True) - - # Use percentiles. - zoom = int(ceil(percentile(abs(difference.data), 95))) - figure.add_map(difference, f"Obs - Model [Mean: {global_mean:.2f}]", 4, - colorbar_range=[-1*zoom, zoom], num_levels=19, - normalize_colors=True) - return figure - - -def radiation_decomposition(clean_clear_sky, clean_sky, clear_sky, all_sky, title): - figure = Figure(num_rows=2, num_columns=2, title=title, size=(16, 10)) - maps = [clean_clear_sky, clean_sky - clean_clear_sky, all_sky - clean_sky, all_sky] - titles = ["Clean-clear Sky", "Cloud Effects", "Aerosol Effects", "All Sky"] - for i, (map_, title) in enumerate(zip(maps, titles)): - global_mean = map_.global_mean() - updated_title = f"{title} [Mean: {global_mean:.2f}]" - if title in ["Cloud Effects", "Aerosol Effects"]: - figure.add_map(map_, updated_title, i + 1, - colorbar_range=_symmetric_colorbar_range(map_.data), - normalize_colors=True) - else: - figure.add_map(map_, updated_title, i + 1, - normalize_colors=True, colorbar_center=global_mean) - return figure - - -def timeseries_and_anomalies(timeseries, map_, title): - figure = Figure(num_rows=1, num_columns=2, title=title, size=(16, 10)) - figure.add_line_plot(timeseries, "Timeseries", 1) - figure.add_map(map_, "Zonal Mean Anomalies", 2, - colorbar_range=_symmetric_colorbar_range(map_.data), - normalize_colors=True) - return figure - - -def zonal_mean_vertical_and_column_integrated_map(zonal_mean, lon_lat, title): - figure = Figure(num_rows=1, num_columns=2, title=title, size=(16, 10)) - figure.add_map(zonal_mean, "Zonal Mean Vertical Profile", 1) - figure.add_map(lon_lat, "Column-integrated", 2) - return figure - - -def _symmetric_colorbar_range(data): - colorbar_range = [int(floor(min(data))), int(ceil(max(data)))] - if abs(colorbar_range[0]) > abs(colorbar_range[1]): - colorbar_range[1] = -1*colorbar_range[0] - else: - colorbar_range[0] = -1*colorbar_range[1] - return colorbar_range diff --git a/figure_tools/figure_tools/figure.py b/figure_tools/figure_tools/figure.py deleted file mode 100644 index f79a99b..0000000 --- a/figure_tools/figure_tools/figure.py +++ /dev/null @@ -1,130 +0,0 @@ -from math import ceil - -import cartopy.crs as ccrs -import matplotlib as mpl -import matplotlib.colors as colors -import matplotlib.pyplot as plt -from numpy import linspace, max, min, unravel_index - -from .lon_lat_map import LonLatMap -from .zonal_mean_map import ZonalMeanMap - - -class Figure(object): - def __init__(self, num_rows=1, num_columns=1, size=(16, 12), title=None): - """Creates a figure for the input number of plots. - - Args: - num_rows: Number of rows of plots. - num_columns: Number of columns of plots. - """ - self.figure = plt.figure(figsize=size, layout="compressed") - if title is not None: - self.figure.suptitle(title.title()) - self.num_rows = num_rows - self.num_columns = num_columns - self.plot = [[None for y in range(num_columns)] for x in range(num_rows)] - - def add_map(self, map_, title, position=1, colorbar_range=None, colormap="coolwarm", - normalize_colors=False, colorbar_center=0, num_levels=51, extend=None): - """Adds a map to the figure. - - Args: - map_: LonLatMap or ZonalMeanMap object. - total: String title for the plot. - position: Integer position index for the plot in the figure. - colorbar_range: List of integers describing the colorbar limits. - """ - # Create the plotting axes. - optional_args = {} - if isinstance(map_, LonLatMap): - optional_args["projection"] = map_.projection - plot = self.figure.add_subplot(self.num_rows, self.num_columns, - position, **optional_args) - - # Set the colorbar properties. - if colorbar_range == None: - levels = num_levels - else: - # There seems to be some strange behavior if the number of level is too - # big for a given range. - levels = linspace(colorbar_range[0], colorbar_range[-1], num_levels, endpoint=True) - if extend == None: - data_max = max(map_.data) - data_min = min(map_.data) - if data_max > colorbar_range[1] and data_min < colorbar_range[0]: - extend = "both" - elif data_max > colorbar_range[1]: - extend = "max" - elif data_min < colorbar_range[0]: - extend = "min" - if normalize_colors: - norm = colors.CenteredNorm(vcenter=colorbar_center) - else: - norm = None - - # Make the map. - optional_args = {"levels": levels, "cmap": colormap, "norm": norm, "extend": extend} - if isinstance(map_, LonLatMap): - optional_args["transform"] = ccrs.PlateCarree() - cs = plot.contourf(map_.x_data, map_.y_data, map_.data, **optional_args) - - # Set the metadata. - self.figure.colorbar(cs, ax=plot, label=map_.data_label) - if isinstance(map_, LonLatMap): - plot.coastlines() - grid = plot.gridlines(draw_labels=True, dms=True, x_inline=False, y_inline=False) - grid.bottom_labels = False - grid.top_labels = False - if isinstance(map_, ZonalMeanMap) and map_.invert_y_axis: - plot.invert_yaxis() - plot.set_title(title.replace("_", " ").title()) - plot.set_xlabel(map_.x_label) - plot.set_ylabel(map_.y_label) - - # Add date information if necessary. - if hasattr(map_, "timestamp") and map_.timestamp != None: - plot.text(0.85, 1, map_.timestamp, transform=plot.transAxes) - - # Store the plot in the figure object. - x, y = self._plot_position_to_indices(position) - self.plot[x][y] = plot - - def add_line_plot(self, line_plot, title, position=1): - """Adds a line plot to the figure. - - Args: - map_: LonLatMap or ZonalMeanMap object. - total: String title for the plot. - position: Integer position index for the plot in the figure. - """ - plot = self.figure.add_subplot(self.num_rows, self.num_columns, position) - plot.plot(line_plot.x_data, line_plot.data) -# if isinstance(line_plot, GlobalMeanVerticalPlot) and line_plot.invert_y_axis: -# plot.invert_yaxis() - plot.set_title(title) - plot.set_xlabel(line_plot.x_label) - plot.set_ylabel(line_plot.y_label) - - # Store the plot in the figure object. - x, y = self._plot_position_to_indices(position) - self.plot[x][y] = plot - - def display(self): - """Shows the figure in a new window.""" - plt.show() - - def save(self, path): - plt.savefig(path) - plt.clf() - - def _plot_position_to_indices(self, position): - """Converts from a plot position to its x and y indices. - - Args: - position: Plot position (from 1 to num_rows*num_columns). - - Returns: - The x and y indices for the plot. - """ - return unravel_index(position - 1, (self.num_rows, self.num_columns)) diff --git a/figure_tools/figure_tools/global_mean_timeseries.py b/figure_tools/figure_tools/global_mean_timeseries.py deleted file mode 100644 index 68b2f94..0000000 --- a/figure_tools/figure_tools/global_mean_timeseries.py +++ /dev/null @@ -1,48 +0,0 @@ -from numpy import array, cos, mean, pi, sum - -from .time_subsets import TimeSubset - - -class GlobalMeanTimeSeries(object): - def __init__(self, data, time_data, units): - self.data = data[...] - self.x_data = time_data[...] - self.x_label = "Time" - self.y_label = units - - @classmethod - def from_xarray_dataset(cls, dataset, variable): - """Instantiates an AnomalyTimeSeries object from an xarray dataset.""" - v = dataset.data_vars[variable] - axis_attrs = _dimension_order(dataset, v) - - time = TimeSubset(array(dataset.coords[v.dims[0]].data)) - latitude = array(dataset.coords[v.dims[-2]].data) - time, data = time.annual_means(v.data) - data = _global_mean(data, latitude) - - return cls(data, time, v.attrs["units"]) - - -def _dimension_order(dataset, variable): - """Raises a ValueError if the variable's dimensions are not in an expected order. - - Returns: - A list of the dimension axis attribute strings. - """ - axis_attrs = [dataset.coords[x].attrs["axis"].lower() - if "axis" in dataset.coords[x].attrs else None - for x in variable.dims] - if axis_attrs == ["t", "y", "x"]: - return axis_attrs - raise ValueError(f"variable {variable} contains unexpected axes ordering {axis_attrs}.") - - -def _global_mean(data, latitude): - """Performs a global mean over the longitude and latitude dimensions. - - Returns: - Gobal mean value. - """ - weights = cos(2.*pi*latitude/360.) - return sum(mean(data, axis=-1)*weights, axis=-1)/sum(weights) diff --git a/figure_tools/figure_tools/lon_lat_map.py b/figure_tools/figure_tools/lon_lat_map.py deleted file mode 100644 index 382fc4b..0000000 --- a/figure_tools/figure_tools/lon_lat_map.py +++ /dev/null @@ -1,141 +0,0 @@ -import cartopy.crs as ccrs -from cartopy.util import add_cyclic -from numpy import array, array_equal, cos, mean, ndarray, pi, sum -from xarray import DataArray - -from .time_subsets import TimeSubset - - -class LonLatMap(object): - """Longitude-latitude data map. - - Attributes: - coastlines: Flag that determines if coastlines are drawn on the map. - data: numpy array of data values. - data_label: String units for the colorbar. - projection: Cartopy map projection to use. - x_data: numpy array of data values for the x-axis. - xlabel: String label for the x-axis ("Longitude") - x_data: numpy array of data values for the y-axis. - ylabel: String label for the y-axis ("Latitude") - """ - def __init__(self, data, longitude, latitude, units=None, - projection=ccrs.Mollweide(), coastlines=True, add_cyclic_point=True, - timestamp=None): - if add_cyclic_point: - self.data, self.x_data, self.y_data = add_cyclic(data[...], longitude[...], - latitude[...]) - else: - self.data = data[...] - self.x_data = longitude[...] - self.y_data = latitude[...] - self.projection = projection - self.coastlines = coastlines - self.x_label = "Longitude" - self.y_label = "Latitude" - self.data_label = units - self.timestamp = timestamp - - def __add__(self, arg): - """Allows LonLatMap objects to be added together.""" - self._compatible(arg) - return LonLatMap(self.data + arg.data, self.x_data, self.y_data, - units=self.data_label, projection=self.projection, - coastlines=self.coastlines, add_cyclic_point=False, - timestamp=self.timestamp) - - def __sub__(self, arg): - """Allows LonLatMap objects to be subtracted from one another.""" - self._compatible(arg) - return LonLatMap(self.data - arg.data, self.x_data, self.y_data, - units=self.data_label, projection=self.projection, - coastlines=self.coastlines, add_cyclic_point=False, - timestamp=self.timestamp) - - @classmethod - def from_xarray_dataset(cls, dataset, variable, time_method=None, time_index=None, - year=None): - """Instantiates a LonLatMap object from an xarray dataset.""" - v = dataset.data_vars[variable] - data = array(v.data[...]) - axis_attrs = _dimension_order(dataset, v) - longitude = array(dataset.coords[v.dims[-1]].data[...]) - latitude = array(dataset.coords[v.dims[-2]].data[...]) - - if axis_attrs[0] == "t": - time = array(dataset.coords[v.dims[0]].data[...]) - if time_method == "instantaneous": - if time_method == None: - raise ValueError("time_index is required when time_method='instantaneous.'") - data = data[time_index, ...] - timestamp = f"@ {str(time[time_index])}" - elif time_method == "annual mean": - if year == None: - raise ValueError("year is required when time_method='annual mean'.") - time = TimeSubset(time) - data = time.annual_mean(data, year) - timestamp = r"$\bar{t} = $" + str(year) - else: - raise ValueError("time_method must be either 'instantaneous' or 'annual mean.'") - else: - timestamp = None - - return cls(data, longitude, latitude, units=v.attrs["units"], timestamp=timestamp) - - def global_mean(self): - """Performs a global mean over the longitude and latitude dimensions. - - Returns: - Gobal mean value. - """ - - weights = cos(2.*pi*self.y_data/360.) - return sum(mean(self.data, axis=-1)*weights, axis=-1)/sum(weights) - - def regrid_to_map(self, map_): - """Regrid the data to match in the input map. - - Args: - map_: A LonLatMap to regrid to. - """ - if not isinstance(map_, LonLatMap): - raise TypeError("input map must be a LonLatMap.") - try: - self._compatible(map_) - return - except ValueError: - da = DataArray(self.data, dims=["y", "x"], - coords={"x": self.x_data, "y": self.y_data}) - da2 = DataArray(map_.data, dims=["y", "x"], - coords={"x": map_.x_data, "y": map_.y_data}) - self.data = array(da.interp_like(da2, kwargs={"fill_value": "extrapolate"})) - self.x_data = map_.x_data - self.y_data = map_.y_data - - def _compatible(self, arg): - """Raises a ValueError if two objects are not compatible.""" - if not isinstance(arg, LonLatMap): - raise TypeError("input map must be a LonLatMap.") - for attr in ["x_data", "y_data", "projection", "data_label"]: - if isinstance(getattr(self, attr), ndarray): - equal = array_equal(getattr(self, attr), getattr(arg, attr)) - else: - equal = getattr(self, attr) == getattr(arg, attr) - if not equal: - raise ValueError(f"The same {attr} is required for both objects.") - - -def _dimension_order(dataset, variable): - """Raises a ValueError if the variable's dimensoins are not in an expected order. - - Returns: - A list of the dimension axis attribute strings. - """ - axis_attrs = [dataset.coords[x].attrs["axis"].lower() - if "axis" in dataset.coords[x].attrs else None - for x in variable.dims] - allowed = [x + ["y", "x"] for x in [["t",], ["z",], [None,], []]] - for config in allowed: - if axis_attrs == config: - return axis_attrs - raise ValueError(f"variable {variable} contains unexpected axes ordering {axis_attrs}.") diff --git a/figure_tools/figure_tools/time_subsets.py b/figure_tools/figure_tools/time_subsets.py deleted file mode 100644 index a5c89ad..0000000 --- a/figure_tools/figure_tools/time_subsets.py +++ /dev/null @@ -1,72 +0,0 @@ -from numpy import array, datetime64, mean, zeros - - -class TimeSubset(object): - def __init__(self, data): - """Instantiates an object. - - Args: - data: An xarray DataArray for the time dimension of an xarray Dataset. - """ - self.data = data - - def annual_mean(self, data, year): - """Calculates the annual mean of the input date for the input year. - - Args: - data: Numpy array of data to be averaged. - year: Integer year to average over. - """ - start, end = None, None - for i, point in enumerate(self.data): - month, y = self._month_and_year(point) - if y == year: - if month == 1: - start = i - elif month == 12: - end = i + 1 - if None not in [start, end]: break - else: - raise ValueError(f"could not find year {year}.") - return mean(array(data[start:end, ...]), axis=0) - - def annual_means(self, data): - """Calculates the annual means of the input date for each year. - - Args: - data: Numpy array of data to be averaged. - - Returns: - Numpy array of years that were averaged over and a numpy array of the - average data. - """ - years = {} - for i, point in enumerate(self.data): - month, year = self._month_and_year(point) - if year not in years: - years[year] = [None, None] - if month == 1: - years[year][0] = i - elif month == 12: - years[year][1] = i + 1 - - years_data = sorted(years.keys()) - means_data = zeros(tuple([len(years_data),] + list(data.shape[1:]))) - for i, key in enumerate(years_data): - start, end = years[key] - means_data[i, ...] = mean(array(data[start:end, ...]), axis=0) - return array(years_data), means_data - - def _month_and_year(self, time): - """Returns the integer month and year for the input time point. - - Args: - Integer month and year values. - """ - if isinstance(time, datetime64): - year = time.astype("datetime64[Y]").astype(int) + 1970 - month = time.astype("datetime64[M]").astype(int) % 12 + 1 - else: - year = time.year - month = time.month - return month, year diff --git a/figure_tools/figure_tools/zonal_mean_map.py b/figure_tools/figure_tools/zonal_mean_map.py deleted file mode 100644 index 29af2b1..0000000 --- a/figure_tools/figure_tools/zonal_mean_map.py +++ /dev/null @@ -1,86 +0,0 @@ -from numpy import array, array_equal, mean, ndarray - -from .time_subsets import TimeSubset - - -class ZonalMeanMap(object): - def __init__(self, data, latitude, y_axis_data, units=None, y_label=None, - invert_y_axis=False, timestamp=None): - self.data = data[...] - self.x_data = latitude[...] - self.y_data = y_axis_data[...] - self.invert_y_axis = invert_y_axis - self.x_label = "Latitude" - self.y_label = y_label - self.data_label = units - self.timestamp = timestamp - - def __add__(self, arg): - self._compatible(arg) - return ZonalMeanMap(self.data + arg.data, self.x_data, self.y_data, - units=self.data_label, y_label=self.y_label, - invert_y_axis=self.invert_y_axis, timestamp=self.timestamp) - - def __sub__(self, arg): - self._compatible(arg) - return ZonalMeanMap(self.data - arg.data, self.x_data, self.y_data, - units=self.data_label, y_label=self.y_label, - invert_y_axis=self.invert_y_axis, timestamp=self.timestamp) - - @classmethod - def from_xarray_dataset(cls, dataset, variable, time_method=None, time_index=None, - year=None, y_axis=None, y_label=None, invert_y_axis=False): - """Instantiates a ZonalMeanMap object from an xarray dataset.""" - v = dataset.data_vars[variable] - data = array(v.data[...]) - axis_attrs = _dimension_order(dataset, v) - latitude = array(dataset.coords[v.dims[-2]].data[...]) - y_dim = array(dataset.coords[v.dims[-3]].data[...]) - y_dim_units = y_label or dataset.coords[v.dims[-3]].attrs["units"] - - if axis_attrs[0] == "t": - time = array(dataset.coords[v.dims[0]].data[...]) - if time_method == "instantaneous": - if time_method == None: - raise ValueError("time_index is required when time_method='instantaneous.'") - data = data[time_index, ...] - timestamp = str(time[time_index]) - elif time_method == "annual mean": - if year == None: - raise ValueError("year is required when time_method='annual mean'.") - time = TimeSubset(time) - data = time.annual_mean(data, year) - timestamp = str(year) - else: - raise ValueError("time_method must be either 'instantaneous' or 'annual mean'.") - else: - timestamp = None - - return cls(mean(data, -1), latitude, y_dim, v.attrs["units"], y_dim_units, - invert_y_axis, timestamp) - - def _compatible(self, arg): - """Raises a ValueError if two objects are not compatible.""" - for attr in ["x_data", "y_data", "invert_y_axis", "y_label", "data_label"]: - if isinstance(getattr(self, attr), ndarray): - equal = array_equal(getattr(self, attr), getattr(arg, attr)) - else: - equal = getattr(self, attr) == getattr(arg, attr) - if not equal: - raise ValueError(f"The same {attr} is required for both objects.") - - -def _dimension_order(dataset, variable): - """Raises a ValueError if the variable's dimensions are not in an expected order. - - Returns: - A list of the dimension axis attribute strings. - """ - axis_attrs = [dataset.coords[x].attrs["axis"].lower() - if "axis" in dataset.coords[x].attrs else None - for x in variable.dims] - allowed = [x + ["y", "x"] for x in [["z",], [None,], ["t", "z"], ["t", None]]] - for config in allowed: - if axis_attrs == config: - return axis_attrs - raise ValueError(f"variable contains unexpected axes ordering {axis_attrs}.") diff --git a/figure_tools/pyproject.toml b/figure_tools/pyproject.toml deleted file mode 100644 index 5d674f2..0000000 --- a/figure_tools/pyproject.toml +++ /dev/null @@ -1,30 +0,0 @@ -[build-system] -requires = [ - "setuptools >= 40.9.0", -] -build-backend = "setuptools.build_meta" - -[project] -name = "figure_tools" -version = "0.1" -dependencies = [ - "cartopy", - "matplotlib", - "numpy", - "xarray", -] -requires-python = ">= 3.6" -authors = [ - {name = "developers"}, -] -maintainers = [ - {name = "developer", email = "developer-email@address.domain"}, -] -description = "Helper tools to make common plots" -readme = "README.md" -classifiers = [ - "Programming Language :: Python" -] - -[project.urls] -repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/freanalysis_aerosol/README.md b/freanalysis_aerosol/README.md deleted file mode 100644 index 6c0811e..0000000 --- a/freanalysis_aerosol/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# freanalysis_aerosol -A plugin for analyzing GFDL model aerosol mass output. - -### Motivation -To create figures to analyze the aerosol mass output from GFDL models. - -### Requirements -The software packages that are required are: - -- analysis-scripts -- figure_tools -- intake -- intake-esm - -### How to use this plugin -This plugin is designed to be used by the freanalysis package. diff --git a/freanalysis_aerosol/freanalysis_aerosol/__init__.py b/freanalysis_aerosol/freanalysis_aerosol/__init__.py deleted file mode 100644 index 613ed76..0000000 --- a/freanalysis_aerosol/freanalysis_aerosol/__init__.py +++ /dev/null @@ -1,209 +0,0 @@ -from dataclasses import dataclass -import json -from pathlib import Path - -from analysis_scripts import AnalysisScript -from figure_tools import LonLatMap, zonal_mean_vertical_and_column_integrated_map, \ - ZonalMeanMap -import intake - - -@dataclass -class Metadata: - """Helper class that stores the metadata needed by the plugin.""" - frequency: str = "monthly" - realm: str = "atmos" - - @staticmethod - def variables(): - """Helper function to make maintaining this script easier if the - catalog variable ids change. - - Returns: - Dictionary mapping the names used in this script to the catalog - variable ids. - """ - return { - "black_carbon": "blk_crb", - "black_carbon_column": "blk_crb_col", - "large_dust": "lg_dust", - "large_dust_column": "lg_dust_col", - "small_dust": "sm_dust", - "small_dust_column": "sm_dust_col", - "organic_carbon": "org_crb", - "organic_carbon_column": "org_crb_col", - "large_seasalt": "lg_ssalt", - "large_seasalt_column": "lg_ssalt_col", - "small_seasalt": "sm_ssalt", - "small_seasalt_column": "sm_ssalt_col", - "seasalt": "salt", - "seasalt_column": "salt_col", - "sulfate": "sulfate", - "sulfate_column": "sulfate_col", - } - - -class AerosolAnalysisScript(AnalysisScript): - """Aerosol analysis script. - - Attributes: - description: Longer form description for the analysis. - title: Title that describes the analysis. - """ - def __init__(self): - self.metadata = Metadata() - self.description = "Calculates aerosol mass metrics." - self.title = "Aerosol Masses" - - def requires(self): - """Provides metadata describing what is needed for this analysis to run. - - Returns: - A json string containing the metadata. - """ - columns = Metadata.__annotations__.keys() - settings = {x: getattr(self.metadata, x) for x in columns} - return json.dumps({ - "settings": settings, - "dimensions": { - "lat": {"standard_name": "latitude"}, - "lon": {"standard_name": "longitude"}, - "pfull": {"standard_name": "air_pressure"}, - "time": {"standard_name": "time"} - }, - "varlist": { - "blk_crb": { - "standard_name": "black_carbon_mass", - "units": "kg m-3", - "dimensions": ["time", "pfull", "lat", "lon"] - }, - "blk_crb_col": { - "standard_name": "column_integrated_black_carbon_mass", - "units": "kg m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lg_dust": { - "standard_name": "large_dust_mass", - "units": "kg m-3", - "dimensions": ["time", "pfull", "lat", "lon"] - }, - "lg_dust_col": { - "standard_name": "column_integrated_large_dust_mass", - "units": "kg m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lg_ssalt": { - "standard_name": "large_seasalt_mass", - "units": "kg m-3", - "dimensions": ["time", "pfull", "lat", "lon"] - }, - "lg_ssalt_col": { - "standard_name": "column_integrated_large_ssalt_mass", - "units": "kg m-2", - "dimensions": ["time", "lat", "lon"] - }, - "org_crb": { - "standard_name": "organic_carbon_mass", - "units": "kg m-3", - "dimensions": ["time", "pfull", "lat", "lon"] - }, - "org_crb_col": { - "standard_name": "column_integrated_organic_carbon_mass", - "units": "kg m-2", - "dimensions": ["time", "lat", "lon"] - }, - "salt": { - "standard_name": "seasalt_mass", - "units": "kg m-3", - "dimensions": ["time", "pfull", "lat", "lon"] - }, - "salt_col": { - "standard_name": "column_integrated_seasalt_mass", - "units": "kg m-2", - "dimensions": ["time", "lat", "lon"] - }, - "sm_dust": { - "standard_name": "small_dust_mass", - "units": "kg m-3", - "dimensions": ["time", "pfull", "lat", "lon"] - }, - "sm_dust_col": { - "standard_name": "column_integrated_small_dust_mass", - "units": "kg m-2", - "dimensions": ["time", "lat", "lon"] - }, - "sm_ssalt": { - "standard_name": "small_seasalt_mass", - "units": "kg m-3", - "dimensions": ["time", "pfull", "lat", "lon"] - }, - "sm_ssalt_col": { - "standard_name": "column_integrated_small_ssalt_mass", - "units": "kg m-2", - "dimensions": ["time", "lat", "lon"] - }, - "sulfate": { - "standard_name": "sulfate_mass", - "units": "kg m-3", - "dimensions": ["time", "pfull", "lat", "lon"] - }, - "sulfate_col": { - "standard_name": "column_integrated_sulfate_mass", - "units": "kg m-2", - "dimensions": ["time", "lat", "lon"] - }, - }, - }) - - def run_analysis(self, catalog, png_dir, reference_catalog=None, config={}): - """Runs the analysis and generates all plots and associated datasets. - - Args: - catalog: Path to a catalog. - png_dir: Path to the directory where the figures will be made. - reference_catalog: Path to a catalog of reference data. - config: Dictionary of catalog metadata. Will overwrite the - data defined in the Metadata helper class if they both - contain the same keys. - - Returns: - A list of paths to the figures that were created. - - Raises: - ValueError if the catalog cannot be filtered correctly. - """ - - # Connect to the catalog and find the necessary datasets. - catalog = intake.open_esm_datastore(catalog) - - maps = {} - for name, variable in self.metadata.variables().items(): - # Filter the catalog down to a single dataset for each variable. - query_params = {"variable_id": variable} - query_params.update(vars(self.metadata)) - query_params.update(config) - datasets = catalog.search(**query_params).to_dataset_dict(progressbar=False) - if len(list(datasets.values())) != 1: - raise ValueError("could not filter the catalog down to a single dataset.") - dataset = list(datasets.values())[0] - - if name.endswith("column"): - # Lon-lat maps. - maps[name] = LonLatMap.from_xarray_dataset(dataset, variable, year=1980, - time_method="annual mean") - else: - maps[name] = ZonalMeanMap.from_xarray_dataset(dataset, variable, year=1980, - time_method="annual mean", - invert_y_axis=True) - - figure_paths = [] - for name in self.metadata.variables().keys(): - if name.endswith("column"): continue - figure = zonal_mean_vertical_and_column_integrated_map( - maps[name], - maps[f"{name}_column"], - f"{name.replace('_', ' ')} Mass", - ) - figure.save(Path(png_dir) / f"{name}.png") - figure_paths.append(Path(png_dir)/ f"{name}.png") - return figure_paths \ No newline at end of file diff --git a/freanalysis_aerosol/pyproject.toml b/freanalysis_aerosol/pyproject.toml deleted file mode 100644 index 55c8be8..0000000 --- a/freanalysis_aerosol/pyproject.toml +++ /dev/null @@ -1,28 +0,0 @@ -[build-system] -requires = [ - "setuptools >= 40.9.0", -] -build-backend = "setuptools.build_meta" - -[project] -name = "freanalysis_aerosol" -version = "0.1" -dependencies = [ - "intake", - "intake-esm", -] -requires-python = ">= 3.6" -authors = [ - {name = "developers"}, -] -maintainers = [ - {name = "developer", email = "developer-email@address.domain"}, -] -description = "Aerosol mass analyzer for GFDL model output" -readme = "README.md" -classifiers = [ - "Programming Language :: Python" -] - -[project.urls] -repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/freanalysis_clouds/README.md b/freanalysis_clouds/README.md deleted file mode 100644 index 2003180..0000000 --- a/freanalysis_clouds/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# freanalysis_clouds -A plugin for analyzing GFDL model clouds output. - -### Motivation -To create figures to analyze the clouds output from GFDL models. - -### Requirements -The software packages that are required are: - -- analysis-scripts -- figure_tools -- intake -- intake-esm - -### How to use this plugin -This plugin is designed to be used by the freanalysis package. diff --git a/freanalysis_clouds/freanalysis_clouds/__init__.py b/freanalysis_clouds/freanalysis_clouds/__init__.py deleted file mode 100644 index 4b15c52..0000000 --- a/freanalysis_clouds/freanalysis_clouds/__init__.py +++ /dev/null @@ -1,122 +0,0 @@ -from dataclasses import dataclass -import json -from pathlib import Path - -from analysis_scripts import AnalysisScript -from figure_tools import Figure, LonLatMap -import intake - - -@dataclass -class Metadata: - """Helper class that stores the metadata needed by the plugin.""" - frequency: str = "mon" - realm: str = "atmos" - - @staticmethod - def variables(): - """Helper function to make maintaining this script easier if the - catalog variable ids change. - - Returns: - Dictionary mapping the names used in this script to the catalog - variable ids. - """ - return { - "high_cloud_fraction": "high_cld_amt", - "low_cloud_fraction": "low_cld_amt", - "middle_cloud_fraction": "mid_cld_amt", - } - - -class CloudAnalysisScript(AnalysisScript): - """Cloud analysis script. - - Attributes: - description: Longer form description for the analysis. - title: Title that describes the analysis. - """ - def __init__(self): - self.metadata = Metadata() - self.description = "Calculates cloud metrics." - self.title = "Cloud Fractions" - - def requires(self): - """Provides metadata describing what is needed for this analysis to run. - - Returns: - A json string containing the metadata. - """ - columns = Metadata.__annotations__.keys() - settings = {x: getattr(self.metadata, x) for x in columns} - return json.dumps({ - "settings": settings, - "dimensions": { - "lat": {"standard_name": "latitude"}, - "lon": {"standard_name": "longitude"}, - "time": {"standard_name": "time"} - }, - "varlist": { - "high_cld_amt": { - "standard_name": "high_cloud_fraction", - "units": "%", - "dimensions": ["time", "lat", "lon"] - }, - "low_cld_amt": { - "standard_name": "low_cloud_fraction", - "units": "%", - "dimensions": ["time", "lat", "lon"] - }, - "mid_cld_amt": { - "standard_name": "middle_cloud_fraction", - "units": "%", - "dimensions": ["time", "lat", "lon"] - }, - }, - }) - - def run_analysis(self, catalog, png_dir, reference_catalog=None, config={}): - """Runs the analysis and generates all plots and associated datasets. - - Args: - catalog: Path to a catalog. - png_dir: Path to the directory where the figures will be made. - reference_catalog: Path to a catalog of reference data. - config: Dictonary of catalog metadata. Will overwrite the - data defined in the Metadata helper class if they both - contain the same keys. - - Returns: - A list of paths to the figures that were created. - - Raises: - ValueError if the catalog cannot be filtered correctly. - """ - - # Connect to the catalog. - catalog = intake.open_esm_datastore(catalog) - print(catalog) - - maps = {} - for name, variable in self.metadata.variables().items(): - # Filter the catalog down to a single dataset for each variable. - query_params = {"variable_id": variable} - query_params.update(vars(self.metadata)) - query_params.update(config) - datasets = catalog.search(**query_params).to_dataset_dict(progressbar=False) - if len(list(datasets.values())) != 1: - raise ValueError("could not filter the catalog down to a single dataset.", datasets) - dataset = list(datasets.values())[0] - - # Create Lon-lat maps. - maps[name] = LonLatMap.from_xarray_dataset(dataset, variable, year=1980, - time_method="annual mean") - - # Create the figure. - figure = Figure(num_rows=3, num_columns=1, title="Cloud Fraction", size=(16, 10)) - figure.add_map(maps["high_cloud_fraction"], "High Clouds", 1, colorbar_range= [0, 100]) - figure.add_map(maps["middle_cloud_fraction"], "Middle Clouds", 2, colorbar_range=[0, 100]) - figure.add_map(maps["low_cloud_fraction"], "Low Clouds", 3, colorbar_range=[0, 100]) - output = Path(png_dir) / "cloud-fraction.png" - figure.save(output) - return [output,] diff --git a/freanalysis_clouds/pyproject.toml b/freanalysis_clouds/pyproject.toml deleted file mode 100644 index 634b0ac..0000000 --- a/freanalysis_clouds/pyproject.toml +++ /dev/null @@ -1,28 +0,0 @@ -[build-system] -requires = [ - "setuptools >= 40.9.0", -] -build-backend = "setuptools.build_meta" - -[project] -name = "freanalysis_clouds" -version = "0.1" -dependencies = [ - "intake", - "intake-esm", -] -requires-python = ">= 3.6" -authors = [ - {name = "developers"}, -] -maintainers = [ - {name = "developer", email = "developer-email@address.domain"}, -] -description = "Cloud fraction analyzer for GFDL model output" -readme = "README.md" -classifiers = [ - "Programming Language :: Python" -] - -[project.urls] -repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/freanalysis_land/README.md b/freanalysis_land/README.md deleted file mode 100644 index ca4e23a..0000000 --- a/freanalysis_land/README.md +++ /dev/null @@ -1 +0,0 @@ -This is the README. \ No newline at end of file diff --git a/freanalysis_land/freanalysis_land/__init__.py b/freanalysis_land/freanalysis_land/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/freanalysis_land/freanalysis_land/land.py b/freanalysis_land/freanalysis_land/land.py deleted file mode 100644 index 0bd3f0a..0000000 --- a/freanalysis_land/freanalysis_land/land.py +++ /dev/null @@ -1,243 +0,0 @@ -import json -from analysis_scripts import AnalysisScript -import intake -import matplotlib.pyplot as plt -import cartopy -import cartopy.crs as ccrs -import datetime -import pandas as pd -import re -import xarray as xr - - -class LandAnalysisScript(AnalysisScript): - """A class for performing various analysis tasks relating to GFDL land model output, - inherits from the AnalysisScipt base class. - - Attributes: - description: Longer form description for the analysis. - title: Title that describes the analysis. - """ - def __init__(self): - """Instantiates an object. The user should provide a description and title.""" - self.description = "This is for analysis of land model (stand-alone)" - self.title = "Soil Carbon" - - def requires(self): - """Provides metadata describing what is needed for this analysis to run. - - Returns: - A json string describing the metadata. - """ - raise NotImplementedError("you must override this function.") - return json.dumps("{json of metadata MDTF format.}") - - def global_map(self,dataset,var,dates,plt_time=None,colormap='viridis',title=''): - """ - Generate a global map and regional subplots for a specified variable from an xarray dataset. - - This function creates a global map and several regional subplots (North America, South America, Europe, Africa, Asia, and Australia) - using the specified variable from an xarray dataset. The generated map will be saved as a PNG file. - - Parameters: - ---------- - dataset : xarray.Dataset - The input xarray dataset containing the variable to be plotted. - - var : str - The name of the variable in the dataset to be plotted. - - dates: list - The list of dates from the dataframe, converted to period index. - - plt_time : int, optional - The time index to plot from the variable data. Defaults to the length of `dates` - 1, or last date in dataset. - - colormap : str, optional - The colormap to use for plotting the data. Defaults to 'viridis'. - - title : str, optional - The title for the figure. Defaults to an empty string. - - Returns: - ------- - fig : matplotlib figure object - The function returns the plot for saving - - Notes: - ----- - The function uses Cartopy for map projections and Matplotlib for plotting. - Ensure Cartopy and Matplotlib are installed in your Python environment. - - The output file is saved with the format '_global_map.png'. - - Examples: - -------- - global_map(my_dataset, 'mrso', plt_time=0, colormap='coolwarm', title='Global Temperature Map') - """ - lon = dataset.lon - lat = dataset.lat - - if plt_time is None: plt_time=len(dates)-1 - - data = dataset[var][plt_time].values - projection = ccrs.PlateCarree() - fig = plt.figure(figsize=(8.5, 11)) - - # Global map - ax_global = fig.add_subplot(3, 1, 1, projection=projection) - ax_global.set_title('Global Map') - mesh = ax_global.pcolormesh(lon, lat, data, transform=projection, cmap=colormap) - ax_global.coastlines() - - # List of bounding boxes for different continents (min_lon, max_lon, min_lat, max_lat) - regions = { - 'North America': [-170, -47, 0, 85], - 'South America': [-90, -30, -60, 15], - 'Europe': [-10, 60, 30, 75], - 'Africa': [-20, 50, -35, 37], - 'Asia': [60, 150, 5, 75], - 'Australia': [110, 180, -50, 0] - } - - # Create subplots for each region - for i, (region, bbox) in enumerate(regions.items(), start=1): - ax = fig.add_subplot(3, 3, i + 3, projection=projection) - ax.set_extent(bbox, crs=projection) - ax.set_title(region) - ax.pcolormesh(lon, lat, data, transform=projection, cmap=colormap) - ax.coastlines() - ax.add_feature(cartopy.feature.BORDERS) - - # Add colorbar - fig.colorbar(mesh, ax=ax_global, orientation='horizontal', pad=0.05, aspect=50) - plt.suptitle(title) - plt.tight_layout() - - return fig - # plt.savefig(var+'_global_map.png') - # plt.close() - - def timeseries(self, dataset,var,dates_period,var_range=None,minlon = 0,maxlon = 360,minlat = -90,maxlat=90,timerange=None,title=''): - ''' - Generate a time series plot of the specified variable from a dataset within a given geographic and temporal range. - - - Parameters: - ----------- - dataset : xarray.Dataset - The dataset containing the variable to be plotted. - var : str - The name of the variable to plot from the dataset. - dates_period : pandas.DatetimeIndex - The dates for the time series data. - var_range : tuple of float, optional - The range of variable values to include in the plot (min, max). If not provided, the default range is (0, inf). - minlon : float, optional - The minimum longitude to include in the plot. Default is 0. - maxlon : float, optional - The maximum longitude to include in the plot. Default is 360. - minlat : float, optional - The minimum latitude to include in the plot. Default is -90. - maxlat : float, optional - The maximum latitude to include in the plot. Default is 90. - timerange : tuple of int, optional - The range of years to plot (start_year, end_year). If not provided, all available years in the dataset will be plotted. - title : str, optional - The title of the plot. Default is an empty string. - - Returns: - -------- - matplotlib.figure.Figure - The figure object containing the generated time series plot. - - Notes: - ------ - The function filters the dataset based on the provided variable range, longitude, and latitude bounds. It then - calculates the monthly and annual means of the specified variable and plots the seasonal and annual means. - - ''' - if var_range is not None: - data_filtered = dataset.where((dataset[var] > var_range[0]) & (dataset[var] <= var_range[1]) & - (dataset.lat >= minlat) & (dataset.lon >= minlon) & - (dataset.lat <= maxlat) & (dataset.lon <= maxlon)) - else: - data_filtered = dataset.where((dataset[var] > 0) & - (dataset.lat >= minlat) & (dataset.lon >= minlon) & - (dataset.lat <= maxlat) & (dataset.lon <= maxlon)) - data_filtered['time'] = dates_period - - data_df = pd.DataFrame(index = dates_period) - data_df['monthly_mean'] = data_filtered.resample(time='YE').mean(dim=['lat','lon'],skipna=True)[var].values - data_df['monthly_shift'] = data_df['monthly_mean'].shift(1) - - if timerange is not None: - ys, ye = (str(timerange[0]),str(timerange[1])) - plot_df = data_df.loc[f'{ys}-1-1':f'{ye}-1-1'] - else: - plot_df = data_df - - fig, ax = plt.subplots() - plot_df.resample('Q').mean()['monthly_shift'].plot(ax=ax,label='Seasonal Mean') - plot_df.resample('Y').mean()['monthly_mean'].plot(ax=ax,label='Annual Mean') - plt.legend() - plt.title(title) - plt.xlabel('Years') - return fig - - def run_analysis(self, catalog, png_dir, reference_catalog=None): - """Runs the analysis and generates all plots and associated datasets. - - Args: - catalog: Path to a model output catalog. - png_dir: Directory to store output png figures in. - reference_catalog: Path to a catalog of reference data. - - Returns: - A list of png figures. - """ - print ('WARNING: THESE FIGURES ARE FOR TESTING THE NEW ANALYSIS WORKFLOW ONLY AND SHOULD NOT BE USED IN ANY OFFICIAL MANNER FOR ANALYSIS OF LAND MODEL OUTPUT.') - col = intake.open_esm_datastore(catalog) - df = col.df - - # Soil Carbon - var = 'cSoil' - print ('Soil Carbon Analysis') - cat = col.search(variable_id=var,realm='land_cmip') - other_dict = cat.to_dataset_dict(cdf_kwargs={'chunks':{'time':12},'decode_times':False}) - combined_dataset = xr.concat(list(dict(sorted(other_dict.items())).values()), dim='time') - - # Other data: - # land_static_file = re.search('land_static:\s([\w.]*)',combined_dataset.associated_files).group(1) - # STATIC FILES SHOULD BE PART OF THE CATALOG FOR EASY ACCESS - - # Select Data and plot - dates = [datetime.date(1,1,1) + datetime.timedelta(d) for d in combined_dataset['time'].values] # Needs to be made dynamic - dates_period = pd.PeriodIndex(dates,freq='D') - - sm_fig = self.global_map(combined_dataset,var,dates,title='Soil Carbon Content (kg/m^2)') - plt.savefig(png_dir+var+'_global_map.png') - plt.close() - - ts_fig = self.timeseries(combined_dataset,var,dates_period,title='Global Average Soil Carbon') - plt.savefig(png_dir+var+'_global_ts.png') - plt.close() - - # Soil Moisture - var = 'mrso' - print ('Soil Moisture Analysis') - cat = col.search(variable_id=var,realm='land_cmip') - other_dict = cat.to_dataset_dict(cdf_kwargs={'chunks':{'time':12},'decode_times':False}) - combined_dataset = xr.concat(list(dict(sorted(other_dict.items())).values()), dim='time') - - # Other data: - # soil_area_file = re.search('soil_area:\s([\w.]*)',combined_dataset.associated_files).group(1) - # STATIC FILES SHOULD BE PART OF THE CATALOG FOR EASY ACCESS - - # Select Data and plot - dates = [datetime.date(1,1,1) + datetime.timedelta(d) for d in combined_dataset['time'].values] # Needs to be made dynamic - dates_period = pd.PeriodIndex(dates,freq='D') - - sm_fig = self.global_map(combined_dataset,var,dates,title='Soil Moisture (kg/m^2)') - plt.savefig(png_dir+var+'_global_map.png') - plt.close() diff --git a/freanalysis_land/pyproject.toml b/freanalysis_land/pyproject.toml deleted file mode 100644 index c0cc45d..0000000 --- a/freanalysis_land/pyproject.toml +++ /dev/null @@ -1,35 +0,0 @@ -[build-system] -requires = [ - "setuptools >= 40.9.0", -] -build-backend = "setuptools.build_meta" - -[project] -name = "freanalysis_land" -version = "0.1" -dependencies = [ - "setuptools", - "intake", - "intake-esm", - "xarray", - "matplotlib", - "cartopy", - "pandas", - "xarray" - -] -requires-python = ">= 3.6" -authors = [ - {name = "developers"}, -] -maintainers = [ - {name = "Anthony Preucil", email = "Anthony.Preucil@noaa.gov"}, -] -description = "Land Analysis for GFDL stand-alone model" -readme = "README.md" -classifiers = [ - "Programming Language :: Python" -] - -[project.urls] -repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/freanalysis_radiation/README.md b/freanalysis_radiation/README.md deleted file mode 100644 index 31120ee..0000000 --- a/freanalysis_radiation/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# freanalysis_radiation -A plugin for analyzing GFDL model radiative flux output. - -### Motivation -To create figures to analyze the radiative flux output from GFDL models. - -### Requirements -The software packages that are required are: - -- analysis-scripts -- figure_tools -- intake -- intake-esm - -### How to use this plugin -This plugin is designed to be used by the freanalysis package. diff --git a/freanalysis_radiation/freanalysis_radiation/__init__.py b/freanalysis_radiation/freanalysis_radiation/__init__.py deleted file mode 100644 index 827cc18..0000000 --- a/freanalysis_radiation/freanalysis_radiation/__init__.py +++ /dev/null @@ -1,309 +0,0 @@ -from dataclasses import dataclass -import json -from pathlib import Path - -from analysis_scripts import AnalysisScript -from figure_tools import AnomalyTimeSeries, GlobalMeanTimeSeries, LonLatMap, \ - observation_vs_model_maps, radiation_decomposition, \ - timeseries_and_anomalies -import intake -import intake_esm -from xarray import open_dataset - - -@dataclass -class Metadata: - activity_id: str = "dev" - institution_id: str = "" - source_id: str = "am5" - experiment_id: str = "c96L65_am5f7b11r0_amip" - frequency: str = "P1M" - modeling_realm: str = "atmos" - table_id: str = "" - member_id: str = "na" - grid_label: str = "" - temporal_subset: str = "" - chunk_freq: str = "" - platform: str = "" - cell_methods: str = "" - chunk_freq: str = "P1Y" - - def catalog_search_args(self, name): - return { - "experiment_id": self.experiment_id, - "frequency": self.frequency, - "member_id": self.member_id, - "modeling_realm": self.modeling_realm, - "variable_id": name, - } - - def variables(self): - return { - "rlds": "lwdn_sfc", - "rldsaf": "lwdn_sfc_ad", - "rldscs": "lwdn_sfc_clr", - "rldscsaf": "lwdn_sfc_ad_clr", - "rlus": "lwup_sfc", - "rlusaf": "lwup_sfc_ad", - "rluscs": "lwup_sfc_clr", - "rluscsaf": "lwup_sfc_ad_clr", - "rlut": "olr", - "rlutaf": "lwtoa_ad", - "rlutcs": "olr_clr", - "rlutcsaf": "lwtoa_ad_clr", - "rsds": "swdn_sfc", - "rsdsaf": "swdn_sfc_ad", - "rsdscs": "swdn_sfc_clr", - "rsdscsaf": "swdn_sfc_ad_clr", - "rsus": "swup_sfc", - "rsusaf": "swup_sfc_ad", - "rsuscs": "swup_sfc_clr", - "rsuscsaf": "swup_sfc_ad_clr", - "rsut": "swup_toa", - "rsutaf": "swup_toa_ad", - "rsutcs": "swup_toa_clr", - "rsutcsaf": "swup_toa_ad_clr", - "rsdt": "swdn_toa", - } - - -class RadiationAnalysisScript(AnalysisScript): - """Abstract base class for analysis scripts. User-defined analysis scripts - should inhert from this class and override the requires and run_analysis methods. - - Attributes: - description: Longer form description for the analysis. - title: Title that describes the analysis. - """ - def __init__(self): - self.metadata = Metadata() - self.description = "Calculates radiative flux metrics." - self.title = "Radiative Fluxes" - - def requires(self): - """Provides metadata describing what is needed for this analysis to run. - - Returns: - A json string containing the metadata. - """ - columns = Metadata.__annotations__.keys() - settings = {x: getattr(self.metadata, x) for x in columns} - return json.dumps({ - "settings": settings, - "dimensions": { - "lat": {"standard_name": "latitude"}, - "lon": {"standard_name": "longitude"}, - "time": {"standard_name": "time"} - }, - "varlist": { - "lwup_sfc": { - "standard_name": "surface_outgoing_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lwup_sfc_ad": { - "standard_name": "surface_outgoing_aerosol_free_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lwup_sfc_clr": { - "standard_name": "surface_outgoing_clear_sky_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lwup_sfc_ad_clr": { - "standard_name": "surface_outgoing_clear_sky_aerosol_free_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lwdn_sfc": { - "standard_name": "surface_incoming_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lwdn_sfc_ad": { - "standard_name": "surface_incoming_aerosol_free_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lwdn_sfc_clr": { - "standard_name": "surface_incoming_clear_sky_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lwdn_sfc_ad_clr": { - "standard_name": "surface_incoming_clear_sky_aerosol_free_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "olr": { - "standard_name": "toa_outgoing_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lwtoa_ad": { - "standard_name": "toa_outgoing_aerosol_free_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "olr_clr": { - "standard_name": "toa_outgoing_clear_sky_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "lwtoa_ad_clr": { - "standard_name": "toa_outgoing_clear_sky_aerosol_free_longwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swup_sfc": { - "standard_name": "surface_outgoing_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swup_sfc_ad": { - "standard_name": "surface_outgoing_aerosol_free_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swup_sfc_clr": { - "standard_name": "surface_outgoing_clear_sky_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swup_sfc_ad_clr": { - "standard_name": "surface_outgoing_clear_sky_aerosol_free_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swdn_sfc": { - "standard_name": "surface_incoming_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swdn_sfc_ad": { - "standard_name": "surface_incoming_aerosol_free_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swdn_sfc_clr": { - "standard_name": "surface_incoming_clear_sky_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swdn_sfc_ad_clr": { - "standard_name": "surface_incoming_clear_sky_aerosol_free_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swup_toa": { - "standard_name": "toa_outgoing_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swup_toa_ad": { - "standard_name": "toa_outgoing_aerosol_free_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swup_toa_clr": { - "standard_name": "toa_outgoing_clear_sky_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swup_toa_ad_clr": { - "standard_name": "toa_outgoing_clear_sky_aerosol_free_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - "swdn_toa": { - "standard_name": "toa_downwelling_shortwave_flux", - "units": "W m-2", - "dimensions": ["time", "lat", "lon"] - }, - }, - }) - - def run_analysis(self, catalog, png_dir, reference_catalog=None): - """Runs the analysis and generates all plots and associated datasets. - - Args: - catalog: Path to a catalog. - png_dir: Path to the directory where the figures will be made. - reference_catalog: Path to a catalog of reference data. - - Returns: - A list of paths to the figures that were created. - """ - - # Connect to the catalog and find the necessary datasets. - catalog = intake.open_esm_datastore(catalog) - - anomalies = {} - maps = {} - timeseries = {} - for name, variable in self.metadata.variables().items(): - # Get the dataset out of the catalog. - args = self.metadata.catalog_search_args(variable) - - datasets = catalog.search( - **self.metadata.catalog_search_args(variable) - ).to_dataset_dict(progressbar=False) - dataset = list(datasets.values())[0] - - # Lon-lat maps. - maps[name] = LonLatMap.from_xarray_dataset( - dataset, - variable, - time_method="annual mean", - year=1980, - ) - - if name == "rlut": - anomalies[name] = AnomalyTimeSeries.from_xarray_dataset( - dataset, - variable, - ) - timeseries[name] = GlobalMeanTimeSeries.from_xarray_dataset( - dataset, - variable, - ) - - figure_paths = [] - - # OLR anomally timeseries. - figure = timeseries_and_anomalies(timeseries["rlut"], anomalies["rlut"], - "OLR Global Mean & Anomalies") - figure.save(Path(png_dir) / "olr-anomalies.png") - figure_paths.append(Path(png_dir) / "olr-anomalies.png") - - # OLR. - figure = radiation_decomposition(maps["rlutcsaf"], maps["rlutaf"], - maps["rlutcs"], maps["rlut"], "OLR") - figure.save(Path(png_dir) / "olr.png") - figure_paths.append(Path(png_dir) / "olr.png") - - # SW TOTA. - figure = radiation_decomposition(maps["rsutcsaf"], maps["rsutaf"], - maps["rsutcs"], maps["rsut"], - "Shortwave Outgoing Toa") - figure.save(Path(png_dir) / "sw-up-toa.png") - figure_paths.append(Path(png_dir) / "sw-up-toa.png") - - # Surface radiation budget. - surface_budget = [] - for suffix in ["csaf", "af", "cs", ""]: - surface_budget.append(maps[f"rlds{suffix}"] + maps[f"rsds{suffix}"] - - maps[f"rlus{suffix}"] - maps[f"rsus{suffix}"]) - figure = radiation_decomposition(*surface_budget, "Surface Radiation Budget") - figure.save(Path(png_dir) / "surface-radiation-budget.png") - figure_paths.append(Path(png_dir) / "surface-radiation-budget.png") - - # TOA radiation budget. - toa_budget = [] - for suffix in ["csaf", "af", "cs", ""]: - toa_budget.append(maps[f"rsdt"] - maps[f"rlut{suffix}"] - maps[f"rsut{suffix}"]) - figure = radiation_decomposition(*toa_budget, "TOA Radiation Budget") - figure.save(Path(png_dir) / "toa-radiation-budget.png") - figure_paths.append(Path(png_dir) / "toa-radiation-budget.png") - return figure_paths diff --git a/freanalysis_radiation/pyproject.toml b/freanalysis_radiation/pyproject.toml deleted file mode 100644 index 14e3b49..0000000 --- a/freanalysis_radiation/pyproject.toml +++ /dev/null @@ -1,28 +0,0 @@ -[build-system] -requires = [ - "setuptools >= 40.9.0", -] -build-backend = "setuptools.build_meta" - -[project] -name = "freanalysis_radiation" -version = "0.1" -dependencies = [ - "intake", - "intake-esm", -] -requires-python = ">= 3.6" -authors = [ - {name = "developers"}, -] -maintainers = [ - {name = "developer", email = "developer-email@address.domain"}, -] -description = "Radiative flux analyzer for GFDL model output" -readme = "README.md" -classifiers = [ - "Programming Language :: Python" -] - -[project.urls] -repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" From f601448d24ca927623d499c32d8e08c69a7ed8b9 Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Wed, 4 Dec 2024 15:10:14 -0500 Subject: [PATCH 06/15] add tests, add conda metadata --- README.md | 113 +++++++++++++----- analysis-scripts/README.md | 84 ------------- analysis-scripts/pyproject.toml | 28 ----- analysis_scripts/__init__.py | 2 + .../base_class.py | 0 .../plugins.py | 14 ++- freanalysis/README.md | 40 ------- freanalysis/freanalysis/__init__.py | 1 - freanalysis/freanalysis/create_catalog.py | 104 ---------------- meta.yaml | 19 +++ freanalysis/pyproject.toml => pyproject.toml | 7 +- tests/mdtf_timeslice_catalog.yaml | 16 --- tests/test_base_class.py | 29 +++++ tests/test_freanalysis_clouds.py | 60 ---------- tests/test_freanalysis_land.py | 10 -- tests/test_plugins.py | 21 ++++ 16 files changed, 172 insertions(+), 376 deletions(-) delete mode 100644 analysis-scripts/README.md delete mode 100644 analysis-scripts/pyproject.toml create mode 100644 analysis_scripts/__init__.py rename analysis-scripts/analysis_scripts/__init__.py => analysis_scripts/base_class.py (100%) rename {freanalysis/freanalysis => analysis_scripts}/plugins.py (85%) delete mode 100644 freanalysis/README.md delete mode 100644 freanalysis/freanalysis/__init__.py delete mode 100644 freanalysis/freanalysis/create_catalog.py create mode 100644 meta.yaml rename freanalysis/pyproject.toml => pyproject.toml (86%) delete mode 100644 tests/mdtf_timeslice_catalog.yaml create mode 100644 tests/test_base_class.py delete mode 100644 tests/test_freanalysis_clouds.py delete mode 100644 tests/test_freanalysis_land.py create mode 100644 tests/test_plugins.py diff --git a/README.md b/README.md index f9cff0e..85061f0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # analysis-scripts -A framework for analyzing GFDL model output +A framework for analyzing GFDL model output. ### Motivation The goal of this project is to provide a simple API to guide the development of @@ -7,42 +7,97 @@ scripts that produce figures and tables from GFDL model output. This work will by a simple web application to provide users an easy interface to interact with model output data. -### Repository Structure -The code in this repository is broken up into components: - -- analysis-scripts - A very simple package that just defines an abstract base class that - all user-created plugins should inherit from. -- figure_tools - An optional package that contains some helper functions and classes - for making common plots. -- freanalysis - A package that is designed to be used by whatever workflow is responsible - for running the analysis. -- freanalysis_aerosol - An example plugin that creates aerosl mass figures. -- freanalysis_clouds - An example plugin that creates cloud amount figures. -- freanalysis_radiation - An example plugin that creates radiative flux figures. - -### Installing the basic components -For now I'd recommend creating a virtual enviroment, and then installing each of the -packages listed above: +### How to install this package +For now I'd recommend creating a virtual enviroment, and then installing the package inside +of it. ```bash $ python3 -m venv env $ source env/bin/activate $ pip install --upgrade pip -$ cd analysis-scripts; pip install .; cd .. -$ cd figure_tools; pip install .; cd .. -$ cd freanalysis; pip install .; cd .. -$ cd freanalysis_aerosol; pip install .; cd .. -$ cd freanalysis_clouds; pip install .; cd .. -$ cd freanalysis_radiation; pip install .; cd .. +$ pip install . +``` + +### How to use this framework +Custom analysis scripts classes can be created that inherit the `AnalysisScript` base +class, then override its contructor, `requires`, and `run_analysis` methods. For example: + +```python3 +from analysis_scripts import AnalysisScript + + +class NewAnalysisScript(AnalysisScript): + """Analysis script for some task. + + Attributes: + description: Longer form description for the analysis. + title: Title that describes the analysis. + """ + def __init__(self): + """Instantiates an object. The user should provide a description and title.""" + self.description = "This analysis does something cool." + self.title = "Brief, but descriptive name for the analysis." + + def requires(self): + """Provides metadata describing what is needed for this analysis to run. + + Returns: + A json string describing the metadata. + """ + return json.dumps({ + "settings": { + "activity_id": "", + "institution_id": "", + "source_id": "", + "experiment_id": "", + "frequency": "", + "modeling_realm": "", + "table_id": "", + "member_id": "", + "grid_label": "", + "temporal_subset": "", + "chunk_freq": "", + "platform": "", + "cell_methods": "" + }, + "dimensions": { + # These are just examples, you can put more/different ones. + "lat": {"standard_name": "latitude"}, + "lon": {"standard_name": "longitude"}, + "time": {"standard_name": "time"} + }, + "varlist": { + "": { + "standard_name": "", + "units": "", + "dimensions": ["time", "lat", "lon"] + }, + } + }) + + def run_analysis(self, catalog, png_dir, config=None, reference_catalog=None): + """Runs the analysis and generates all plots and associated datasets. + + Args: + catalog: Path to a model output catalog. + png_dir: Directory to store output png figures in. + config: Dictionary of configuration options. + reference_catalog: Path to a catalog of reference data. + + Returns: + A list of png figures. + """ + Do some stuff to create the figures. + return ["figure1.png", "figure2.png",] ``` -# Running an analysis plugin -In order to run a plugin, you must first create a data catalog and can then perform -the analysis: +### Running a custom analysis script +In order to run a custom analysis script, you must first create a data catalog and +can then perform the analysis: ```python3 -from freanalysis.create_catalog import create_catalog -from freanalysis.plugins import list_plugins, plugin_requirements, run_plugin +from analysis_scripts.create_catalog import create_catalog +from analysis_scripts.plugins import list_plugins, plugin_requirements, run_plugin # Create a data catalog. @@ -52,7 +107,7 @@ create_catalog(pp_dir, "catalog.json") list_plugins() # Run the radiative fluxes plugin. -name = "freanalysis_radiation" +name = "freanalysis_radiation" # Name of the custom analysis script you want to run. reqs = plugin_requirements(name) print(reqs) run_plugin(name, "catalog.json", "pngs") diff --git a/analysis-scripts/README.md b/analysis-scripts/README.md deleted file mode 100644 index 0884e27..0000000 --- a/analysis-scripts/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# analysis-scripts -A framework for analyzing GFDL model output. - -### Motivation -The goal of this project is to provide a simple API to guide the development of -scripts that produce figures and tables from GFDL model output. This work will be used -by a simple web application to provide users with an easy interface to interact with model -output data. - -### Requirements -No external packages are required. - -### How to use this framework. -Custom analysis scripts classes can be created that inherit the `AnalysisScript` base -class, then override its contructor, `requires`, and `run_analysis` methods. For example: - -```python3 -from analysis_scripts import AnalysisScript - - -class NewAnalysisScript(AnalysisScript): - """Analysis script for some task. - - Attributes: - description: Longer form description for the analysis. - title: Title that describes the analysis. - """ - def __init__(self): - """Instantiates an object. The user should provide a description and title.""" - self.description = "This analysis does something cool." - self.title = "Brief, but descriptive name for the analysis." - - def requires(self): - """Provides metadata describing what is needed for this analysis to run. - - Returns: - A json string describing the metadata. - """ - return json.dumps({ - "settings": { - "activity_id": "", - "institution_id": "", - "source_id": "", - "experiment_id": "", - "frequency": "", - "modeling_realm": "", - "table_id": "", - "member_id": "", - "grid_label": "", - "temporal_subset": "", - "chunk_freq": "", - "platform": "", - "cell_methods": "" - }, - "dimensions": { - # These are just examples, you can put more/different ones. - "lat": {"standard_name": "latitude"}, - "lon": {"standard_name": "longitude"}, - "time": {"standard_name": "time"} - }, - "varlist": { - "": { - "standard_name": "", - "units": "", - "dimensions": ["time", "lat", "lon"] - }, - } - }) - - def run_analysis(self, catalog, png_dir, config=None, reference_catalog=None): - """Runs the analysis and generates all plots and associated datasets. - - Args: - catalog: Path to a model output catalog. - png_dir: Directory to store output png figures in. - config: Dictionary of configuration options. - reference_catalog: Path to a catalog of reference data. - - Returns: - A list of png figures. - """ - Do some stuff to create the figures. - return ["figure1.png", "figure2.png",] -``` diff --git a/analysis-scripts/pyproject.toml b/analysis-scripts/pyproject.toml deleted file mode 100644 index 131cc63..0000000 --- a/analysis-scripts/pyproject.toml +++ /dev/null @@ -1,28 +0,0 @@ -[build-system] -requires = [ - "setuptools >= 40.9.0", -] -build-backend = "setuptools.build_meta" - -[project] -name = "analysis-scripts" -version = "0.1" -dependencies = [ - "intake", - "intake-esm", -] -requires-python = ">= 3.6" -authors = [ - {name = "developers"}, -] -maintainers = [ - {name = "developer", email = "developer-email@address.domain"}, -] -description = "Framework for analyzing GFDL model output" -readme = "README.md" -classifiers = [ - "Programming Language :: Python" -] - -[project.urls] -repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/analysis_scripts/__init__.py b/analysis_scripts/__init__.py new file mode 100644 index 0000000..538aa34 --- /dev/null +++ b/analysis_scripts/__init__.py @@ -0,0 +1,2 @@ +from .base_class import AnalysisScript +from .plugins import available_plugins, plugin_requirements, run_plugin, UnknownPluginError diff --git a/analysis-scripts/analysis_scripts/__init__.py b/analysis_scripts/base_class.py similarity index 100% rename from analysis-scripts/analysis_scripts/__init__.py rename to analysis_scripts/base_class.py diff --git a/freanalysis/freanalysis/plugins.py b/analysis_scripts/plugins.py similarity index 85% rename from freanalysis/freanalysis/plugins.py rename to analysis_scripts/plugins.py index 2c21edc..194c7c5 100644 --- a/freanalysis/freanalysis/plugins.py +++ b/analysis_scripts/plugins.py @@ -2,7 +2,7 @@ import inspect import pkgutil -from analysis_scripts import AnalysisScript +from .base_class import AnalysisScript # Find all installed python modules with names that start with "freanalysis_" @@ -12,6 +12,11 @@ discovered_plugins[name] = importlib.import_module(name) +class UnknownPluginError(BaseException): + """Custom exception for when an invalid plugin name is used.""" + pass + + def _plugin_object(name): """Searches for a class that inherits from AnalysisScript in the plugin module. @@ -27,7 +32,12 @@ def _plugin_object(name): plugin module. """ # Loop through all attributes in the plugin package with the input name. - for attribute in vars(discovered_plugins[name]).values(): + try: + plugin_module = discovered_plugins[name] + except KeyError: + raise UnknownPluginError(f"could not find analysis script plugin {name}.") + + for attribute in vars(plugin_module).values(): # Try to find a class that inherits from the AnalysisScript class. if inspect.isclass(attribute) and AnalysisScript in attribute.__bases__: # Instantiate an object of this class. diff --git a/freanalysis/README.md b/freanalysis/README.md deleted file mode 100644 index 74d50fb..0000000 --- a/freanalysis/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# freanalysis -Package that can run GFDL model analysis plugins. - -### Motivation -This creates a simple way for FRE to discover and run user generated analysis packages -to create figures to analyze the GFDL models. - -### Requirements -The software packages that are required are: - -- analysis-scripts - -### How to use -Users can create their own python packages that start with the name `freanalysis_`. -This package will search for installed packages that start with this name, and -then try to use them: - -```python3 -from freanalysis import available_plugins, list_plugins, plugin_requirements, run_plugin - - -# Get a list of all available plugins: -plugins = available_plugins() - - -# Print out a list of all available plugins: -list_plugins() - - -# Get the metadata for each plugin. This can be used to create/verify against a data -# catalog (i.e., if you want to check if a plugin is compatable with a catalog). -for name in plugins: - metadata = plugin_requirements(name) - - -# Run the plugins. You need to pass in a path to a data catalog, and a directory where -# you want the figures to be created. -for name in plugins: - figures = run_plugin(name, catalog, png_dir, config=None, reference_catalog=None) -``` diff --git a/freanalysis/freanalysis/__init__.py b/freanalysis/freanalysis/__init__.py deleted file mode 100644 index 0b16a29..0000000 --- a/freanalysis/freanalysis/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Make this directory a python package. diff --git a/freanalysis/freanalysis/create_catalog.py b/freanalysis/freanalysis/create_catalog.py deleted file mode 100644 index 829fc17..0000000 --- a/freanalysis/freanalysis/create_catalog.py +++ /dev/null @@ -1,104 +0,0 @@ -import json -from pathlib import Path - - -# Catalog column names. -columns = [ - "activity_id", - "institution_id", - "source_id", - "experiment_id", - "frequency", - "modeling_realm", - "table_id", - "member_id", - "grid_label", - "variable_id", - "temporal_subset", - "chunk_freq", - "platform", - "cell_methods", - "path", -] - - -def create_catalog(pp_dir, output_path): - """Creates a catalog json. - - Args: - pp_dir: Path to the base of a FRE post-process directory tree. - output_path: Path to the output file. - """ - parent_dir = Path(output_path).parent - csv_path = parent_dir / f"{Path(output_path).stem}.csv" - with open(output_path, "w") as output: - json_str = { - "esmcat_version": "0.0.1", - "attributes": [ - {"column_name": "activity_id", "vocabulary": ""}, - {"column_name": "institution_id", "vocabulary": ""}, - {"column_name": "source_id", "vocabulary": ""}, - {"column_name": "experiment_id", "vocabulary": ""}, - {"column_name": "frequency", "vocabulary": ""}, - {"column_name": "modeling_realm", "vocabulary": ""}, - {"column_name": "table_id", "vocabulary": ""}, - {"column_name": "member_id", "vocabulary": ""}, - {"column_name": "grid_label", "vocabulary": ""}, - {"column_name": "variable_id", "vocabulary": ""}, - {"column_name": "temporal_subset", "vocabulary": ""}, - {"column_name": "chunk_freq", "vocabulary": ""}, - {"column_name": "platform", "vocabulary": ""}, - {"column_name": "cell_methods", "vocabulary": ""}, - {"column_name": "path", "vocabulary": ""} - ], - "assets": {"column_name": "path", "format": "netcdf","format_column_name": None}, - "aggregation_control": { - "variable_column_name": "variable_id", - "groupby_attrs": [ - "source_id", - "experiment_id", - "frequency", - "member_id", - "modeling_realm", - "variable_id", - "chunk_freq" - ], - "aggregations": [ - {"type": "union", "attribute_name": "variable_id", "options": {}}, - { - "type": "join_existing", - "attribute_name": "temporal_subset", - "options": { - "dim": "time", - "coords": "minimal", - "compat": "override" - } - } - ] - }, - "id": "", - "description": None, - "title": None, - "last_updated": "2023-05-07T16:35:52Z", - "catalog_file": str(csv_path) - } - json.dump(json_str, output) - - with open(csv_path, "w") as output: - output.write(",".join(columns) + "\n") - path = Path(pp_dir) - for name in path.iterdir(): - realm = name.stem - if not name.is_dir() or str(name.stem).startswith("."): continue - full = name / "ts/monthly/1yr" - for file_ in full.iterdir(): - attrs = {column: "" for column in columns} - if not str(file_).endswith(".nc"): continue - attrs["activity_id"] = "dev" - attrs["experiment_id"] = "c96L65_am5f4b4r1-newrad_amip" - attrs["modeling_realm"] = realm - attrs["frequency"] = "monthly" - attrs["member_id"] = "na" - attrs["variable_id"] = str(file_.stem).split(".")[-1] - attrs["path"] = str(file_) - output.write(",".join([attrs[column] for column in columns]) + "\n") diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 0000000..be464af --- /dev/null +++ b/meta.yaml @@ -0,0 +1,19 @@ +package: + name: analysis_scripts + version: 0.0.1 + +source: + path: . + +build: + script: "{{ PYTHON }} -m pip install . -vv" + number: 0 + noarch: python + +requirements: + host: + - python + - pip + run: + - python + - pytest diff --git a/freanalysis/pyproject.toml b/pyproject.toml similarity index 86% rename from freanalysis/pyproject.toml rename to pyproject.toml index ec42453..dde62e3 100644 --- a/freanalysis/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,11 @@ requires = [ build-backend = "setuptools.build_meta" [project] -name = "freanalysis" -version = "0.1" +name = "analysis_scripts" +version = "0.0.1" +dependencies = [ + "pytest", +] requires-python = ">= 3.6" authors = [ {name = "developers"}, diff --git a/tests/mdtf_timeslice_catalog.yaml b/tests/mdtf_timeslice_catalog.yaml deleted file mode 100644 index 1da1594..0000000 --- a/tests/mdtf_timeslice_catalog.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Catalog headers -# The headerlist is expected column names in your catalog/csv file. This is usually -# determined by the users in conjuction with the ESM collection specification standards -# and the appropriate workflows. -headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", - "frequency", "realm", "table_id", - "member_id", "grid_label", "variable_id", - "time_range", "chunk_freq", "grid_label", "platform", - "dimensions", "cell_methods", "standard_name","path"] - -output_path_template: ['NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', - 'source_id', 'experiment_id', 'NA', 'platform', 'custom_pp', - 'realm', 'cell_methods', 'frequency', 'chunk_freq'] - -output_file_template: ['realm', 'time_range', 'variable_id'] - diff --git a/tests/test_base_class.py b/tests/test_base_class.py new file mode 100644 index 0000000..ab8f4c3 --- /dev/null +++ b/tests/test_base_class.py @@ -0,0 +1,29 @@ +from pytest import raises + +from analysis_scripts import AnalysisScript + + +def test_no_init(): + #The constructor is not overridden. + with raises(NotImplementedError): + class FakeAnalysisScript(AnalysisScript): + pass + _ = FakeAnalysisScript() + + +def test_no_requires(): + #The requires function is not overridden. + with raises(NotImplementedError): + class FakeAnalysisScript(AnalysisScript): + def __init__(self): + pass + _ = FakeAnalysisScript().requires() + + +def test_no_run_analysis(): + #The requires function is not overridden. + with raises(NotImplementedError): + class FakeAnalysisScript(AnalysisScript): + def __init__(self): + pass + _ = FakeAnalysisScript().run_analysis("fake catalog", "fake png directory") diff --git a/tests/test_freanalysis_clouds.py b/tests/test_freanalysis_clouds.py deleted file mode 100644 index 87e24c1..0000000 --- a/tests/test_freanalysis_clouds.py +++ /dev/null @@ -1,60 +0,0 @@ -from ftplib import FTP -from os import chdir, environ -from pathlib import Path -from subprocess import run -from tempfile import TemporaryDirectory -import sys - -from freanalysis.plugins import list_plugins, plugin_requirements, run_plugin -import catalogbuilder -from catalogbuilder.scripts import gen_intake_gfdl - -def download_test_data(stem): - """Downloads test datasets from a FTP server. - - Args: - stem: Directory to create the directory tree inside. - - Returns: - Path to the directory that will be used as the root of the data catalog. - """ - # Create local directory tree with the appropriate directory structure. - catalog_root = Path(stem) / "archive" / "oar.gfdl.mdtf" / "MDTF-examples" / \ - "mdtf-time-slice-example" / "gfdl.ncrc5-deploy-prod-openmp" / "pp" - data_directory = catalog_root / "atmos" / "ts" / "monthly" / "1yr" - data_directory.mkdir(parents=True, exist_ok=True) - - # Download the datasets from the FTP server. - path = "1/oar.gfdl.mdtf/MDTF-examples/GFDL-CM4/data/atmos/ts/monthly/1yr" - with FTP("nomads.gfdl.noaa.gov") as ftp: - ftp.login() - ftp.cwd(path) - for variable in ["high_cld_amt", "mid_cld_amt", "low_cld_amt"]: - name = f"atmos.198001-198012.{variable}.nc" - ftp.retrbinary(f"RETR {name}", open(data_directory / name, "wb").write) - return catalog_root.resolve() - -def plugin(json, pngs_directory="pngs"): - """Run the plugin to create the figure. - - Args: - json: Path to the catalog json file. - pngs_directory: Directory to store the output in. - """ - name = "freanalysis_clouds" - reqs = plugin_requirements(name) - Path(pngs_directory).mkdir(parents=True, exist_ok=True) - run_plugin(name, json, pngs_directory) - - -def test_freanalysis_clouds(): - - with TemporaryDirectory() as tmp: - chdir(Path(tmp)) - path = download_test_data(stem=tmp) - yaml = Path(__file__).resolve().parent / "mdtf_timeslice_catalog.yaml" - outputpath = Path(__file__).resolve().parent / "data-catalog" - #Creates data catalog using the scripts in catalogbuilder - csv, json = gen_intake_gfdl.create_catalog(input_path=str(path),output_path=outputpath,config=str(yaml)) - print(json,csv) - plugin(json) diff --git a/tests/test_freanalysis_land.py b/tests/test_freanalysis_land.py deleted file mode 100644 index 99c778c..0000000 --- a/tests/test_freanalysis_land.py +++ /dev/null @@ -1,10 +0,0 @@ - - -from freanalysis_land.land import LandAnalysisScript - -def test_land_analysis_script(): - land = LandAnalysisScript() - land.run_analysis("/work/a2p/lm4p2sc_GSWP3_hist_irr_catalog.json","/work/a2p/") - - -test_land_analysis_script() diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 0000000..34e7852 --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,21 @@ +from pytest import raises + +from analysis_scripts import available_plugins, plugin_requirements, run_plugin, \ + UnknownPluginError + + +def test_no_plugins(): + """No valid plugins are installed.""" + assert available_plugins() == [] + + +def test_invalid_plugin_requirements(): + """An invalid plugin name is passed in to plugin_requirements.""" + with raises(UnknownPluginError): + _ = plugin_requirements("fake_plugin") + + +def test_invalid_run_plugin(): + """An invalid plugin name is passed in to run_plugin.""" + with raises(UnknownPluginError): + _ = run_plugin("fake_plugin", "fake_catalog.json", "fake_png_directory") From 87b5ac1a42d5372f14f209a642edfb31751fb4bb Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Wed, 4 Dec 2024 15:20:23 -0500 Subject: [PATCH 07/15] update github actions --- .github/workflows/ci-analysis.yml | 40 +++++++------------------------ 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index d6a972f..5c5144d 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -1,5 +1,5 @@ -# Installs the Python dependencies and runs the freanalysis_clouds plugin. -name: Test freanalysis_clouds plugin +# Test the core package. +name: Test analysis_scripts on: [pull_request] @@ -13,40 +13,16 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - + - name: Install dependencies run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - conda config --add channels noaa-gfdl - conda config --append channels conda-forge - conda create --name analysis-script-testing - conda install -n analysis-script-testing catalogbuilder -c noaa-gfdl - #conda activate analysis-script-testing - conda install pip - cd analysis-scripts - $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. - cd figure_tools - $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. - cd freanalysis - $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. - cd freanalysis_clouds - $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. - - - name: Generate catalog and run freanalysis_clouds - run: | - $CONDA/envs/analysis-script-testing/bin/pytest --capture=tee-sys tests/test_freanalysis_clouds.py - - name: upload-artifacts - uses: actions/upload-artifact@v4 - with: - name: workflow-artifacts1 - path: | - ${{ github.workspace }}/tests/data-catalog.json - ${{ github.workspace }}/tests/data-catalog.csv - - name: Run freanalysis_land + python -m pip install . + + - name: Run the unit tests run: | - $CONDA/envs/analysis-script-testing/bin/pytest tests/test_freanalysis_land + pytest --capture=tee-sys tests From 868ec83b6141732a9e14ce17d1adfedd9abbfa40 Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Wed, 4 Dec 2024 15:29:02 -0500 Subject: [PATCH 08/15] fix readme --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 85061f0..9169933 100644 --- a/README.md +++ b/README.md @@ -96,19 +96,17 @@ In order to run a custom analysis script, you must first create a data catalog a can then perform the analysis: ```python3 -from analysis_scripts.create_catalog import create_catalog -from analysis_scripts.plugins import list_plugins, plugin_requirements, run_plugin +from analysis_scripts.plugins import available_plugins, plugin_requirements, run_plugin # Create a data catalog. -create_catalog(pp_dir, "catalog.json") +# Some code to create a data "catalog.json" ... # Show the installed plugins. -list_plugins() +print(available_plugins()) # Run the radiative fluxes plugin. name = "freanalysis_radiation" # Name of the custom analysis script you want to run. -reqs = plugin_requirements(name) -print(reqs) -run_plugin(name, "catalog.json", "pngs") +print(plugin_requirements(name)) +figures = run_plugin(name, "catalog.json", "pngs") ``` From 86d4e5c2e7c96f349aa0b8ab8ae048cc25553ff1 Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Wed, 4 Dec 2024 15:33:15 -0500 Subject: [PATCH 09/15] fix readme again --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9169933..67ca3d5 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ In order to run a custom analysis script, you must first create a data catalog a can then perform the analysis: ```python3 -from analysis_scripts.plugins import available_plugins, plugin_requirements, run_plugin +from analysis_scripts import available_plugins, plugin_requirements, run_plugin # Create a data catalog. From a82fe7b28b5c86c7d2c9f68f84144d76bcadfe2e Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Fri, 6 Dec 2024 13:28:34 -0500 Subject: [PATCH 10/15] updates to the plugin discovery --- analysis_scripts/__init__.py | 3 ++- analysis_scripts/plugins.py | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/analysis_scripts/__init__.py b/analysis_scripts/__init__.py index 538aa34..c4a4f0a 100644 --- a/analysis_scripts/__init__.py +++ b/analysis_scripts/__init__.py @@ -1,2 +1,3 @@ from .base_class import AnalysisScript -from .plugins import available_plugins, plugin_requirements, run_plugin, UnknownPluginError +from .plugins import available_plugins, find_plugins, plugin_requirements, \ + run_plugin, UnknownPluginError diff --git a/analysis_scripts/plugins.py b/analysis_scripts/plugins.py index 194c7c5..9ac325e 100644 --- a/analysis_scripts/plugins.py +++ b/analysis_scripts/plugins.py @@ -5,11 +5,21 @@ from .base_class import AnalysisScript -# Find all installed python modules with names that start with "freanalysis_" +# Dictionary of found plugins. discovered_plugins = {} -for finder, name, ispkg in pkgutil.iter_modules(): - if name.startswith("freanalysis_"): - discovered_plugins[name] = importlib.import_module(name) + + +def find_plugins(path=None): + """Find all installed python modules with names that start with 'freanalysis_'.""" + if path: + path = [path,] + for finder, name, ispkg in pkgutil.iter_modules(path): + if name.startswith("freanalysis_"): + discovered_plugins[name] = importlib.import_module(name) + + +# Update plugin dictionary. +find_plugins() class UnknownPluginError(BaseException): From 084702b3c6c6d03820251b88688ca0a178756d06 Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Fri, 6 Dec 2024 13:32:12 -0500 Subject: [PATCH 11/15] move stuff around --- README.md => core/README.md | 0 {analysis_scripts => core/analysis_scripts}/__init__.py | 0 {analysis_scripts => core/analysis_scripts}/base_class.py | 0 {analysis_scripts => core/analysis_scripts}/plugins.py | 0 meta.yaml => core/meta.yaml | 0 pyproject.toml => core/pyproject.toml | 0 {tests => core/tests}/test_base_class.py | 0 {tests => core/tests}/test_plugins.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename README.md => core/README.md (100%) rename {analysis_scripts => core/analysis_scripts}/__init__.py (100%) rename {analysis_scripts => core/analysis_scripts}/base_class.py (100%) rename {analysis_scripts => core/analysis_scripts}/plugins.py (100%) rename meta.yaml => core/meta.yaml (100%) rename pyproject.toml => core/pyproject.toml (100%) rename {tests => core/tests}/test_base_class.py (100%) rename {tests => core/tests}/test_plugins.py (100%) diff --git a/README.md b/core/README.md similarity index 100% rename from README.md rename to core/README.md diff --git a/analysis_scripts/__init__.py b/core/analysis_scripts/__init__.py similarity index 100% rename from analysis_scripts/__init__.py rename to core/analysis_scripts/__init__.py diff --git a/analysis_scripts/base_class.py b/core/analysis_scripts/base_class.py similarity index 100% rename from analysis_scripts/base_class.py rename to core/analysis_scripts/base_class.py diff --git a/analysis_scripts/plugins.py b/core/analysis_scripts/plugins.py similarity index 100% rename from analysis_scripts/plugins.py rename to core/analysis_scripts/plugins.py diff --git a/meta.yaml b/core/meta.yaml similarity index 100% rename from meta.yaml rename to core/meta.yaml diff --git a/pyproject.toml b/core/pyproject.toml similarity index 100% rename from pyproject.toml rename to core/pyproject.toml diff --git a/tests/test_base_class.py b/core/tests/test_base_class.py similarity index 100% rename from tests/test_base_class.py rename to core/tests/test_base_class.py diff --git a/tests/test_plugins.py b/core/tests/test_plugins.py similarity index 100% rename from tests/test_plugins.py rename to core/tests/test_plugins.py From 4e42910cb664c01422dedd3dbdb8ca70619e8fc4 Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Fri, 6 Dec 2024 13:33:57 -0500 Subject: [PATCH 12/15] Revert "removed plugins to get down to just the core stuff" This reverts commit 5d2dd9a1835a56737ce982d678e0b048730b76ea. --- figure_tools/README.md | 43 +++ figure_tools/figure_tools/__init__.py | 7 + .../figure_tools/anomaly_timeseries.py | 46 +++ figure_tools/figure_tools/common_plots.py | 85 +++++ figure_tools/figure_tools/figure.py | 130 ++++++++ .../figure_tools/global_mean_timeseries.py | 48 +++ figure_tools/figure_tools/lon_lat_map.py | 141 ++++++++ figure_tools/figure_tools/time_subsets.py | 72 ++++ figure_tools/figure_tools/zonal_mean_map.py | 86 +++++ figure_tools/pyproject.toml | 30 ++ freanalysis_aerosol/README.md | 16 + .../freanalysis_aerosol/__init__.py | 209 ++++++++++++ freanalysis_aerosol/pyproject.toml | 28 ++ freanalysis_clouds/README.md | 16 + .../freanalysis_clouds/__init__.py | 122 +++++++ freanalysis_clouds/pyproject.toml | 28 ++ freanalysis_land/README.md | 1 + freanalysis_land/freanalysis_land/__init__.py | 0 freanalysis_land/freanalysis_land/land.py | 243 ++++++++++++++ freanalysis_land/pyproject.toml | 35 ++ freanalysis_radiation/README.md | 16 + .../freanalysis_radiation/__init__.py | 309 ++++++++++++++++++ freanalysis_radiation/pyproject.toml | 28 ++ 23 files changed, 1739 insertions(+) create mode 100644 figure_tools/README.md create mode 100644 figure_tools/figure_tools/__init__.py create mode 100644 figure_tools/figure_tools/anomaly_timeseries.py create mode 100644 figure_tools/figure_tools/common_plots.py create mode 100644 figure_tools/figure_tools/figure.py create mode 100644 figure_tools/figure_tools/global_mean_timeseries.py create mode 100644 figure_tools/figure_tools/lon_lat_map.py create mode 100644 figure_tools/figure_tools/time_subsets.py create mode 100644 figure_tools/figure_tools/zonal_mean_map.py create mode 100644 figure_tools/pyproject.toml create mode 100644 freanalysis_aerosol/README.md create mode 100644 freanalysis_aerosol/freanalysis_aerosol/__init__.py create mode 100644 freanalysis_aerosol/pyproject.toml create mode 100644 freanalysis_clouds/README.md create mode 100644 freanalysis_clouds/freanalysis_clouds/__init__.py create mode 100644 freanalysis_clouds/pyproject.toml create mode 100644 freanalysis_land/README.md create mode 100644 freanalysis_land/freanalysis_land/__init__.py create mode 100644 freanalysis_land/freanalysis_land/land.py create mode 100644 freanalysis_land/pyproject.toml create mode 100644 freanalysis_radiation/README.md create mode 100644 freanalysis_radiation/freanalysis_radiation/__init__.py create mode 100644 freanalysis_radiation/pyproject.toml diff --git a/figure_tools/README.md b/figure_tools/README.md new file mode 100644 index 0000000..5d015d6 --- /dev/null +++ b/figure_tools/README.md @@ -0,0 +1,43 @@ +# figure_tools +Tools to make common analysis figures. + +### Motivation +The goal of this project is to provide a simple API to guide the development of +scripts that produce figures from xarry datasets (such as those produced from climate +models). + +### Requirements +The only software packages that are required are: + +- cartopy +- matplotlib +- numpy +- xarray + +### How to install this package +For now I'd recommend creating and installing this package in a virtual enviroment: + +```bash +$ python3 -m venv env +$ source env/bin/activate +$ pip install --upgrade pip +$ git clone https://github.com/NOAA-GFDL/analysis-scripts.git +$ cd analysis-scripts/figure_tools +$ pip install . +``` + +### Creating plots +Longitude-latitude maps, heatmaps, and line plots can be made from the provided +objects. These objects can be instantiated directly from `xarray` datasets. For example: + +```python3 +# Longitude-latitude map. +from figure_tools import Figure, LonLatMap + + +map = LonLatMap.from_xarray_dataset(, , + time_method="annual mean", year=2010) +figure = Figure(title=) +figure.add_map(map_) +figure.save() +``` diff --git a/figure_tools/figure_tools/__init__.py b/figure_tools/figure_tools/__init__.py new file mode 100644 index 0000000..34f376b --- /dev/null +++ b/figure_tools/figure_tools/__init__.py @@ -0,0 +1,7 @@ +from .anomaly_timeseries import AnomalyTimeSeries +from .common_plots import observation_vs_model_maps, radiation_decomposition, \ + timeseries_and_anomalies, zonal_mean_vertical_and_column_integrated_map +from .figure import Figure +from .global_mean_timeseries import GlobalMeanTimeSeries +from .lon_lat_map import LonLatMap +from .zonal_mean_map import ZonalMeanMap diff --git a/figure_tools/figure_tools/anomaly_timeseries.py b/figure_tools/figure_tools/anomaly_timeseries.py new file mode 100644 index 0000000..7422d48 --- /dev/null +++ b/figure_tools/figure_tools/anomaly_timeseries.py @@ -0,0 +1,46 @@ +from numpy import array, mean, transpose + +from .time_subsets import TimeSubset + + +class AnomalyTimeSeries(object): + def __init__(self, data, time_data, latitude, units): + self.data = data[...] + self.x_data = time_data[...] + self.y_data = latitude[...] + self.x_label = "Time" + self.y_label = "Latitude" + self.data_label = units + + @classmethod + def from_xarray_dataset(cls, dataset, variable): + """Instantiates an AnomalyTimeSeries object from an xarray dataset.""" + v = dataset.data_vars[variable] + axis_attrs = _dimension_order(dataset, v) + + time = TimeSubset(array(dataset.coords[v.dims[0]].data)) + latitude = array(dataset.coords[v.dims[-2]].data) + data = mean(array(v.data), axis=-1) # Average over longitude. + + time, data = time.annual_means(data) + average = mean(data, axis=0) # Average over longitude and time. + anomaly = data + for i in range(time.size): + anomaly[i, :] -= average[:] + anomaly = transpose(anomaly) + + return cls(anomaly, time, latitude, v.attrs["units"]) + + +def _dimension_order(dataset, variable): + """Raises a ValueError if the variable's dimensions are not in an expected order. + + Returns: + A list of the dimension axis attribute strings. + """ + axis_attrs = [dataset.coords[x].attrs["axis"].lower() + if "axis" in dataset.coords[x].attrs else None + for x in variable.dims] + if axis_attrs == ["t", "y", "x"]: + return axis_attrs + raise ValueError(f"variable {variable} contains unexpected axes ordering {axis_attrs}.") diff --git a/figure_tools/figure_tools/common_plots.py b/figure_tools/figure_tools/common_plots.py new file mode 100644 index 0000000..b5e27ee --- /dev/null +++ b/figure_tools/figure_tools/common_plots.py @@ -0,0 +1,85 @@ +from math import ceil, floor + +from numpy import abs, max, min, percentile + +from .figure import Figure + + +def observation_vs_model_maps(reference, model, title): + figure = Figure(num_rows=2, num_columns=2, title=title, size=(14, 12)) + + # Create common color bar for reference and model. + reference_range = [floor(min(reference.data)), ceil(max(reference.data))] + model_range = [floor(min(model.data)), ceil(max(model.data))] + colorbar_range = [None, None] + colorbar_range[0] = reference_range[0] if reference_range[0] < model_range[0] \ + else model_range[0] + colorbar_range[1] = reference_range[1] if reference_range[1] > model_range[1] \ + else model_range[1] + + # Reference data. + global_mean = reference.global_mean() + figure.add_map(reference, f"Observations [Mean: {global_mean:.2f}]", 1, + colorbar_range=colorbar_range) + + # Model data. + global_mean = model.global_mean() + figure.add_map(model, f"Model [Mean: {global_mean:.2f}]", 2, + colorbar_range=colorbar_range) + + # Difference between the reference and model. + difference = reference - model + color_range = _symmetric_colorbar_range(difference.data) + global_mean = difference.global_mean() + figure.add_map(difference, f"Obs - Model [Mean: {global_mean:.2f}]", 3, + colorbar_range=color_range, + normalize_colors=True) + + # Use percentiles. + zoom = int(ceil(percentile(abs(difference.data), 95))) + figure.add_map(difference, f"Obs - Model [Mean: {global_mean:.2f}]", 4, + colorbar_range=[-1*zoom, zoom], num_levels=19, + normalize_colors=True) + return figure + + +def radiation_decomposition(clean_clear_sky, clean_sky, clear_sky, all_sky, title): + figure = Figure(num_rows=2, num_columns=2, title=title, size=(16, 10)) + maps = [clean_clear_sky, clean_sky - clean_clear_sky, all_sky - clean_sky, all_sky] + titles = ["Clean-clear Sky", "Cloud Effects", "Aerosol Effects", "All Sky"] + for i, (map_, title) in enumerate(zip(maps, titles)): + global_mean = map_.global_mean() + updated_title = f"{title} [Mean: {global_mean:.2f}]" + if title in ["Cloud Effects", "Aerosol Effects"]: + figure.add_map(map_, updated_title, i + 1, + colorbar_range=_symmetric_colorbar_range(map_.data), + normalize_colors=True) + else: + figure.add_map(map_, updated_title, i + 1, + normalize_colors=True, colorbar_center=global_mean) + return figure + + +def timeseries_and_anomalies(timeseries, map_, title): + figure = Figure(num_rows=1, num_columns=2, title=title, size=(16, 10)) + figure.add_line_plot(timeseries, "Timeseries", 1) + figure.add_map(map_, "Zonal Mean Anomalies", 2, + colorbar_range=_symmetric_colorbar_range(map_.data), + normalize_colors=True) + return figure + + +def zonal_mean_vertical_and_column_integrated_map(zonal_mean, lon_lat, title): + figure = Figure(num_rows=1, num_columns=2, title=title, size=(16, 10)) + figure.add_map(zonal_mean, "Zonal Mean Vertical Profile", 1) + figure.add_map(lon_lat, "Column-integrated", 2) + return figure + + +def _symmetric_colorbar_range(data): + colorbar_range = [int(floor(min(data))), int(ceil(max(data)))] + if abs(colorbar_range[0]) > abs(colorbar_range[1]): + colorbar_range[1] = -1*colorbar_range[0] + else: + colorbar_range[0] = -1*colorbar_range[1] + return colorbar_range diff --git a/figure_tools/figure_tools/figure.py b/figure_tools/figure_tools/figure.py new file mode 100644 index 0000000..f79a99b --- /dev/null +++ b/figure_tools/figure_tools/figure.py @@ -0,0 +1,130 @@ +from math import ceil + +import cartopy.crs as ccrs +import matplotlib as mpl +import matplotlib.colors as colors +import matplotlib.pyplot as plt +from numpy import linspace, max, min, unravel_index + +from .lon_lat_map import LonLatMap +from .zonal_mean_map import ZonalMeanMap + + +class Figure(object): + def __init__(self, num_rows=1, num_columns=1, size=(16, 12), title=None): + """Creates a figure for the input number of plots. + + Args: + num_rows: Number of rows of plots. + num_columns: Number of columns of plots. + """ + self.figure = plt.figure(figsize=size, layout="compressed") + if title is not None: + self.figure.suptitle(title.title()) + self.num_rows = num_rows + self.num_columns = num_columns + self.plot = [[None for y in range(num_columns)] for x in range(num_rows)] + + def add_map(self, map_, title, position=1, colorbar_range=None, colormap="coolwarm", + normalize_colors=False, colorbar_center=0, num_levels=51, extend=None): + """Adds a map to the figure. + + Args: + map_: LonLatMap or ZonalMeanMap object. + total: String title for the plot. + position: Integer position index for the plot in the figure. + colorbar_range: List of integers describing the colorbar limits. + """ + # Create the plotting axes. + optional_args = {} + if isinstance(map_, LonLatMap): + optional_args["projection"] = map_.projection + plot = self.figure.add_subplot(self.num_rows, self.num_columns, + position, **optional_args) + + # Set the colorbar properties. + if colorbar_range == None: + levels = num_levels + else: + # There seems to be some strange behavior if the number of level is too + # big for a given range. + levels = linspace(colorbar_range[0], colorbar_range[-1], num_levels, endpoint=True) + if extend == None: + data_max = max(map_.data) + data_min = min(map_.data) + if data_max > colorbar_range[1] and data_min < colorbar_range[0]: + extend = "both" + elif data_max > colorbar_range[1]: + extend = "max" + elif data_min < colorbar_range[0]: + extend = "min" + if normalize_colors: + norm = colors.CenteredNorm(vcenter=colorbar_center) + else: + norm = None + + # Make the map. + optional_args = {"levels": levels, "cmap": colormap, "norm": norm, "extend": extend} + if isinstance(map_, LonLatMap): + optional_args["transform"] = ccrs.PlateCarree() + cs = plot.contourf(map_.x_data, map_.y_data, map_.data, **optional_args) + + # Set the metadata. + self.figure.colorbar(cs, ax=plot, label=map_.data_label) + if isinstance(map_, LonLatMap): + plot.coastlines() + grid = plot.gridlines(draw_labels=True, dms=True, x_inline=False, y_inline=False) + grid.bottom_labels = False + grid.top_labels = False + if isinstance(map_, ZonalMeanMap) and map_.invert_y_axis: + plot.invert_yaxis() + plot.set_title(title.replace("_", " ").title()) + plot.set_xlabel(map_.x_label) + plot.set_ylabel(map_.y_label) + + # Add date information if necessary. + if hasattr(map_, "timestamp") and map_.timestamp != None: + plot.text(0.85, 1, map_.timestamp, transform=plot.transAxes) + + # Store the plot in the figure object. + x, y = self._plot_position_to_indices(position) + self.plot[x][y] = plot + + def add_line_plot(self, line_plot, title, position=1): + """Adds a line plot to the figure. + + Args: + map_: LonLatMap or ZonalMeanMap object. + total: String title for the plot. + position: Integer position index for the plot in the figure. + """ + plot = self.figure.add_subplot(self.num_rows, self.num_columns, position) + plot.plot(line_plot.x_data, line_plot.data) +# if isinstance(line_plot, GlobalMeanVerticalPlot) and line_plot.invert_y_axis: +# plot.invert_yaxis() + plot.set_title(title) + plot.set_xlabel(line_plot.x_label) + plot.set_ylabel(line_plot.y_label) + + # Store the plot in the figure object. + x, y = self._plot_position_to_indices(position) + self.plot[x][y] = plot + + def display(self): + """Shows the figure in a new window.""" + plt.show() + + def save(self, path): + plt.savefig(path) + plt.clf() + + def _plot_position_to_indices(self, position): + """Converts from a plot position to its x and y indices. + + Args: + position: Plot position (from 1 to num_rows*num_columns). + + Returns: + The x and y indices for the plot. + """ + return unravel_index(position - 1, (self.num_rows, self.num_columns)) diff --git a/figure_tools/figure_tools/global_mean_timeseries.py b/figure_tools/figure_tools/global_mean_timeseries.py new file mode 100644 index 0000000..68b2f94 --- /dev/null +++ b/figure_tools/figure_tools/global_mean_timeseries.py @@ -0,0 +1,48 @@ +from numpy import array, cos, mean, pi, sum + +from .time_subsets import TimeSubset + + +class GlobalMeanTimeSeries(object): + def __init__(self, data, time_data, units): + self.data = data[...] + self.x_data = time_data[...] + self.x_label = "Time" + self.y_label = units + + @classmethod + def from_xarray_dataset(cls, dataset, variable): + """Instantiates an AnomalyTimeSeries object from an xarray dataset.""" + v = dataset.data_vars[variable] + axis_attrs = _dimension_order(dataset, v) + + time = TimeSubset(array(dataset.coords[v.dims[0]].data)) + latitude = array(dataset.coords[v.dims[-2]].data) + time, data = time.annual_means(v.data) + data = _global_mean(data, latitude) + + return cls(data, time, v.attrs["units"]) + + +def _dimension_order(dataset, variable): + """Raises a ValueError if the variable's dimensions are not in an expected order. + + Returns: + A list of the dimension axis attribute strings. + """ + axis_attrs = [dataset.coords[x].attrs["axis"].lower() + if "axis" in dataset.coords[x].attrs else None + for x in variable.dims] + if axis_attrs == ["t", "y", "x"]: + return axis_attrs + raise ValueError(f"variable {variable} contains unexpected axes ordering {axis_attrs}.") + + +def _global_mean(data, latitude): + """Performs a global mean over the longitude and latitude dimensions. + + Returns: + Gobal mean value. + """ + weights = cos(2.*pi*latitude/360.) + return sum(mean(data, axis=-1)*weights, axis=-1)/sum(weights) diff --git a/figure_tools/figure_tools/lon_lat_map.py b/figure_tools/figure_tools/lon_lat_map.py new file mode 100644 index 0000000..382fc4b --- /dev/null +++ b/figure_tools/figure_tools/lon_lat_map.py @@ -0,0 +1,141 @@ +import cartopy.crs as ccrs +from cartopy.util import add_cyclic +from numpy import array, array_equal, cos, mean, ndarray, pi, sum +from xarray import DataArray + +from .time_subsets import TimeSubset + + +class LonLatMap(object): + """Longitude-latitude data map. + + Attributes: + coastlines: Flag that determines if coastlines are drawn on the map. + data: numpy array of data values. + data_label: String units for the colorbar. + projection: Cartopy map projection to use. + x_data: numpy array of data values for the x-axis. + xlabel: String label for the x-axis ("Longitude") + x_data: numpy array of data values for the y-axis. + ylabel: String label for the y-axis ("Latitude") + """ + def __init__(self, data, longitude, latitude, units=None, + projection=ccrs.Mollweide(), coastlines=True, add_cyclic_point=True, + timestamp=None): + if add_cyclic_point: + self.data, self.x_data, self.y_data = add_cyclic(data[...], longitude[...], + latitude[...]) + else: + self.data = data[...] + self.x_data = longitude[...] + self.y_data = latitude[...] + self.projection = projection + self.coastlines = coastlines + self.x_label = "Longitude" + self.y_label = "Latitude" + self.data_label = units + self.timestamp = timestamp + + def __add__(self, arg): + """Allows LonLatMap objects to be added together.""" + self._compatible(arg) + return LonLatMap(self.data + arg.data, self.x_data, self.y_data, + units=self.data_label, projection=self.projection, + coastlines=self.coastlines, add_cyclic_point=False, + timestamp=self.timestamp) + + def __sub__(self, arg): + """Allows LonLatMap objects to be subtracted from one another.""" + self._compatible(arg) + return LonLatMap(self.data - arg.data, self.x_data, self.y_data, + units=self.data_label, projection=self.projection, + coastlines=self.coastlines, add_cyclic_point=False, + timestamp=self.timestamp) + + @classmethod + def from_xarray_dataset(cls, dataset, variable, time_method=None, time_index=None, + year=None): + """Instantiates a LonLatMap object from an xarray dataset.""" + v = dataset.data_vars[variable] + data = array(v.data[...]) + axis_attrs = _dimension_order(dataset, v) + longitude = array(dataset.coords[v.dims[-1]].data[...]) + latitude = array(dataset.coords[v.dims[-2]].data[...]) + + if axis_attrs[0] == "t": + time = array(dataset.coords[v.dims[0]].data[...]) + if time_method == "instantaneous": + if time_method == None: + raise ValueError("time_index is required when time_method='instantaneous.'") + data = data[time_index, ...] + timestamp = f"@ {str(time[time_index])}" + elif time_method == "annual mean": + if year == None: + raise ValueError("year is required when time_method='annual mean'.") + time = TimeSubset(time) + data = time.annual_mean(data, year) + timestamp = r"$\bar{t} = $" + str(year) + else: + raise ValueError("time_method must be either 'instantaneous' or 'annual mean.'") + else: + timestamp = None + + return cls(data, longitude, latitude, units=v.attrs["units"], timestamp=timestamp) + + def global_mean(self): + """Performs a global mean over the longitude and latitude dimensions. + + Returns: + Gobal mean value. + """ + + weights = cos(2.*pi*self.y_data/360.) + return sum(mean(self.data, axis=-1)*weights, axis=-1)/sum(weights) + + def regrid_to_map(self, map_): + """Regrid the data to match in the input map. + + Args: + map_: A LonLatMap to regrid to. + """ + if not isinstance(map_, LonLatMap): + raise TypeError("input map must be a LonLatMap.") + try: + self._compatible(map_) + return + except ValueError: + da = DataArray(self.data, dims=["y", "x"], + coords={"x": self.x_data, "y": self.y_data}) + da2 = DataArray(map_.data, dims=["y", "x"], + coords={"x": map_.x_data, "y": map_.y_data}) + self.data = array(da.interp_like(da2, kwargs={"fill_value": "extrapolate"})) + self.x_data = map_.x_data + self.y_data = map_.y_data + + def _compatible(self, arg): + """Raises a ValueError if two objects are not compatible.""" + if not isinstance(arg, LonLatMap): + raise TypeError("input map must be a LonLatMap.") + for attr in ["x_data", "y_data", "projection", "data_label"]: + if isinstance(getattr(self, attr), ndarray): + equal = array_equal(getattr(self, attr), getattr(arg, attr)) + else: + equal = getattr(self, attr) == getattr(arg, attr) + if not equal: + raise ValueError(f"The same {attr} is required for both objects.") + + +def _dimension_order(dataset, variable): + """Raises a ValueError if the variable's dimensoins are not in an expected order. + + Returns: + A list of the dimension axis attribute strings. + """ + axis_attrs = [dataset.coords[x].attrs["axis"].lower() + if "axis" in dataset.coords[x].attrs else None + for x in variable.dims] + allowed = [x + ["y", "x"] for x in [["t",], ["z",], [None,], []]] + for config in allowed: + if axis_attrs == config: + return axis_attrs + raise ValueError(f"variable {variable} contains unexpected axes ordering {axis_attrs}.") diff --git a/figure_tools/figure_tools/time_subsets.py b/figure_tools/figure_tools/time_subsets.py new file mode 100644 index 0000000..a5c89ad --- /dev/null +++ b/figure_tools/figure_tools/time_subsets.py @@ -0,0 +1,72 @@ +from numpy import array, datetime64, mean, zeros + + +class TimeSubset(object): + def __init__(self, data): + """Instantiates an object. + + Args: + data: An xarray DataArray for the time dimension of an xarray Dataset. + """ + self.data = data + + def annual_mean(self, data, year): + """Calculates the annual mean of the input date for the input year. + + Args: + data: Numpy array of data to be averaged. + year: Integer year to average over. + """ + start, end = None, None + for i, point in enumerate(self.data): + month, y = self._month_and_year(point) + if y == year: + if month == 1: + start = i + elif month == 12: + end = i + 1 + if None not in [start, end]: break + else: + raise ValueError(f"could not find year {year}.") + return mean(array(data[start:end, ...]), axis=0) + + def annual_means(self, data): + """Calculates the annual means of the input date for each year. + + Args: + data: Numpy array of data to be averaged. + + Returns: + Numpy array of years that were averaged over and a numpy array of the + average data. + """ + years = {} + for i, point in enumerate(self.data): + month, year = self._month_and_year(point) + if year not in years: + years[year] = [None, None] + if month == 1: + years[year][0] = i + elif month == 12: + years[year][1] = i + 1 + + years_data = sorted(years.keys()) + means_data = zeros(tuple([len(years_data),] + list(data.shape[1:]))) + for i, key in enumerate(years_data): + start, end = years[key] + means_data[i, ...] = mean(array(data[start:end, ...]), axis=0) + return array(years_data), means_data + + def _month_and_year(self, time): + """Returns the integer month and year for the input time point. + + Args: + Integer month and year values. + """ + if isinstance(time, datetime64): + year = time.astype("datetime64[Y]").astype(int) + 1970 + month = time.astype("datetime64[M]").astype(int) % 12 + 1 + else: + year = time.year + month = time.month + return month, year diff --git a/figure_tools/figure_tools/zonal_mean_map.py b/figure_tools/figure_tools/zonal_mean_map.py new file mode 100644 index 0000000..29af2b1 --- /dev/null +++ b/figure_tools/figure_tools/zonal_mean_map.py @@ -0,0 +1,86 @@ +from numpy import array, array_equal, mean, ndarray + +from .time_subsets import TimeSubset + + +class ZonalMeanMap(object): + def __init__(self, data, latitude, y_axis_data, units=None, y_label=None, + invert_y_axis=False, timestamp=None): + self.data = data[...] + self.x_data = latitude[...] + self.y_data = y_axis_data[...] + self.invert_y_axis = invert_y_axis + self.x_label = "Latitude" + self.y_label = y_label + self.data_label = units + self.timestamp = timestamp + + def __add__(self, arg): + self._compatible(arg) + return ZonalMeanMap(self.data + arg.data, self.x_data, self.y_data, + units=self.data_label, y_label=self.y_label, + invert_y_axis=self.invert_y_axis, timestamp=self.timestamp) + + def __sub__(self, arg): + self._compatible(arg) + return ZonalMeanMap(self.data - arg.data, self.x_data, self.y_data, + units=self.data_label, y_label=self.y_label, + invert_y_axis=self.invert_y_axis, timestamp=self.timestamp) + + @classmethod + def from_xarray_dataset(cls, dataset, variable, time_method=None, time_index=None, + year=None, y_axis=None, y_label=None, invert_y_axis=False): + """Instantiates a ZonalMeanMap object from an xarray dataset.""" + v = dataset.data_vars[variable] + data = array(v.data[...]) + axis_attrs = _dimension_order(dataset, v) + latitude = array(dataset.coords[v.dims[-2]].data[...]) + y_dim = array(dataset.coords[v.dims[-3]].data[...]) + y_dim_units = y_label or dataset.coords[v.dims[-3]].attrs["units"] + + if axis_attrs[0] == "t": + time = array(dataset.coords[v.dims[0]].data[...]) + if time_method == "instantaneous": + if time_method == None: + raise ValueError("time_index is required when time_method='instantaneous.'") + data = data[time_index, ...] + timestamp = str(time[time_index]) + elif time_method == "annual mean": + if year == None: + raise ValueError("year is required when time_method='annual mean'.") + time = TimeSubset(time) + data = time.annual_mean(data, year) + timestamp = str(year) + else: + raise ValueError("time_method must be either 'instantaneous' or 'annual mean'.") + else: + timestamp = None + + return cls(mean(data, -1), latitude, y_dim, v.attrs["units"], y_dim_units, + invert_y_axis, timestamp) + + def _compatible(self, arg): + """Raises a ValueError if two objects are not compatible.""" + for attr in ["x_data", "y_data", "invert_y_axis", "y_label", "data_label"]: + if isinstance(getattr(self, attr), ndarray): + equal = array_equal(getattr(self, attr), getattr(arg, attr)) + else: + equal = getattr(self, attr) == getattr(arg, attr) + if not equal: + raise ValueError(f"The same {attr} is required for both objects.") + + +def _dimension_order(dataset, variable): + """Raises a ValueError if the variable's dimensions are not in an expected order. + + Returns: + A list of the dimension axis attribute strings. + """ + axis_attrs = [dataset.coords[x].attrs["axis"].lower() + if "axis" in dataset.coords[x].attrs else None + for x in variable.dims] + allowed = [x + ["y", "x"] for x in [["z",], [None,], ["t", "z"], ["t", None]]] + for config in allowed: + if axis_attrs == config: + return axis_attrs + raise ValueError(f"variable contains unexpected axes ordering {axis_attrs}.") diff --git a/figure_tools/pyproject.toml b/figure_tools/pyproject.toml new file mode 100644 index 0000000..5d674f2 --- /dev/null +++ b/figure_tools/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = [ + "setuptools >= 40.9.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "figure_tools" +version = "0.1" +dependencies = [ + "cartopy", + "matplotlib", + "numpy", + "xarray", +] +requires-python = ">= 3.6" +authors = [ + {name = "developers"}, +] +maintainers = [ + {name = "developer", email = "developer-email@address.domain"}, +] +description = "Helper tools to make common plots" +readme = "README.md" +classifiers = [ + "Programming Language :: Python" +] + +[project.urls] +repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/freanalysis_aerosol/README.md b/freanalysis_aerosol/README.md new file mode 100644 index 0000000..6c0811e --- /dev/null +++ b/freanalysis_aerosol/README.md @@ -0,0 +1,16 @@ +# freanalysis_aerosol +A plugin for analyzing GFDL model aerosol mass output. + +### Motivation +To create figures to analyze the aerosol mass output from GFDL models. + +### Requirements +The software packages that are required are: + +- analysis-scripts +- figure_tools +- intake +- intake-esm + +### How to use this plugin +This plugin is designed to be used by the freanalysis package. diff --git a/freanalysis_aerosol/freanalysis_aerosol/__init__.py b/freanalysis_aerosol/freanalysis_aerosol/__init__.py new file mode 100644 index 0000000..613ed76 --- /dev/null +++ b/freanalysis_aerosol/freanalysis_aerosol/__init__.py @@ -0,0 +1,209 @@ +from dataclasses import dataclass +import json +from pathlib import Path + +from analysis_scripts import AnalysisScript +from figure_tools import LonLatMap, zonal_mean_vertical_and_column_integrated_map, \ + ZonalMeanMap +import intake + + +@dataclass +class Metadata: + """Helper class that stores the metadata needed by the plugin.""" + frequency: str = "monthly" + realm: str = "atmos" + + @staticmethod + def variables(): + """Helper function to make maintaining this script easier if the + catalog variable ids change. + + Returns: + Dictionary mapping the names used in this script to the catalog + variable ids. + """ + return { + "black_carbon": "blk_crb", + "black_carbon_column": "blk_crb_col", + "large_dust": "lg_dust", + "large_dust_column": "lg_dust_col", + "small_dust": "sm_dust", + "small_dust_column": "sm_dust_col", + "organic_carbon": "org_crb", + "organic_carbon_column": "org_crb_col", + "large_seasalt": "lg_ssalt", + "large_seasalt_column": "lg_ssalt_col", + "small_seasalt": "sm_ssalt", + "small_seasalt_column": "sm_ssalt_col", + "seasalt": "salt", + "seasalt_column": "salt_col", + "sulfate": "sulfate", + "sulfate_column": "sulfate_col", + } + + +class AerosolAnalysisScript(AnalysisScript): + """Aerosol analysis script. + + Attributes: + description: Longer form description for the analysis. + title: Title that describes the analysis. + """ + def __init__(self): + self.metadata = Metadata() + self.description = "Calculates aerosol mass metrics." + self.title = "Aerosol Masses" + + def requires(self): + """Provides metadata describing what is needed for this analysis to run. + + Returns: + A json string containing the metadata. + """ + columns = Metadata.__annotations__.keys() + settings = {x: getattr(self.metadata, x) for x in columns} + return json.dumps({ + "settings": settings, + "dimensions": { + "lat": {"standard_name": "latitude"}, + "lon": {"standard_name": "longitude"}, + "pfull": {"standard_name": "air_pressure"}, + "time": {"standard_name": "time"} + }, + "varlist": { + "blk_crb": { + "standard_name": "black_carbon_mass", + "units": "kg m-3", + "dimensions": ["time", "pfull", "lat", "lon"] + }, + "blk_crb_col": { + "standard_name": "column_integrated_black_carbon_mass", + "units": "kg m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lg_dust": { + "standard_name": "large_dust_mass", + "units": "kg m-3", + "dimensions": ["time", "pfull", "lat", "lon"] + }, + "lg_dust_col": { + "standard_name": "column_integrated_large_dust_mass", + "units": "kg m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lg_ssalt": { + "standard_name": "large_seasalt_mass", + "units": "kg m-3", + "dimensions": ["time", "pfull", "lat", "lon"] + }, + "lg_ssalt_col": { + "standard_name": "column_integrated_large_ssalt_mass", + "units": "kg m-2", + "dimensions": ["time", "lat", "lon"] + }, + "org_crb": { + "standard_name": "organic_carbon_mass", + "units": "kg m-3", + "dimensions": ["time", "pfull", "lat", "lon"] + }, + "org_crb_col": { + "standard_name": "column_integrated_organic_carbon_mass", + "units": "kg m-2", + "dimensions": ["time", "lat", "lon"] + }, + "salt": { + "standard_name": "seasalt_mass", + "units": "kg m-3", + "dimensions": ["time", "pfull", "lat", "lon"] + }, + "salt_col": { + "standard_name": "column_integrated_seasalt_mass", + "units": "kg m-2", + "dimensions": ["time", "lat", "lon"] + }, + "sm_dust": { + "standard_name": "small_dust_mass", + "units": "kg m-3", + "dimensions": ["time", "pfull", "lat", "lon"] + }, + "sm_dust_col": { + "standard_name": "column_integrated_small_dust_mass", + "units": "kg m-2", + "dimensions": ["time", "lat", "lon"] + }, + "sm_ssalt": { + "standard_name": "small_seasalt_mass", + "units": "kg m-3", + "dimensions": ["time", "pfull", "lat", "lon"] + }, + "sm_ssalt_col": { + "standard_name": "column_integrated_small_ssalt_mass", + "units": "kg m-2", + "dimensions": ["time", "lat", "lon"] + }, + "sulfate": { + "standard_name": "sulfate_mass", + "units": "kg m-3", + "dimensions": ["time", "pfull", "lat", "lon"] + }, + "sulfate_col": { + "standard_name": "column_integrated_sulfate_mass", + "units": "kg m-2", + "dimensions": ["time", "lat", "lon"] + }, + }, + }) + + def run_analysis(self, catalog, png_dir, reference_catalog=None, config={}): + """Runs the analysis and generates all plots and associated datasets. + + Args: + catalog: Path to a catalog. + png_dir: Path to the directory where the figures will be made. + reference_catalog: Path to a catalog of reference data. + config: Dictionary of catalog metadata. Will overwrite the + data defined in the Metadata helper class if they both + contain the same keys. + + Returns: + A list of paths to the figures that were created. + + Raises: + ValueError if the catalog cannot be filtered correctly. + """ + + # Connect to the catalog and find the necessary datasets. + catalog = intake.open_esm_datastore(catalog) + + maps = {} + for name, variable in self.metadata.variables().items(): + # Filter the catalog down to a single dataset for each variable. + query_params = {"variable_id": variable} + query_params.update(vars(self.metadata)) + query_params.update(config) + datasets = catalog.search(**query_params).to_dataset_dict(progressbar=False) + if len(list(datasets.values())) != 1: + raise ValueError("could not filter the catalog down to a single dataset.") + dataset = list(datasets.values())[0] + + if name.endswith("column"): + # Lon-lat maps. + maps[name] = LonLatMap.from_xarray_dataset(dataset, variable, year=1980, + time_method="annual mean") + else: + maps[name] = ZonalMeanMap.from_xarray_dataset(dataset, variable, year=1980, + time_method="annual mean", + invert_y_axis=True) + + figure_paths = [] + for name in self.metadata.variables().keys(): + if name.endswith("column"): continue + figure = zonal_mean_vertical_and_column_integrated_map( + maps[name], + maps[f"{name}_column"], + f"{name.replace('_', ' ')} Mass", + ) + figure.save(Path(png_dir) / f"{name}.png") + figure_paths.append(Path(png_dir)/ f"{name}.png") + return figure_paths \ No newline at end of file diff --git a/freanalysis_aerosol/pyproject.toml b/freanalysis_aerosol/pyproject.toml new file mode 100644 index 0000000..55c8be8 --- /dev/null +++ b/freanalysis_aerosol/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ + "setuptools >= 40.9.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "freanalysis_aerosol" +version = "0.1" +dependencies = [ + "intake", + "intake-esm", +] +requires-python = ">= 3.6" +authors = [ + {name = "developers"}, +] +maintainers = [ + {name = "developer", email = "developer-email@address.domain"}, +] +description = "Aerosol mass analyzer for GFDL model output" +readme = "README.md" +classifiers = [ + "Programming Language :: Python" +] + +[project.urls] +repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/freanalysis_clouds/README.md b/freanalysis_clouds/README.md new file mode 100644 index 0000000..2003180 --- /dev/null +++ b/freanalysis_clouds/README.md @@ -0,0 +1,16 @@ +# freanalysis_clouds +A plugin for analyzing GFDL model clouds output. + +### Motivation +To create figures to analyze the clouds output from GFDL models. + +### Requirements +The software packages that are required are: + +- analysis-scripts +- figure_tools +- intake +- intake-esm + +### How to use this plugin +This plugin is designed to be used by the freanalysis package. diff --git a/freanalysis_clouds/freanalysis_clouds/__init__.py b/freanalysis_clouds/freanalysis_clouds/__init__.py new file mode 100644 index 0000000..4b15c52 --- /dev/null +++ b/freanalysis_clouds/freanalysis_clouds/__init__.py @@ -0,0 +1,122 @@ +from dataclasses import dataclass +import json +from pathlib import Path + +from analysis_scripts import AnalysisScript +from figure_tools import Figure, LonLatMap +import intake + + +@dataclass +class Metadata: + """Helper class that stores the metadata needed by the plugin.""" + frequency: str = "mon" + realm: str = "atmos" + + @staticmethod + def variables(): + """Helper function to make maintaining this script easier if the + catalog variable ids change. + + Returns: + Dictionary mapping the names used in this script to the catalog + variable ids. + """ + return { + "high_cloud_fraction": "high_cld_amt", + "low_cloud_fraction": "low_cld_amt", + "middle_cloud_fraction": "mid_cld_amt", + } + + +class CloudAnalysisScript(AnalysisScript): + """Cloud analysis script. + + Attributes: + description: Longer form description for the analysis. + title: Title that describes the analysis. + """ + def __init__(self): + self.metadata = Metadata() + self.description = "Calculates cloud metrics." + self.title = "Cloud Fractions" + + def requires(self): + """Provides metadata describing what is needed for this analysis to run. + + Returns: + A json string containing the metadata. + """ + columns = Metadata.__annotations__.keys() + settings = {x: getattr(self.metadata, x) for x in columns} + return json.dumps({ + "settings": settings, + "dimensions": { + "lat": {"standard_name": "latitude"}, + "lon": {"standard_name": "longitude"}, + "time": {"standard_name": "time"} + }, + "varlist": { + "high_cld_amt": { + "standard_name": "high_cloud_fraction", + "units": "%", + "dimensions": ["time", "lat", "lon"] + }, + "low_cld_amt": { + "standard_name": "low_cloud_fraction", + "units": "%", + "dimensions": ["time", "lat", "lon"] + }, + "mid_cld_amt": { + "standard_name": "middle_cloud_fraction", + "units": "%", + "dimensions": ["time", "lat", "lon"] + }, + }, + }) + + def run_analysis(self, catalog, png_dir, reference_catalog=None, config={}): + """Runs the analysis and generates all plots and associated datasets. + + Args: + catalog: Path to a catalog. + png_dir: Path to the directory where the figures will be made. + reference_catalog: Path to a catalog of reference data. + config: Dictonary of catalog metadata. Will overwrite the + data defined in the Metadata helper class if they both + contain the same keys. + + Returns: + A list of paths to the figures that were created. + + Raises: + ValueError if the catalog cannot be filtered correctly. + """ + + # Connect to the catalog. + catalog = intake.open_esm_datastore(catalog) + print(catalog) + + maps = {} + for name, variable in self.metadata.variables().items(): + # Filter the catalog down to a single dataset for each variable. + query_params = {"variable_id": variable} + query_params.update(vars(self.metadata)) + query_params.update(config) + datasets = catalog.search(**query_params).to_dataset_dict(progressbar=False) + if len(list(datasets.values())) != 1: + raise ValueError("could not filter the catalog down to a single dataset.", datasets) + dataset = list(datasets.values())[0] + + # Create Lon-lat maps. + maps[name] = LonLatMap.from_xarray_dataset(dataset, variable, year=1980, + time_method="annual mean") + + # Create the figure. + figure = Figure(num_rows=3, num_columns=1, title="Cloud Fraction", size=(16, 10)) + figure.add_map(maps["high_cloud_fraction"], "High Clouds", 1, colorbar_range= [0, 100]) + figure.add_map(maps["middle_cloud_fraction"], "Middle Clouds", 2, colorbar_range=[0, 100]) + figure.add_map(maps["low_cloud_fraction"], "Low Clouds", 3, colorbar_range=[0, 100]) + output = Path(png_dir) / "cloud-fraction.png" + figure.save(output) + return [output,] diff --git a/freanalysis_clouds/pyproject.toml b/freanalysis_clouds/pyproject.toml new file mode 100644 index 0000000..634b0ac --- /dev/null +++ b/freanalysis_clouds/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ + "setuptools >= 40.9.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "freanalysis_clouds" +version = "0.1" +dependencies = [ + "intake", + "intake-esm", +] +requires-python = ">= 3.6" +authors = [ + {name = "developers"}, +] +maintainers = [ + {name = "developer", email = "developer-email@address.domain"}, +] +description = "Cloud fraction analyzer for GFDL model output" +readme = "README.md" +classifiers = [ + "Programming Language :: Python" +] + +[project.urls] +repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/freanalysis_land/README.md b/freanalysis_land/README.md new file mode 100644 index 0000000..ca4e23a --- /dev/null +++ b/freanalysis_land/README.md @@ -0,0 +1 @@ +This is the README. \ No newline at end of file diff --git a/freanalysis_land/freanalysis_land/__init__.py b/freanalysis_land/freanalysis_land/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/freanalysis_land/freanalysis_land/land.py b/freanalysis_land/freanalysis_land/land.py new file mode 100644 index 0000000..0bd3f0a --- /dev/null +++ b/freanalysis_land/freanalysis_land/land.py @@ -0,0 +1,243 @@ +import json +from analysis_scripts import AnalysisScript +import intake +import matplotlib.pyplot as plt +import cartopy +import cartopy.crs as ccrs +import datetime +import pandas as pd +import re +import xarray as xr + + +class LandAnalysisScript(AnalysisScript): + """A class for performing various analysis tasks relating to GFDL land model output, + inherits from the AnalysisScipt base class. + + Attributes: + description: Longer form description for the analysis. + title: Title that describes the analysis. + """ + def __init__(self): + """Instantiates an object. The user should provide a description and title.""" + self.description = "This is for analysis of land model (stand-alone)" + self.title = "Soil Carbon" + + def requires(self): + """Provides metadata describing what is needed for this analysis to run. + + Returns: + A json string describing the metadata. + """ + raise NotImplementedError("you must override this function.") + return json.dumps("{json of metadata MDTF format.}") + + def global_map(self,dataset,var,dates,plt_time=None,colormap='viridis',title=''): + """ + Generate a global map and regional subplots for a specified variable from an xarray dataset. + + This function creates a global map and several regional subplots (North America, South America, Europe, Africa, Asia, and Australia) + using the specified variable from an xarray dataset. The generated map will be saved as a PNG file. + + Parameters: + ---------- + dataset : xarray.Dataset + The input xarray dataset containing the variable to be plotted. + + var : str + The name of the variable in the dataset to be plotted. + + dates: list + The list of dates from the dataframe, converted to period index. + + plt_time : int, optional + The time index to plot from the variable data. Defaults to the length of `dates` - 1, or last date in dataset. + + colormap : str, optional + The colormap to use for plotting the data. Defaults to 'viridis'. + + title : str, optional + The title for the figure. Defaults to an empty string. + + Returns: + ------- + fig : matplotlib figure object + The function returns the plot for saving + + Notes: + ----- + The function uses Cartopy for map projections and Matplotlib for plotting. + Ensure Cartopy and Matplotlib are installed in your Python environment. + + The output file is saved with the format '_global_map.png'. + + Examples: + -------- + global_map(my_dataset, 'mrso', plt_time=0, colormap='coolwarm', title='Global Temperature Map') + """ + lon = dataset.lon + lat = dataset.lat + + if plt_time is None: plt_time=len(dates)-1 + + data = dataset[var][plt_time].values + projection = ccrs.PlateCarree() + fig = plt.figure(figsize=(8.5, 11)) + + # Global map + ax_global = fig.add_subplot(3, 1, 1, projection=projection) + ax_global.set_title('Global Map') + mesh = ax_global.pcolormesh(lon, lat, data, transform=projection, cmap=colormap) + ax_global.coastlines() + + # List of bounding boxes for different continents (min_lon, max_lon, min_lat, max_lat) + regions = { + 'North America': [-170, -47, 0, 85], + 'South America': [-90, -30, -60, 15], + 'Europe': [-10, 60, 30, 75], + 'Africa': [-20, 50, -35, 37], + 'Asia': [60, 150, 5, 75], + 'Australia': [110, 180, -50, 0] + } + + # Create subplots for each region + for i, (region, bbox) in enumerate(regions.items(), start=1): + ax = fig.add_subplot(3, 3, i + 3, projection=projection) + ax.set_extent(bbox, crs=projection) + ax.set_title(region) + ax.pcolormesh(lon, lat, data, transform=projection, cmap=colormap) + ax.coastlines() + ax.add_feature(cartopy.feature.BORDERS) + + # Add colorbar + fig.colorbar(mesh, ax=ax_global, orientation='horizontal', pad=0.05, aspect=50) + plt.suptitle(title) + plt.tight_layout() + + return fig + # plt.savefig(var+'_global_map.png') + # plt.close() + + def timeseries(self, dataset,var,dates_period,var_range=None,minlon = 0,maxlon = 360,minlat = -90,maxlat=90,timerange=None,title=''): + ''' + Generate a time series plot of the specified variable from a dataset within a given geographic and temporal range. + + + Parameters: + ----------- + dataset : xarray.Dataset + The dataset containing the variable to be plotted. + var : str + The name of the variable to plot from the dataset. + dates_period : pandas.DatetimeIndex + The dates for the time series data. + var_range : tuple of float, optional + The range of variable values to include in the plot (min, max). If not provided, the default range is (0, inf). + minlon : float, optional + The minimum longitude to include in the plot. Default is 0. + maxlon : float, optional + The maximum longitude to include in the plot. Default is 360. + minlat : float, optional + The minimum latitude to include in the plot. Default is -90. + maxlat : float, optional + The maximum latitude to include in the plot. Default is 90. + timerange : tuple of int, optional + The range of years to plot (start_year, end_year). If not provided, all available years in the dataset will be plotted. + title : str, optional + The title of the plot. Default is an empty string. + + Returns: + -------- + matplotlib.figure.Figure + The figure object containing the generated time series plot. + + Notes: + ------ + The function filters the dataset based on the provided variable range, longitude, and latitude bounds. It then + calculates the monthly and annual means of the specified variable and plots the seasonal and annual means. + + ''' + if var_range is not None: + data_filtered = dataset.where((dataset[var] > var_range[0]) & (dataset[var] <= var_range[1]) & + (dataset.lat >= minlat) & (dataset.lon >= minlon) & + (dataset.lat <= maxlat) & (dataset.lon <= maxlon)) + else: + data_filtered = dataset.where((dataset[var] > 0) & + (dataset.lat >= minlat) & (dataset.lon >= minlon) & + (dataset.lat <= maxlat) & (dataset.lon <= maxlon)) + data_filtered['time'] = dates_period + + data_df = pd.DataFrame(index = dates_period) + data_df['monthly_mean'] = data_filtered.resample(time='YE').mean(dim=['lat','lon'],skipna=True)[var].values + data_df['monthly_shift'] = data_df['monthly_mean'].shift(1) + + if timerange is not None: + ys, ye = (str(timerange[0]),str(timerange[1])) + plot_df = data_df.loc[f'{ys}-1-1':f'{ye}-1-1'] + else: + plot_df = data_df + + fig, ax = plt.subplots() + plot_df.resample('Q').mean()['monthly_shift'].plot(ax=ax,label='Seasonal Mean') + plot_df.resample('Y').mean()['monthly_mean'].plot(ax=ax,label='Annual Mean') + plt.legend() + plt.title(title) + plt.xlabel('Years') + return fig + + def run_analysis(self, catalog, png_dir, reference_catalog=None): + """Runs the analysis and generates all plots and associated datasets. + + Args: + catalog: Path to a model output catalog. + png_dir: Directory to store output png figures in. + reference_catalog: Path to a catalog of reference data. + + Returns: + A list of png figures. + """ + print ('WARNING: THESE FIGURES ARE FOR TESTING THE NEW ANALYSIS WORKFLOW ONLY AND SHOULD NOT BE USED IN ANY OFFICIAL MANNER FOR ANALYSIS OF LAND MODEL OUTPUT.') + col = intake.open_esm_datastore(catalog) + df = col.df + + # Soil Carbon + var = 'cSoil' + print ('Soil Carbon Analysis') + cat = col.search(variable_id=var,realm='land_cmip') + other_dict = cat.to_dataset_dict(cdf_kwargs={'chunks':{'time':12},'decode_times':False}) + combined_dataset = xr.concat(list(dict(sorted(other_dict.items())).values()), dim='time') + + # Other data: + # land_static_file = re.search('land_static:\s([\w.]*)',combined_dataset.associated_files).group(1) + # STATIC FILES SHOULD BE PART OF THE CATALOG FOR EASY ACCESS + + # Select Data and plot + dates = [datetime.date(1,1,1) + datetime.timedelta(d) for d in combined_dataset['time'].values] # Needs to be made dynamic + dates_period = pd.PeriodIndex(dates,freq='D') + + sm_fig = self.global_map(combined_dataset,var,dates,title='Soil Carbon Content (kg/m^2)') + plt.savefig(png_dir+var+'_global_map.png') + plt.close() + + ts_fig = self.timeseries(combined_dataset,var,dates_period,title='Global Average Soil Carbon') + plt.savefig(png_dir+var+'_global_ts.png') + plt.close() + + # Soil Moisture + var = 'mrso' + print ('Soil Moisture Analysis') + cat = col.search(variable_id=var,realm='land_cmip') + other_dict = cat.to_dataset_dict(cdf_kwargs={'chunks':{'time':12},'decode_times':False}) + combined_dataset = xr.concat(list(dict(sorted(other_dict.items())).values()), dim='time') + + # Other data: + # soil_area_file = re.search('soil_area:\s([\w.]*)',combined_dataset.associated_files).group(1) + # STATIC FILES SHOULD BE PART OF THE CATALOG FOR EASY ACCESS + + # Select Data and plot + dates = [datetime.date(1,1,1) + datetime.timedelta(d) for d in combined_dataset['time'].values] # Needs to be made dynamic + dates_period = pd.PeriodIndex(dates,freq='D') + + sm_fig = self.global_map(combined_dataset,var,dates,title='Soil Moisture (kg/m^2)') + plt.savefig(png_dir+var+'_global_map.png') + plt.close() diff --git a/freanalysis_land/pyproject.toml b/freanalysis_land/pyproject.toml new file mode 100644 index 0000000..c0cc45d --- /dev/null +++ b/freanalysis_land/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = [ + "setuptools >= 40.9.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "freanalysis_land" +version = "0.1" +dependencies = [ + "setuptools", + "intake", + "intake-esm", + "xarray", + "matplotlib", + "cartopy", + "pandas", + "xarray" + +] +requires-python = ">= 3.6" +authors = [ + {name = "developers"}, +] +maintainers = [ + {name = "Anthony Preucil", email = "Anthony.Preucil@noaa.gov"}, +] +description = "Land Analysis for GFDL stand-alone model" +readme = "README.md" +classifiers = [ + "Programming Language :: Python" +] + +[project.urls] +repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/freanalysis_radiation/README.md b/freanalysis_radiation/README.md new file mode 100644 index 0000000..31120ee --- /dev/null +++ b/freanalysis_radiation/README.md @@ -0,0 +1,16 @@ +# freanalysis_radiation +A plugin for analyzing GFDL model radiative flux output. + +### Motivation +To create figures to analyze the radiative flux output from GFDL models. + +### Requirements +The software packages that are required are: + +- analysis-scripts +- figure_tools +- intake +- intake-esm + +### How to use this plugin +This plugin is designed to be used by the freanalysis package. diff --git a/freanalysis_radiation/freanalysis_radiation/__init__.py b/freanalysis_radiation/freanalysis_radiation/__init__.py new file mode 100644 index 0000000..827cc18 --- /dev/null +++ b/freanalysis_radiation/freanalysis_radiation/__init__.py @@ -0,0 +1,309 @@ +from dataclasses import dataclass +import json +from pathlib import Path + +from analysis_scripts import AnalysisScript +from figure_tools import AnomalyTimeSeries, GlobalMeanTimeSeries, LonLatMap, \ + observation_vs_model_maps, radiation_decomposition, \ + timeseries_and_anomalies +import intake +import intake_esm +from xarray import open_dataset + + +@dataclass +class Metadata: + activity_id: str = "dev" + institution_id: str = "" + source_id: str = "am5" + experiment_id: str = "c96L65_am5f7b11r0_amip" + frequency: str = "P1M" + modeling_realm: str = "atmos" + table_id: str = "" + member_id: str = "na" + grid_label: str = "" + temporal_subset: str = "" + chunk_freq: str = "" + platform: str = "" + cell_methods: str = "" + chunk_freq: str = "P1Y" + + def catalog_search_args(self, name): + return { + "experiment_id": self.experiment_id, + "frequency": self.frequency, + "member_id": self.member_id, + "modeling_realm": self.modeling_realm, + "variable_id": name, + } + + def variables(self): + return { + "rlds": "lwdn_sfc", + "rldsaf": "lwdn_sfc_ad", + "rldscs": "lwdn_sfc_clr", + "rldscsaf": "lwdn_sfc_ad_clr", + "rlus": "lwup_sfc", + "rlusaf": "lwup_sfc_ad", + "rluscs": "lwup_sfc_clr", + "rluscsaf": "lwup_sfc_ad_clr", + "rlut": "olr", + "rlutaf": "lwtoa_ad", + "rlutcs": "olr_clr", + "rlutcsaf": "lwtoa_ad_clr", + "rsds": "swdn_sfc", + "rsdsaf": "swdn_sfc_ad", + "rsdscs": "swdn_sfc_clr", + "rsdscsaf": "swdn_sfc_ad_clr", + "rsus": "swup_sfc", + "rsusaf": "swup_sfc_ad", + "rsuscs": "swup_sfc_clr", + "rsuscsaf": "swup_sfc_ad_clr", + "rsut": "swup_toa", + "rsutaf": "swup_toa_ad", + "rsutcs": "swup_toa_clr", + "rsutcsaf": "swup_toa_ad_clr", + "rsdt": "swdn_toa", + } + + +class RadiationAnalysisScript(AnalysisScript): + """Abstract base class for analysis scripts. User-defined analysis scripts + should inhert from this class and override the requires and run_analysis methods. + + Attributes: + description: Longer form description for the analysis. + title: Title that describes the analysis. + """ + def __init__(self): + self.metadata = Metadata() + self.description = "Calculates radiative flux metrics." + self.title = "Radiative Fluxes" + + def requires(self): + """Provides metadata describing what is needed for this analysis to run. + + Returns: + A json string containing the metadata. + """ + columns = Metadata.__annotations__.keys() + settings = {x: getattr(self.metadata, x) for x in columns} + return json.dumps({ + "settings": settings, + "dimensions": { + "lat": {"standard_name": "latitude"}, + "lon": {"standard_name": "longitude"}, + "time": {"standard_name": "time"} + }, + "varlist": { + "lwup_sfc": { + "standard_name": "surface_outgoing_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lwup_sfc_ad": { + "standard_name": "surface_outgoing_aerosol_free_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lwup_sfc_clr": { + "standard_name": "surface_outgoing_clear_sky_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lwup_sfc_ad_clr": { + "standard_name": "surface_outgoing_clear_sky_aerosol_free_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lwdn_sfc": { + "standard_name": "surface_incoming_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lwdn_sfc_ad": { + "standard_name": "surface_incoming_aerosol_free_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lwdn_sfc_clr": { + "standard_name": "surface_incoming_clear_sky_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lwdn_sfc_ad_clr": { + "standard_name": "surface_incoming_clear_sky_aerosol_free_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "olr": { + "standard_name": "toa_outgoing_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lwtoa_ad": { + "standard_name": "toa_outgoing_aerosol_free_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "olr_clr": { + "standard_name": "toa_outgoing_clear_sky_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "lwtoa_ad_clr": { + "standard_name": "toa_outgoing_clear_sky_aerosol_free_longwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swup_sfc": { + "standard_name": "surface_outgoing_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swup_sfc_ad": { + "standard_name": "surface_outgoing_aerosol_free_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swup_sfc_clr": { + "standard_name": "surface_outgoing_clear_sky_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swup_sfc_ad_clr": { + "standard_name": "surface_outgoing_clear_sky_aerosol_free_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swdn_sfc": { + "standard_name": "surface_incoming_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swdn_sfc_ad": { + "standard_name": "surface_incoming_aerosol_free_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swdn_sfc_clr": { + "standard_name": "surface_incoming_clear_sky_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swdn_sfc_ad_clr": { + "standard_name": "surface_incoming_clear_sky_aerosol_free_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swup_toa": { + "standard_name": "toa_outgoing_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swup_toa_ad": { + "standard_name": "toa_outgoing_aerosol_free_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swup_toa_clr": { + "standard_name": "toa_outgoing_clear_sky_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swup_toa_ad_clr": { + "standard_name": "toa_outgoing_clear_sky_aerosol_free_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + "swdn_toa": { + "standard_name": "toa_downwelling_shortwave_flux", + "units": "W m-2", + "dimensions": ["time", "lat", "lon"] + }, + }, + }) + + def run_analysis(self, catalog, png_dir, reference_catalog=None): + """Runs the analysis and generates all plots and associated datasets. + + Args: + catalog: Path to a catalog. + png_dir: Path to the directory where the figures will be made. + reference_catalog: Path to a catalog of reference data. + + Returns: + A list of paths to the figures that were created. + """ + + # Connect to the catalog and find the necessary datasets. + catalog = intake.open_esm_datastore(catalog) + + anomalies = {} + maps = {} + timeseries = {} + for name, variable in self.metadata.variables().items(): + # Get the dataset out of the catalog. + args = self.metadata.catalog_search_args(variable) + + datasets = catalog.search( + **self.metadata.catalog_search_args(variable) + ).to_dataset_dict(progressbar=False) + dataset = list(datasets.values())[0] + + # Lon-lat maps. + maps[name] = LonLatMap.from_xarray_dataset( + dataset, + variable, + time_method="annual mean", + year=1980, + ) + + if name == "rlut": + anomalies[name] = AnomalyTimeSeries.from_xarray_dataset( + dataset, + variable, + ) + timeseries[name] = GlobalMeanTimeSeries.from_xarray_dataset( + dataset, + variable, + ) + + figure_paths = [] + + # OLR anomally timeseries. + figure = timeseries_and_anomalies(timeseries["rlut"], anomalies["rlut"], + "OLR Global Mean & Anomalies") + figure.save(Path(png_dir) / "olr-anomalies.png") + figure_paths.append(Path(png_dir) / "olr-anomalies.png") + + # OLR. + figure = radiation_decomposition(maps["rlutcsaf"], maps["rlutaf"], + maps["rlutcs"], maps["rlut"], "OLR") + figure.save(Path(png_dir) / "olr.png") + figure_paths.append(Path(png_dir) / "olr.png") + + # SW TOTA. + figure = radiation_decomposition(maps["rsutcsaf"], maps["rsutaf"], + maps["rsutcs"], maps["rsut"], + "Shortwave Outgoing Toa") + figure.save(Path(png_dir) / "sw-up-toa.png") + figure_paths.append(Path(png_dir) / "sw-up-toa.png") + + # Surface radiation budget. + surface_budget = [] + for suffix in ["csaf", "af", "cs", ""]: + surface_budget.append(maps[f"rlds{suffix}"] + maps[f"rsds{suffix}"] - + maps[f"rlus{suffix}"] - maps[f"rsus{suffix}"]) + figure = radiation_decomposition(*surface_budget, "Surface Radiation Budget") + figure.save(Path(png_dir) / "surface-radiation-budget.png") + figure_paths.append(Path(png_dir) / "surface-radiation-budget.png") + + # TOA radiation budget. + toa_budget = [] + for suffix in ["csaf", "af", "cs", ""]: + toa_budget.append(maps[f"rsdt"] - maps[f"rlut{suffix}"] - maps[f"rsut{suffix}"]) + figure = radiation_decomposition(*toa_budget, "TOA Radiation Budget") + figure.save(Path(png_dir) / "toa-radiation-budget.png") + figure_paths.append(Path(png_dir) / "toa-radiation-budget.png") + return figure_paths diff --git a/freanalysis_radiation/pyproject.toml b/freanalysis_radiation/pyproject.toml new file mode 100644 index 0000000..14e3b49 --- /dev/null +++ b/freanalysis_radiation/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = [ + "setuptools >= 40.9.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "freanalysis_radiation" +version = "0.1" +dependencies = [ + "intake", + "intake-esm", +] +requires-python = ">= 3.6" +authors = [ + {name = "developers"}, +] +maintainers = [ + {name = "developer", email = "developer-email@address.domain"}, +] +description = "Radiative flux analyzer for GFDL model output" +readme = "README.md" +classifiers = [ + "Programming Language :: Python" +] + +[project.urls] +repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" From 71f40c3cf38acf42b85be8902545626efd909270 Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Fri, 6 Dec 2024 13:57:48 -0500 Subject: [PATCH 13/15] full reorganization --- core/{ => analysis_scripts}/README.md | 0 core/analysis_scripts/{ => analysis_scripts}/__init__.py | 0 core/analysis_scripts/{ => analysis_scripts}/base_class.py | 0 core/analysis_scripts/{ => analysis_scripts}/plugins.py | 0 core/{ => analysis_scripts}/meta.yaml | 0 core/{ => analysis_scripts}/pyproject.toml | 0 core/{ => analysis_scripts}/tests/test_base_class.py | 0 core/{ => analysis_scripts}/tests/test_plugins.py | 0 {figure_tools => core/figure_tools}/README.md | 0 {figure_tools => core/figure_tools}/figure_tools/__init__.py | 0 .../figure_tools}/figure_tools/anomaly_timeseries.py | 0 {figure_tools => core/figure_tools}/figure_tools/common_plots.py | 0 {figure_tools => core/figure_tools}/figure_tools/figure.py | 0 .../figure_tools}/figure_tools/global_mean_timeseries.py | 0 {figure_tools => core/figure_tools}/figure_tools/lon_lat_map.py | 0 {figure_tools => core/figure_tools}/figure_tools/time_subsets.py | 0 .../figure_tools}/figure_tools/zonal_mean_map.py | 0 {figure_tools => core/figure_tools}/pyproject.toml | 0 .../freanalysis_aerosol}/README.md | 0 .../freanalysis_aerosol}/freanalysis_aerosol/__init__.py | 0 .../freanalysis_aerosol}/pyproject.toml | 0 .../freanalysis_clouds}/README.md | 0 .../freanalysis_clouds}/freanalysis_clouds/__init__.py | 0 .../freanalysis_clouds}/pyproject.toml | 0 .../freanalysis_land}/README.md | 0 .../freanalysis_land}/freanalysis_land/__init__.py | 0 .../freanalysis_land}/freanalysis_land/land.py | 0 .../freanalysis_land}/pyproject.toml | 0 .../freanalysis_radiation}/README.md | 0 .../freanalysis_radiation}/freanalysis_radiation/__init__.py | 0 .../freanalysis_radiation}/pyproject.toml | 0 31 files changed, 0 insertions(+), 0 deletions(-) rename core/{ => analysis_scripts}/README.md (100%) rename core/analysis_scripts/{ => analysis_scripts}/__init__.py (100%) rename core/analysis_scripts/{ => analysis_scripts}/base_class.py (100%) rename core/analysis_scripts/{ => analysis_scripts}/plugins.py (100%) rename core/{ => analysis_scripts}/meta.yaml (100%) rename core/{ => analysis_scripts}/pyproject.toml (100%) rename core/{ => analysis_scripts}/tests/test_base_class.py (100%) rename core/{ => analysis_scripts}/tests/test_plugins.py (100%) rename {figure_tools => core/figure_tools}/README.md (100%) rename {figure_tools => core/figure_tools}/figure_tools/__init__.py (100%) rename {figure_tools => core/figure_tools}/figure_tools/anomaly_timeseries.py (100%) rename {figure_tools => core/figure_tools}/figure_tools/common_plots.py (100%) rename {figure_tools => core/figure_tools}/figure_tools/figure.py (100%) rename {figure_tools => core/figure_tools}/figure_tools/global_mean_timeseries.py (100%) rename {figure_tools => core/figure_tools}/figure_tools/lon_lat_map.py (100%) rename {figure_tools => core/figure_tools}/figure_tools/time_subsets.py (100%) rename {figure_tools => core/figure_tools}/figure_tools/zonal_mean_map.py (100%) rename {figure_tools => core/figure_tools}/pyproject.toml (100%) rename {freanalysis_aerosol => user-analysis-scripts/freanalysis_aerosol}/README.md (100%) rename {freanalysis_aerosol => user-analysis-scripts/freanalysis_aerosol}/freanalysis_aerosol/__init__.py (100%) rename {freanalysis_aerosol => user-analysis-scripts/freanalysis_aerosol}/pyproject.toml (100%) rename {freanalysis_clouds => user-analysis-scripts/freanalysis_clouds}/README.md (100%) rename {freanalysis_clouds => user-analysis-scripts/freanalysis_clouds}/freanalysis_clouds/__init__.py (100%) rename {freanalysis_clouds => user-analysis-scripts/freanalysis_clouds}/pyproject.toml (100%) rename {freanalysis_land => user-analysis-scripts/freanalysis_land}/README.md (100%) rename {freanalysis_land => user-analysis-scripts/freanalysis_land}/freanalysis_land/__init__.py (100%) rename {freanalysis_land => user-analysis-scripts/freanalysis_land}/freanalysis_land/land.py (100%) rename {freanalysis_land => user-analysis-scripts/freanalysis_land}/pyproject.toml (100%) rename {freanalysis_radiation => user-analysis-scripts/freanalysis_radiation}/README.md (100%) rename {freanalysis_radiation => user-analysis-scripts/freanalysis_radiation}/freanalysis_radiation/__init__.py (100%) rename {freanalysis_radiation => user-analysis-scripts/freanalysis_radiation}/pyproject.toml (100%) diff --git a/core/README.md b/core/analysis_scripts/README.md similarity index 100% rename from core/README.md rename to core/analysis_scripts/README.md diff --git a/core/analysis_scripts/__init__.py b/core/analysis_scripts/analysis_scripts/__init__.py similarity index 100% rename from core/analysis_scripts/__init__.py rename to core/analysis_scripts/analysis_scripts/__init__.py diff --git a/core/analysis_scripts/base_class.py b/core/analysis_scripts/analysis_scripts/base_class.py similarity index 100% rename from core/analysis_scripts/base_class.py rename to core/analysis_scripts/analysis_scripts/base_class.py diff --git a/core/analysis_scripts/plugins.py b/core/analysis_scripts/analysis_scripts/plugins.py similarity index 100% rename from core/analysis_scripts/plugins.py rename to core/analysis_scripts/analysis_scripts/plugins.py diff --git a/core/meta.yaml b/core/analysis_scripts/meta.yaml similarity index 100% rename from core/meta.yaml rename to core/analysis_scripts/meta.yaml diff --git a/core/pyproject.toml b/core/analysis_scripts/pyproject.toml similarity index 100% rename from core/pyproject.toml rename to core/analysis_scripts/pyproject.toml diff --git a/core/tests/test_base_class.py b/core/analysis_scripts/tests/test_base_class.py similarity index 100% rename from core/tests/test_base_class.py rename to core/analysis_scripts/tests/test_base_class.py diff --git a/core/tests/test_plugins.py b/core/analysis_scripts/tests/test_plugins.py similarity index 100% rename from core/tests/test_plugins.py rename to core/analysis_scripts/tests/test_plugins.py diff --git a/figure_tools/README.md b/core/figure_tools/README.md similarity index 100% rename from figure_tools/README.md rename to core/figure_tools/README.md diff --git a/figure_tools/figure_tools/__init__.py b/core/figure_tools/figure_tools/__init__.py similarity index 100% rename from figure_tools/figure_tools/__init__.py rename to core/figure_tools/figure_tools/__init__.py diff --git a/figure_tools/figure_tools/anomaly_timeseries.py b/core/figure_tools/figure_tools/anomaly_timeseries.py similarity index 100% rename from figure_tools/figure_tools/anomaly_timeseries.py rename to core/figure_tools/figure_tools/anomaly_timeseries.py diff --git a/figure_tools/figure_tools/common_plots.py b/core/figure_tools/figure_tools/common_plots.py similarity index 100% rename from figure_tools/figure_tools/common_plots.py rename to core/figure_tools/figure_tools/common_plots.py diff --git a/figure_tools/figure_tools/figure.py b/core/figure_tools/figure_tools/figure.py similarity index 100% rename from figure_tools/figure_tools/figure.py rename to core/figure_tools/figure_tools/figure.py diff --git a/figure_tools/figure_tools/global_mean_timeseries.py b/core/figure_tools/figure_tools/global_mean_timeseries.py similarity index 100% rename from figure_tools/figure_tools/global_mean_timeseries.py rename to core/figure_tools/figure_tools/global_mean_timeseries.py diff --git a/figure_tools/figure_tools/lon_lat_map.py b/core/figure_tools/figure_tools/lon_lat_map.py similarity index 100% rename from figure_tools/figure_tools/lon_lat_map.py rename to core/figure_tools/figure_tools/lon_lat_map.py diff --git a/figure_tools/figure_tools/time_subsets.py b/core/figure_tools/figure_tools/time_subsets.py similarity index 100% rename from figure_tools/figure_tools/time_subsets.py rename to core/figure_tools/figure_tools/time_subsets.py diff --git a/figure_tools/figure_tools/zonal_mean_map.py b/core/figure_tools/figure_tools/zonal_mean_map.py similarity index 100% rename from figure_tools/figure_tools/zonal_mean_map.py rename to core/figure_tools/figure_tools/zonal_mean_map.py diff --git a/figure_tools/pyproject.toml b/core/figure_tools/pyproject.toml similarity index 100% rename from figure_tools/pyproject.toml rename to core/figure_tools/pyproject.toml diff --git a/freanalysis_aerosol/README.md b/user-analysis-scripts/freanalysis_aerosol/README.md similarity index 100% rename from freanalysis_aerosol/README.md rename to user-analysis-scripts/freanalysis_aerosol/README.md diff --git a/freanalysis_aerosol/freanalysis_aerosol/__init__.py b/user-analysis-scripts/freanalysis_aerosol/freanalysis_aerosol/__init__.py similarity index 100% rename from freanalysis_aerosol/freanalysis_aerosol/__init__.py rename to user-analysis-scripts/freanalysis_aerosol/freanalysis_aerosol/__init__.py diff --git a/freanalysis_aerosol/pyproject.toml b/user-analysis-scripts/freanalysis_aerosol/pyproject.toml similarity index 100% rename from freanalysis_aerosol/pyproject.toml rename to user-analysis-scripts/freanalysis_aerosol/pyproject.toml diff --git a/freanalysis_clouds/README.md b/user-analysis-scripts/freanalysis_clouds/README.md similarity index 100% rename from freanalysis_clouds/README.md rename to user-analysis-scripts/freanalysis_clouds/README.md diff --git a/freanalysis_clouds/freanalysis_clouds/__init__.py b/user-analysis-scripts/freanalysis_clouds/freanalysis_clouds/__init__.py similarity index 100% rename from freanalysis_clouds/freanalysis_clouds/__init__.py rename to user-analysis-scripts/freanalysis_clouds/freanalysis_clouds/__init__.py diff --git a/freanalysis_clouds/pyproject.toml b/user-analysis-scripts/freanalysis_clouds/pyproject.toml similarity index 100% rename from freanalysis_clouds/pyproject.toml rename to user-analysis-scripts/freanalysis_clouds/pyproject.toml diff --git a/freanalysis_land/README.md b/user-analysis-scripts/freanalysis_land/README.md similarity index 100% rename from freanalysis_land/README.md rename to user-analysis-scripts/freanalysis_land/README.md diff --git a/freanalysis_land/freanalysis_land/__init__.py b/user-analysis-scripts/freanalysis_land/freanalysis_land/__init__.py similarity index 100% rename from freanalysis_land/freanalysis_land/__init__.py rename to user-analysis-scripts/freanalysis_land/freanalysis_land/__init__.py diff --git a/freanalysis_land/freanalysis_land/land.py b/user-analysis-scripts/freanalysis_land/freanalysis_land/land.py similarity index 100% rename from freanalysis_land/freanalysis_land/land.py rename to user-analysis-scripts/freanalysis_land/freanalysis_land/land.py diff --git a/freanalysis_land/pyproject.toml b/user-analysis-scripts/freanalysis_land/pyproject.toml similarity index 100% rename from freanalysis_land/pyproject.toml rename to user-analysis-scripts/freanalysis_land/pyproject.toml diff --git a/freanalysis_radiation/README.md b/user-analysis-scripts/freanalysis_radiation/README.md similarity index 100% rename from freanalysis_radiation/README.md rename to user-analysis-scripts/freanalysis_radiation/README.md diff --git a/freanalysis_radiation/freanalysis_radiation/__init__.py b/user-analysis-scripts/freanalysis_radiation/freanalysis_radiation/__init__.py similarity index 100% rename from freanalysis_radiation/freanalysis_radiation/__init__.py rename to user-analysis-scripts/freanalysis_radiation/freanalysis_radiation/__init__.py diff --git a/freanalysis_radiation/pyproject.toml b/user-analysis-scripts/freanalysis_radiation/pyproject.toml similarity index 100% rename from freanalysis_radiation/pyproject.toml rename to user-analysis-scripts/freanalysis_radiation/pyproject.toml From 4fa4d00657b22401293f17759a68a1a9357d2d4e Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Fri, 6 Dec 2024 14:20:29 -0500 Subject: [PATCH 14/15] add conda upload to noaa-gfdl github action --- ...ripts-core-to-noaa-gfdl-conda-channel.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/add-analysis-scripts-core-to-noaa-gfdl-conda-channel.yaml diff --git a/.github/workflows/add-analysis-scripts-core-to-noaa-gfdl-conda-channel.yaml b/.github/workflows/add-analysis-scripts-core-to-noaa-gfdl-conda-channel.yaml new file mode 100644 index 0000000..c6731f5 --- /dev/null +++ b/.github/workflows/add-analysis-scripts-core-to-noaa-gfdl-conda-channel.yaml @@ -0,0 +1,21 @@ +name: add_to_noaa_gfdl_channel +on: + push: + branches: + - main +jobs: + publish: + runs-on: ubuntu-latest + container: + image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0 + steps: + - name: Checkout Files + uses: actions/checkout@v4 + - name: Run Conda to Build and Publish + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build anaconda-client conda-verify + export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} + conda config --set anaconda_upload yes + conda build core/analysis_scripts From 2430d7acaae4407bc73e9c1b145004d85b8a56e0 Mon Sep 17 00:00:00 2001 From: Raymond Menzel Date: Fri, 6 Dec 2024 14:33:42 -0500 Subject: [PATCH 15/15] add tests back --- .github/workflows/ci-analysis.yml | 4 +- .../tests/test_freanalysis_clouds.py | 63 +++++++++++++++++++ .../tests/test_freanalysis_land.py | 10 +++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 user-analysis-scripts/freanalysis_clouds/tests/test_freanalysis_clouds.py create mode 100644 user-analysis-scripts/freanalysis_land/tests/test_freanalysis_land.py diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index 5c5144d..91415bc 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -1,5 +1,5 @@ # Test the core package. -name: Test analysis_scripts +name: Test analysis_scripts core functionality on: [pull_request] @@ -21,8 +21,10 @@ jobs: - name: Install dependencies run: | + cd core/analysis_scripts python -m pip install . - name: Run the unit tests run: | + cd core/analysis_scripts pytest --capture=tee-sys tests diff --git a/user-analysis-scripts/freanalysis_clouds/tests/test_freanalysis_clouds.py b/user-analysis-scripts/freanalysis_clouds/tests/test_freanalysis_clouds.py new file mode 100644 index 0000000..ea54443 --- /dev/null +++ b/user-analysis-scripts/freanalysis_clouds/tests/test_freanalysis_clouds.py @@ -0,0 +1,63 @@ +from ftplib import FTP +from os import chdir, environ +from pathlib import Path +from subprocess import run +from tempfile import TemporaryDirectory +import sys + +from analysis_scripts import list_plugins, plugin_requirements, run_plugin +import catalogbuilder +from catalogbuilder.scripts import gen_intake_gfdl + + +def download_test_data(stem): + """Downloads test datasets from a FTP server. + + Args: + stem: Directory to create the directory tree inside. + + Returns: + Path to the directory that will be used as the root of the data catalog. + """ + # Create local directory tree with the appropriate directory structure. + catalog_root = Path(stem) / "archive" / "oar.gfdl.mdtf" / "MDTF-examples" / \ + "mdtf-time-slice-example" / "gfdl.ncrc5-deploy-prod-openmp" / "pp" + data_directory = catalog_root / "atmos" / "ts" / "monthly" / "1yr" + data_directory.mkdir(parents=True, exist_ok=True) + + # Download the datasets from the FTP server. + path = "1/oar.gfdl.mdtf/MDTF-examples/GFDL-CM4/data/atmos/ts/monthly/1yr" + with FTP("nomads.gfdl.noaa.gov") as ftp: + ftp.login() + ftp.cwd(path) + for variable in ["high_cld_amt", "mid_cld_amt", "low_cld_amt"]: + name = f"atmos.198001-198012.{variable}.nc" + ftp.retrbinary(f"RETR {name}", open(data_directory / name, "wb").write) + return catalog_root.resolve() + + +def plugin(json, pngs_directory="pngs"): + """Run the plugin to create the figure. + + Args: + json: Path to the catalog json file. + pngs_directory: Directory to store the output in. + """ + name = "freanalysis_clouds" + reqs = plugin_requirements(name) + Path(pngs_directory).mkdir(parents=True, exist_ok=True) + run_plugin(name, json, pngs_directory) + + +def test_freanalysis_clouds(): + with TemporaryDirectory() as tmp: + chdir(Path(tmp)) + path = download_test_data(stem=tmp) + yaml = Path(__file__).resolve().parent / "mdtf_timeslice_catalog.yaml" + outputpath = Path(__file__).resolve().parent / "data-catalog" + # Creates data catalog using the scripts in catalogbuilder + csv, json = gen_intake_gfdl.create_catalog(input_path=str(path), + output_path=outputpath, + config=str(yaml)) + print(json,csv) + plugin(json) diff --git a/user-analysis-scripts/freanalysis_land/tests/test_freanalysis_land.py b/user-analysis-scripts/freanalysis_land/tests/test_freanalysis_land.py new file mode 100644 index 0000000..19225fa --- /dev/null +++ b/user-analysis-scripts/freanalysis_land/tests/test_freanalysis_land.py @@ -0,0 +1,10 @@ +from freanalysis_land.land import LandAnalysisScript + + +def test_land_analysis_script(): + land = LandAnalysisScript() + land.run_analysis("/work/a2p/lm4p2sc_GSWP3_hist_irr_catalog.json", "/work/a2p/") + + +if __name_ == "__main__": + test_land_analysis_script()