Merge pull request #15 from NOAA-GFDL/restructure

Restructure the layout of the repo and add the ability to work with fre-cli

.github/workflows/add-analysis-scripts-core-to-noaa-gfdl-conda-channel.yaml

@@ -0,0 +1,21 @@
+name: add_to_noaa_gfdl_channel
+on:
+  push:
+    branches:
+      - main
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0
+    steps:
+      - name: Checkout Files
+        uses: actions/checkout@v4
+      - name: Run Conda to Build and Publish
+        run: |
+          conda config --append channels conda-forge
+          conda config --append channels noaa-gfdl
+          conda install conda-build anaconda-client conda-verify
+          export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }}
+          conda config --set anaconda_upload yes
+          conda build core/analysis_scripts

.github/workflows/ci-analysis.yml

@@ -1,5 +1,5 @@
-# Installs the Python dependencies and runs the freanalysis_clouds plugin.
-name: Test freanalysis_clouds plugin
+# Test the core package.
+name: Test analysis_scripts core functionality
 
 on: [pull_request]
 
@@ -13,40 +13,18 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-      
+
     - name: Install dependencies
       run: |
-        # $CONDA is an environment variable pointing to the root of the miniconda directory
-        echo $CONDA/bin >> $GITHUB_PATH
-        conda config --add channels noaa-gfdl
-        conda config --append channels conda-forge
-        conda create --name analysis-script-testing
-        conda install -n analysis-script-testing catalogbuilder -c noaa-gfdl
-        #conda activate analysis-script-testing
-        conda install pip
-        cd analysis-scripts
-        $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd ..
-        cd figure_tools
-        $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd ..
-        cd freanalysis
-        $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd ..
-        cd freanalysis_clouds
-        $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd ..
-        
-    - name: Generate catalog and run freanalysis_clouds
-      run: |
-         $CONDA/envs/analysis-script-testing/bin/pytest --capture=tee-sys tests/test_freanalysis_clouds.py
-    - name: upload-artifacts
-      uses: actions/upload-artifact@v4
-      with:
-          name: workflow-artifacts1 
-          path: |
-            ${{ github.workspace }}/tests/data-catalog.json
-            ${{ github.workspace }}/tests/data-catalog.csv
-    - name: Run freanalysis_land
+        cd core/analysis_scripts
+        python -m pip install .
+
+    - name: Run the unit tests
       run: |
-         $CONDA/envs/analysis-script-testing/bin/pytest tests/test_freanalysis_land
+        cd core/analysis_scripts
+        pytest --capture=tee-sys tests

analysis-scripts/README.md → core/analysis_scripts/README.md

@@ -1,16 +1,24 @@
 # analysis-scripts
-A framework for analyzing GFDL model output
+A framework for analyzing GFDL model output.
 
 ### Motivation
 The goal of this project is to provide a simple API to guide the development of
 scripts that produce figures and tables from GFDL model output.  This work will be used
 by a simple web application to provide users an easy interface to interact with model
 output data.
 
-### Requirements
-No external packages are required.
+### How to install this package
+For now I'd recommend creating a virtual enviroment, and then installing the package inside
+of it.
 
-### How to use this framework.
+```bash
+$ python3 -m venv env
+$ source env/bin/activate
+$ pip install --upgrade pip
+$ pip install .
+```
+
+### How to use this framework
 Custom analysis scripts classes can be created that inherit the `AnalysisScript` base
 class, then override its contructor, `requires`, and `run_analysis` methods. For example:
 
@@ -67,12 +75,13 @@ class NewAnalysisScript(AnalysisScript):
             }
         })
 
-    def run_analysis(self, catalog, png_dir, reference_catalog=None):
+    def run_analysis(self, catalog, png_dir, config=None, reference_catalog=None):
         """Runs the analysis and generates all plots and associated datasets.
 
         Args:
             catalog: Path to a model output catalog.
             png_dir: Directory to store output png figures in.
+            config: Dictionary of configuration options.
             reference_catalog: Path to a catalog of reference data.
 
         Returns:
@@ -81,3 +90,23 @@ class NewAnalysisScript(AnalysisScript):
         Do some stuff to create the figures.
         return ["figure1.png", "figure2.png",]
 ```
+
+### Running a custom analysis script
+In order to run a custom analysis script, you must first create a data catalog and
+can then perform the analysis:
+
+```python3
+from analysis_scripts import available_plugins, plugin_requirements, run_plugin
+
+
+# Create a data catalog.
+# Some code to create a data "catalog.json" ...
+
+# Show the installed plugins.
+print(available_plugins())
+
+# Run the radiative fluxes plugin.
+name = "freanalysis_radiation"  # Name of the custom analysis script you want to run.
+print(plugin_requirements(name))
+figures = run_plugin(name, "catalog.json", "pngs")
+```

core/analysis_scripts/analysis_scripts/__init__.py

@@ -0,0 +1,3 @@
+from .base_class import AnalysisScript
+from .plugins import available_plugins, find_plugins, plugin_requirements, \
+                     run_plugin, UnknownPluginError

analysis-scripts/analysis_scripts/__init__.py → core/analysis_scripts/analysis_scripts/base_class.py

@@ -24,12 +24,13 @@ def requires(self):
         raise NotImplementedError("you must override this function.")
         return json.dumps("{json of metadata MDTF format.}")
 
-    def run_analysis(self, catalog, png_dir, reference_catalog=None):
+    def run_analysis(self, catalog, png_dir, config=None, reference_catalog=None):
         """Runs the analysis and generates all plots and associated datasets.
 
         Args:
             catalog: Path to a model output catalog.
             png_dir: Directory to store output png figures in.
+            config: Dictionary of configuration options.
             reference_catalog: Path to a catalog of reference data.
 
         Returns:

freanalysis/freanalysis/plugins.py → core/analysis_scripts/analysis_scripts/plugins.py

@@ -2,14 +2,29 @@
 import inspect
 import pkgutil
 
-from analysis_scripts import AnalysisScript
+from .base_class import AnalysisScript
 
 
-# Find all installed python modules with names that start with "freanalysis_"
+# Dictionary of found plugins.
 discovered_plugins = {}
-for finder, name, ispkg in pkgutil.iter_modules():
-    if name.startswith("freanalysis_"):
-        discovered_plugins[name] = importlib.import_module(name)
+
+
+def find_plugins(path=None):
+    """Find all installed python modules with names that start with 'freanalysis_'."""
+    if path:
+        path = [path,]
+    for finder, name, ispkg in pkgutil.iter_modules(path):
+        if name.startswith("freanalysis_"):
+            discovered_plugins[name] = importlib.import_module(name)
+
+
+# Update plugin dictionary.
+find_plugins()
+
+
+class UnknownPluginError(BaseException):
+    """Custom exception for when an invalid plugin name is used."""
+    pass
 
 
 def _plugin_object(name):
@@ -26,8 +41,16 @@ def _plugin_object(name):
         ValueError if no object that inhertis from AnalysisScript is found in the
             plugin module.
     """
-    for attribute in vars(discovered_plugins[name]).values():
+    # Loop through all attributes in the plugin package with the input name.
+    try:
+        plugin_module = discovered_plugins[name]
+    except KeyError:
+        raise UnknownPluginError(f"could not find analysis script plugin {name}.")
+
+    for attribute in vars(plugin_module).values():
+       # Try to find a class that inherits from the AnalysisScript class.
        if inspect.isclass(attribute) and AnalysisScript in attribute.__bases__:
+           # Instantiate an object of this class.
            return attribute()
     raise ValueError(f"could not find compatible object in {name}.") 
 
@@ -58,16 +81,17 @@ def plugin_requirements(name):
     return _plugin_object(name).requires()
 
 
-def run_plugin(name, catalog, png_dir, reference_catalog=None):
+def run_plugin(name, catalog, png_dir, config=None, reference_catalog=None):
     """Runs the plugin's analysis.
 
     Args:
         name: Name of the plugin.
         catalog: Path to the data catalog.
         png_dir: Directory where the output figures will be stored.
+        config: Dictionary of configuration values.
         catalog: Path to the catalog of reference data.
 
     Returns:
         A list of png figure files that were created by the analysis.
     """
-    return _plugin_object(name).run_analysis(catalog, png_dir, reference_catalog)
+    return _plugin_object(name).run_analysis(catalog, png_dir, config, reference_catalog)

core/analysis_scripts/meta.yaml

@@ -0,0 +1,19 @@
+package:
+  name: analysis_scripts
+  version: 0.0.1
+
+source:
+  path: .
+
+build:
+  script: "{{ PYTHON }} -m pip install . -vv"
+  number: 0
+  noarch: python
+
+requirements:
+  host:
+    - python
+    - pip
+  run:
+    - python
+    - pytest

freanalysis/pyproject.toml → core/analysis_scripts/pyproject.toml

@@ -5,8 +5,11 @@ requires = [
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "freanalysis"
-version = "0.1"
+name = "analysis_scripts"
+version = "0.0.1"
+dependencies = [
+    "pytest",
+]
 requires-python = ">= 3.6"
 authors = [
     {name = "developers"},

core/analysis_scripts/tests/test_base_class.py

@@ -0,0 +1,29 @@
+from pytest import raises
+
+from analysis_scripts import AnalysisScript
+
+
+def test_no_init():
+    #The constructor is not overridden.
+    with raises(NotImplementedError):
+        class FakeAnalysisScript(AnalysisScript):
+            pass
+        _ = FakeAnalysisScript()
+
+
+def test_no_requires():
+    #The requires function is not overridden.
+    with raises(NotImplementedError):
+        class FakeAnalysisScript(AnalysisScript):
+            def __init__(self):
+                pass
+        _ = FakeAnalysisScript().requires()
+
+
+def test_no_run_analysis():
+    #The requires function is not overridden.
+    with raises(NotImplementedError):
+        class FakeAnalysisScript(AnalysisScript):
+            def __init__(self):
+                pass
+        _ = FakeAnalysisScript().run_analysis("fake catalog", "fake png directory")