diff --git a/.github/workflows/add-analysis-scripts-core-to-noaa-gfdl-conda-channel.yaml b/.github/workflows/add-analysis-scripts-core-to-noaa-gfdl-conda-channel.yaml new file mode 100644 index 0000000..c6731f5 --- /dev/null +++ b/.github/workflows/add-analysis-scripts-core-to-noaa-gfdl-conda-channel.yaml @@ -0,0 +1,21 @@ +name: add_to_noaa_gfdl_channel +on: + push: + branches: + - main +jobs: + publish: + runs-on: ubuntu-latest + container: + image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0 + steps: + - name: Checkout Files + uses: actions/checkout@v4 + - name: Run Conda to Build and Publish + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build anaconda-client conda-verify + export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} + conda config --set anaconda_upload yes + conda build core/analysis_scripts diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index d6a972f..91415bc 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -1,5 +1,5 @@ -# Installs the Python dependencies and runs the freanalysis_clouds plugin. -name: Test freanalysis_clouds plugin +# Test the core package. +name: Test analysis_scripts core functionality on: [pull_request] @@ -13,40 +13,18 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - + - name: Install dependencies run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - conda config --add channels noaa-gfdl - conda config --append channels conda-forge - conda create --name analysis-script-testing - conda install -n analysis-script-testing catalogbuilder -c noaa-gfdl - #conda activate analysis-script-testing - conda install pip - cd analysis-scripts - $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. - cd figure_tools - $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. - cd freanalysis - $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. - cd freanalysis_clouds - $CONDA/envs/analysis-script-testing/bin/python -m pip install .; cd .. - - - name: Generate catalog and run freanalysis_clouds - run: | - $CONDA/envs/analysis-script-testing/bin/pytest --capture=tee-sys tests/test_freanalysis_clouds.py - - name: upload-artifacts - uses: actions/upload-artifact@v4 - with: - name: workflow-artifacts1 - path: | - ${{ github.workspace }}/tests/data-catalog.json - ${{ github.workspace }}/tests/data-catalog.csv - - name: Run freanalysis_land + cd core/analysis_scripts + python -m pip install . + + - name: Run the unit tests run: | - $CONDA/envs/analysis-script-testing/bin/pytest tests/test_freanalysis_land + cd core/analysis_scripts + pytest --capture=tee-sys tests diff --git a/README.md b/README.md deleted file mode 100644 index 260c8d3..0000000 --- a/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# analysis-scripts -A framework for analyzing GFDL model output - -### Motivation -The goal of this project is to provide a simple API to guide the development of -scripts that produce figures and tables from GFDL model output. This work will be used -by a simple web application to provide users an easy interface to interact with model -output data. - -### Requirements -The code in this repository is broken up into components: - -- analysis-scripts - A very simple package that just defines an abstract base class that - all user-created plugins should inherit from. -- figure_tools - An optional package that contains some helper functions and classes - for making common plots. -- freanalysis - A package that is designed to be used by whatever workflow is responsible - for running the analysis. -- freanalysis_aerosol - A plugin that creates aerosl mass figures. -- freanalysis_clouds - A plugin that creates cloud amount figures. -- freanalysis_radiation - A plugin that creates radiative flux figures. - -### How to install everything -For now I'd recommend creating a virtual enviroment, and then installing each of the -packages listed above: - -```bash -$ python3 -m venv env -$ source env/bin/activate -$ pip install --upgrade pip -$ cd analysis-scripts; pip install .; cd .. -$ cd figure_tools; pip install .; cd .. -$ cd freanalysis; pip install .; cd .. -$ cd freanalysis_aerosol; pip install .; cd .. -$ cd freanalysis_clouds; pip install .; cd .. -$ cd freanalysis_radiation; pip install .; cd .. -``` - -# Running an analysis plugin -In order to run a plugin, you must first create a data catalog and can then perform -the analysis: - -```python3 -from freanalysis.create_catalog import create_catalog -from freanalysis.plugins import list_plugins, plugin_requirements, run_plugin - - -# Create a data catalog. -create_catalog(pp_dir, "catalog.json") - -# Show the installed plugins. -list_plugins() - -# Run the radiative fluxes plugin. -name = "freanalysis_radiation" -reqs = plugin_requirements(name) -print(reqs) -run_plugin(name, "catalog.json", "pngs") -``` diff --git a/analysis-scripts/pyproject.toml b/analysis-scripts/pyproject.toml deleted file mode 100644 index 131cc63..0000000 --- a/analysis-scripts/pyproject.toml +++ /dev/null @@ -1,28 +0,0 @@ -[build-system] -requires = [ - "setuptools >= 40.9.0", -] -build-backend = "setuptools.build_meta" - -[project] -name = "analysis-scripts" -version = "0.1" -dependencies = [ - "intake", - "intake-esm", -] -requires-python = ">= 3.6" -authors = [ - {name = "developers"}, -] -maintainers = [ - {name = "developer", email = "developer-email@address.domain"}, -] -description = "Framework for analyzing GFDL model output" -readme = "README.md" -classifiers = [ - "Programming Language :: Python" -] - -[project.urls] -repository = "https://github.com/NOAA-GFDL/analysis-scripts.git" diff --git a/analysis-scripts/README.md b/core/analysis_scripts/README.md similarity index 73% rename from analysis-scripts/README.md rename to core/analysis_scripts/README.md index ccf6b17..67ca3d5 100644 --- a/analysis-scripts/README.md +++ b/core/analysis_scripts/README.md @@ -1,5 +1,5 @@ # analysis-scripts -A framework for analyzing GFDL model output +A framework for analyzing GFDL model output. ### Motivation The goal of this project is to provide a simple API to guide the development of @@ -7,10 +7,18 @@ scripts that produce figures and tables from GFDL model output. This work will by a simple web application to provide users an easy interface to interact with model output data. -### Requirements -No external packages are required. +### How to install this package +For now I'd recommend creating a virtual enviroment, and then installing the package inside +of it. -### How to use this framework. +```bash +$ python3 -m venv env +$ source env/bin/activate +$ pip install --upgrade pip +$ pip install . +``` + +### How to use this framework Custom analysis scripts classes can be created that inherit the `AnalysisScript` base class, then override its contructor, `requires`, and `run_analysis` methods. For example: @@ -67,12 +75,13 @@ class NewAnalysisScript(AnalysisScript): } }) - def run_analysis(self, catalog, png_dir, reference_catalog=None): + def run_analysis(self, catalog, png_dir, config=None, reference_catalog=None): """Runs the analysis and generates all plots and associated datasets. Args: catalog: Path to a model output catalog. png_dir: Directory to store output png figures in. + config: Dictionary of configuration options. reference_catalog: Path to a catalog of reference data. Returns: @@ -81,3 +90,23 @@ class NewAnalysisScript(AnalysisScript): Do some stuff to create the figures. return ["figure1.png", "figure2.png",] ``` + +### Running a custom analysis script +In order to run a custom analysis script, you must first create a data catalog and +can then perform the analysis: + +```python3 +from analysis_scripts import available_plugins, plugin_requirements, run_plugin + + +# Create a data catalog. +# Some code to create a data "catalog.json" ... + +# Show the installed plugins. +print(available_plugins()) + +# Run the radiative fluxes plugin. +name = "freanalysis_radiation" # Name of the custom analysis script you want to run. +print(plugin_requirements(name)) +figures = run_plugin(name, "catalog.json", "pngs") +``` diff --git a/core/analysis_scripts/analysis_scripts/__init__.py b/core/analysis_scripts/analysis_scripts/__init__.py new file mode 100644 index 0000000..c4a4f0a --- /dev/null +++ b/core/analysis_scripts/analysis_scripts/__init__.py @@ -0,0 +1,3 @@ +from .base_class import AnalysisScript +from .plugins import available_plugins, find_plugins, plugin_requirements, \ + run_plugin, UnknownPluginError diff --git a/analysis-scripts/analysis_scripts/__init__.py b/core/analysis_scripts/analysis_scripts/base_class.py similarity index 90% rename from analysis-scripts/analysis_scripts/__init__.py rename to core/analysis_scripts/analysis_scripts/base_class.py index df1eb04..266dd09 100644 --- a/analysis-scripts/analysis_scripts/__init__.py +++ b/core/analysis_scripts/analysis_scripts/base_class.py @@ -24,12 +24,13 @@ def requires(self): raise NotImplementedError("you must override this function.") return json.dumps("{json of metadata MDTF format.}") - def run_analysis(self, catalog, png_dir, reference_catalog=None): + def run_analysis(self, catalog, png_dir, config=None, reference_catalog=None): """Runs the analysis and generates all plots and associated datasets. Args: catalog: Path to a model output catalog. png_dir: Directory to store output png figures in. + config: Dictionary of configuration options. reference_catalog: Path to a catalog of reference data. Returns: diff --git a/freanalysis/freanalysis/plugins.py b/core/analysis_scripts/analysis_scripts/plugins.py similarity index 57% rename from freanalysis/freanalysis/plugins.py rename to core/analysis_scripts/analysis_scripts/plugins.py index ceefd8c..9ac325e 100644 --- a/freanalysis/freanalysis/plugins.py +++ b/core/analysis_scripts/analysis_scripts/plugins.py @@ -2,14 +2,29 @@ import inspect import pkgutil -from analysis_scripts import AnalysisScript +from .base_class import AnalysisScript -# Find all installed python modules with names that start with "freanalysis_" +# Dictionary of found plugins. discovered_plugins = {} -for finder, name, ispkg in pkgutil.iter_modules(): - if name.startswith("freanalysis_"): - discovered_plugins[name] = importlib.import_module(name) + + +def find_plugins(path=None): + """Find all installed python modules with names that start with 'freanalysis_'.""" + if path: + path = [path,] + for finder, name, ispkg in pkgutil.iter_modules(path): + if name.startswith("freanalysis_"): + discovered_plugins[name] = importlib.import_module(name) + + +# Update plugin dictionary. +find_plugins() + + +class UnknownPluginError(BaseException): + """Custom exception for when an invalid plugin name is used.""" + pass def _plugin_object(name): @@ -26,8 +41,16 @@ def _plugin_object(name): ValueError if no object that inhertis from AnalysisScript is found in the plugin module. """ - for attribute in vars(discovered_plugins[name]).values(): + # Loop through all attributes in the plugin package with the input name. + try: + plugin_module = discovered_plugins[name] + except KeyError: + raise UnknownPluginError(f"could not find analysis script plugin {name}.") + + for attribute in vars(plugin_module).values(): + # Try to find a class that inherits from the AnalysisScript class. if inspect.isclass(attribute) and AnalysisScript in attribute.__bases__: + # Instantiate an object of this class. return attribute() raise ValueError(f"could not find compatible object in {name}.") @@ -58,16 +81,17 @@ def plugin_requirements(name): return _plugin_object(name).requires() -def run_plugin(name, catalog, png_dir, reference_catalog=None): +def run_plugin(name, catalog, png_dir, config=None, reference_catalog=None): """Runs the plugin's analysis. Args: name: Name of the plugin. catalog: Path to the data catalog. png_dir: Directory where the output figures will be stored. + config: Dictionary of configuration values. catalog: Path to the catalog of reference data. Returns: A list of png figure files that were created by the analysis. """ - return _plugin_object(name).run_analysis(catalog, png_dir, reference_catalog) + return _plugin_object(name).run_analysis(catalog, png_dir, config, reference_catalog) diff --git a/core/analysis_scripts/meta.yaml b/core/analysis_scripts/meta.yaml new file mode 100644 index 0000000..be464af --- /dev/null +++ b/core/analysis_scripts/meta.yaml @@ -0,0 +1,19 @@ +package: + name: analysis_scripts + version: 0.0.1 + +source: + path: . + +build: + script: "{{ PYTHON }} -m pip install . -vv" + number: 0 + noarch: python + +requirements: + host: + - python + - pip + run: + - python + - pytest diff --git a/freanalysis/pyproject.toml b/core/analysis_scripts/pyproject.toml similarity index 86% rename from freanalysis/pyproject.toml rename to core/analysis_scripts/pyproject.toml index ec42453..dde62e3 100644 --- a/freanalysis/pyproject.toml +++ b/core/analysis_scripts/pyproject.toml @@ -5,8 +5,11 @@ requires = [ build-backend = "setuptools.build_meta" [project] -name = "freanalysis" -version = "0.1" +name = "analysis_scripts" +version = "0.0.1" +dependencies = [ + "pytest", +] requires-python = ">= 3.6" authors = [ {name = "developers"}, diff --git a/core/analysis_scripts/tests/test_base_class.py b/core/analysis_scripts/tests/test_base_class.py new file mode 100644 index 0000000..ab8f4c3 --- /dev/null +++ b/core/analysis_scripts/tests/test_base_class.py @@ -0,0 +1,29 @@ +from pytest import raises + +from analysis_scripts import AnalysisScript + + +def test_no_init(): + #The constructor is not overridden. + with raises(NotImplementedError): + class FakeAnalysisScript(AnalysisScript): + pass + _ = FakeAnalysisScript() + + +def test_no_requires(): + #The requires function is not overridden. + with raises(NotImplementedError): + class FakeAnalysisScript(AnalysisScript): + def __init__(self): + pass + _ = FakeAnalysisScript().requires() + + +def test_no_run_analysis(): + #The requires function is not overridden. + with raises(NotImplementedError): + class FakeAnalysisScript(AnalysisScript): + def __init__(self): + pass + _ = FakeAnalysisScript().run_analysis("fake catalog", "fake png directory") diff --git a/core/analysis_scripts/tests/test_plugins.py b/core/analysis_scripts/tests/test_plugins.py new file mode 100644 index 0000000..34e7852 --- /dev/null +++ b/core/analysis_scripts/tests/test_plugins.py @@ -0,0 +1,21 @@ +from pytest import raises + +from analysis_scripts import available_plugins, plugin_requirements, run_plugin, \ + UnknownPluginError + + +def test_no_plugins(): + """No valid plugins are installed.""" + assert available_plugins() == [] + + +def test_invalid_plugin_requirements(): + """An invalid plugin name is passed in to plugin_requirements.""" + with raises(UnknownPluginError): + _ = plugin_requirements("fake_plugin") + + +def test_invalid_run_plugin(): + """An invalid plugin name is passed in to run_plugin.""" + with raises(UnknownPluginError): + _ = run_plugin("fake_plugin", "fake_catalog.json", "fake_png_directory") diff --git a/figure_tools/README.md b/core/figure_tools/README.md similarity index 57% rename from figure_tools/README.md rename to core/figure_tools/README.md index 8e8b399..5d015d6 100644 --- a/figure_tools/README.md +++ b/core/figure_tools/README.md @@ -14,9 +14,21 @@ The only software packages that are required are: - numpy - xarray -### How to -Maps, heatmaps, and line plots can be made from the provided objects. These objects -can be instantiated directly from `xarray` datasets. For example: +### How to install this package +For now I'd recommend creating and installing this package in a virtual enviroment: + +```bash +$ python3 -m venv env +$ source env/bin/activate +$ pip install --upgrade pip +$ git clone https://github.com/NOAA-GFDL/analysis-scripts.git +$ cd analysis-scripts/figure_tools +$ pip install . +``` + +### Creating plots +Longitude-latitude maps, heatmaps, and line plots can be made from the provided +objects. These objects can be instantiated directly from `xarray` datasets. For example: ```python3 # Longitude-latitude map. diff --git a/figure_tools/figure_tools/__init__.py b/core/figure_tools/figure_tools/__init__.py similarity index 100% rename from figure_tools/figure_tools/__init__.py rename to core/figure_tools/figure_tools/__init__.py diff --git a/figure_tools/figure_tools/anomaly_timeseries.py b/core/figure_tools/figure_tools/anomaly_timeseries.py similarity index 100% rename from figure_tools/figure_tools/anomaly_timeseries.py rename to core/figure_tools/figure_tools/anomaly_timeseries.py diff --git a/figure_tools/figure_tools/common_plots.py b/core/figure_tools/figure_tools/common_plots.py similarity index 100% rename from figure_tools/figure_tools/common_plots.py rename to core/figure_tools/figure_tools/common_plots.py diff --git a/figure_tools/figure_tools/figure.py b/core/figure_tools/figure_tools/figure.py similarity index 100% rename from figure_tools/figure_tools/figure.py rename to core/figure_tools/figure_tools/figure.py diff --git a/figure_tools/figure_tools/global_mean_timeseries.py b/core/figure_tools/figure_tools/global_mean_timeseries.py similarity index 100% rename from figure_tools/figure_tools/global_mean_timeseries.py rename to core/figure_tools/figure_tools/global_mean_timeseries.py diff --git a/figure_tools/figure_tools/lon_lat_map.py b/core/figure_tools/figure_tools/lon_lat_map.py similarity index 100% rename from figure_tools/figure_tools/lon_lat_map.py rename to core/figure_tools/figure_tools/lon_lat_map.py diff --git a/figure_tools/figure_tools/time_subsets.py b/core/figure_tools/figure_tools/time_subsets.py similarity index 100% rename from figure_tools/figure_tools/time_subsets.py rename to core/figure_tools/figure_tools/time_subsets.py diff --git a/figure_tools/figure_tools/zonal_mean_map.py b/core/figure_tools/figure_tools/zonal_mean_map.py similarity index 100% rename from figure_tools/figure_tools/zonal_mean_map.py rename to core/figure_tools/figure_tools/zonal_mean_map.py diff --git a/figure_tools/pyproject.toml b/core/figure_tools/pyproject.toml similarity index 97% rename from figure_tools/pyproject.toml rename to core/figure_tools/pyproject.toml index 3dc447d..5d674f2 100644 --- a/figure_tools/pyproject.toml +++ b/core/figure_tools/pyproject.toml @@ -11,7 +11,6 @@ dependencies = [ "cartopy", "matplotlib", "numpy", - "scipy", "xarray", ] requires-python = ">= 3.6" diff --git a/freanalysis/README.md b/freanalysis/README.md deleted file mode 100644 index be530d1..0000000 --- a/freanalysis/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# freanalysis -Package that can run GFDL model analysis plugins. - -### Motivation -This crates a simple way for FRE to discover and run user generated analysis packages -to create figures to analyze the GFDL models. - -### Requirements -The software packages that are required are: - -- analysis-scripts - -### How to use -Users can create their own python packages that start with the name `freanalysis_`. -This package will search for installed packages that start with this name, and -then try to use them: - -```python3 -from freanalysis import available_plugins, list_plugins, plugin_requirements, run_plugin - - -# Get a list of all available plugins: -plugins = available_plugins() - - -# Print out a list of all available plugins: -list_plugins() - - -# Get the metadata for each plugin. This can be used to create/verify against a data -# catalog (i.e., if you want to check if a plugin is compatable with a catalog). -for name in plugins: - metadata = plugin_requirements(name) - - -# Run the plugins. You need to pass in a path to a data catalog, and a directory where -# you want the figures to be created. -for name in plugins: - figures = run_plugin(name, catalog, png_dir, reference_catalog=None) -``` diff --git a/freanalysis/freanalysis/__init__.py b/freanalysis/freanalysis/__init__.py deleted file mode 100644 index 0b16a29..0000000 --- a/freanalysis/freanalysis/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Make this directory a python package. diff --git a/freanalysis/freanalysis/create_catalog.py b/freanalysis/freanalysis/create_catalog.py deleted file mode 100644 index 829fc17..0000000 --- a/freanalysis/freanalysis/create_catalog.py +++ /dev/null @@ -1,104 +0,0 @@ -import json -from pathlib import Path - - -# Catalog column names. -columns = [ - "activity_id", - "institution_id", - "source_id", - "experiment_id", - "frequency", - "modeling_realm", - "table_id", - "member_id", - "grid_label", - "variable_id", - "temporal_subset", - "chunk_freq", - "platform", - "cell_methods", - "path", -] - - -def create_catalog(pp_dir, output_path): - """Creates a catalog json. - - Args: - pp_dir: Path to the base of a FRE post-process directory tree. - output_path: Path to the output file. - """ - parent_dir = Path(output_path).parent - csv_path = parent_dir / f"{Path(output_path).stem}.csv" - with open(output_path, "w") as output: - json_str = { - "esmcat_version": "0.0.1", - "attributes": [ - {"column_name": "activity_id", "vocabulary": ""}, - {"column_name": "institution_id", "vocabulary": ""}, - {"column_name": "source_id", "vocabulary": ""}, - {"column_name": "experiment_id", "vocabulary": ""}, - {"column_name": "frequency", "vocabulary": ""}, - {"column_name": "modeling_realm", "vocabulary": ""}, - {"column_name": "table_id", "vocabulary": ""}, - {"column_name": "member_id", "vocabulary": ""}, - {"column_name": "grid_label", "vocabulary": ""}, - {"column_name": "variable_id", "vocabulary": ""}, - {"column_name": "temporal_subset", "vocabulary": ""}, - {"column_name": "chunk_freq", "vocabulary": ""}, - {"column_name": "platform", "vocabulary": ""}, - {"column_name": "cell_methods", "vocabulary": ""}, - {"column_name": "path", "vocabulary": ""} - ], - "assets": {"column_name": "path", "format": "netcdf","format_column_name": None}, - "aggregation_control": { - "variable_column_name": "variable_id", - "groupby_attrs": [ - "source_id", - "experiment_id", - "frequency", - "member_id", - "modeling_realm", - "variable_id", - "chunk_freq" - ], - "aggregations": [ - {"type": "union", "attribute_name": "variable_id", "options": {}}, - { - "type": "join_existing", - "attribute_name": "temporal_subset", - "options": { - "dim": "time", - "coords": "minimal", - "compat": "override" - } - } - ] - }, - "id": "", - "description": None, - "title": None, - "last_updated": "2023-05-07T16:35:52Z", - "catalog_file": str(csv_path) - } - json.dump(json_str, output) - - with open(csv_path, "w") as output: - output.write(",".join(columns) + "\n") - path = Path(pp_dir) - for name in path.iterdir(): - realm = name.stem - if not name.is_dir() or str(name.stem).startswith("."): continue - full = name / "ts/monthly/1yr" - for file_ in full.iterdir(): - attrs = {column: "" for column in columns} - if not str(file_).endswith(".nc"): continue - attrs["activity_id"] = "dev" - attrs["experiment_id"] = "c96L65_am5f4b4r1-newrad_amip" - attrs["modeling_realm"] = realm - attrs["frequency"] = "monthly" - attrs["member_id"] = "na" - attrs["variable_id"] = str(file_.stem).split(".")[-1] - attrs["path"] = str(file_) - output.write(",".join([attrs[column] for column in columns]) + "\n") diff --git a/tests/mdtf_timeslice_catalog.yaml b/tests/mdtf_timeslice_catalog.yaml deleted file mode 100644 index 1da1594..0000000 --- a/tests/mdtf_timeslice_catalog.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Catalog headers -# The headerlist is expected column names in your catalog/csv file. This is usually -# determined by the users in conjuction with the ESM collection specification standards -# and the appropriate workflows. -headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", - "frequency", "realm", "table_id", - "member_id", "grid_label", "variable_id", - "time_range", "chunk_freq", "grid_label", "platform", - "dimensions", "cell_methods", "standard_name","path"] - -output_path_template: ['NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', - 'source_id', 'experiment_id', 'NA', 'platform', 'custom_pp', - 'realm', 'cell_methods', 'frequency', 'chunk_freq'] - -output_file_template: ['realm', 'time_range', 'variable_id'] - diff --git a/freanalysis_aerosol/README.md b/user-analysis-scripts/freanalysis_aerosol/README.md similarity index 100% rename from freanalysis_aerosol/README.md rename to user-analysis-scripts/freanalysis_aerosol/README.md diff --git a/freanalysis_aerosol/freanalysis_aerosol/__init__.py b/user-analysis-scripts/freanalysis_aerosol/freanalysis_aerosol/__init__.py similarity index 100% rename from freanalysis_aerosol/freanalysis_aerosol/__init__.py rename to user-analysis-scripts/freanalysis_aerosol/freanalysis_aerosol/__init__.py diff --git a/freanalysis_aerosol/pyproject.toml b/user-analysis-scripts/freanalysis_aerosol/pyproject.toml similarity index 100% rename from freanalysis_aerosol/pyproject.toml rename to user-analysis-scripts/freanalysis_aerosol/pyproject.toml diff --git a/freanalysis_clouds/README.md b/user-analysis-scripts/freanalysis_clouds/README.md similarity index 100% rename from freanalysis_clouds/README.md rename to user-analysis-scripts/freanalysis_clouds/README.md diff --git a/freanalysis_clouds/freanalysis_clouds/__init__.py b/user-analysis-scripts/freanalysis_clouds/freanalysis_clouds/__init__.py similarity index 100% rename from freanalysis_clouds/freanalysis_clouds/__init__.py rename to user-analysis-scripts/freanalysis_clouds/freanalysis_clouds/__init__.py diff --git a/freanalysis_clouds/pyproject.toml b/user-analysis-scripts/freanalysis_clouds/pyproject.toml similarity index 100% rename from freanalysis_clouds/pyproject.toml rename to user-analysis-scripts/freanalysis_clouds/pyproject.toml diff --git a/tests/test_freanalysis_clouds.py b/user-analysis-scripts/freanalysis_clouds/tests/test_freanalysis_clouds.py similarity index 87% rename from tests/test_freanalysis_clouds.py rename to user-analysis-scripts/freanalysis_clouds/tests/test_freanalysis_clouds.py index 87e24c1..ea54443 100644 --- a/tests/test_freanalysis_clouds.py +++ b/user-analysis-scripts/freanalysis_clouds/tests/test_freanalysis_clouds.py @@ -5,10 +5,11 @@ from tempfile import TemporaryDirectory import sys -from freanalysis.plugins import list_plugins, plugin_requirements, run_plugin +from analysis_scripts import list_plugins, plugin_requirements, run_plugin import catalogbuilder from catalogbuilder.scripts import gen_intake_gfdl + def download_test_data(stem): """Downloads test datasets from a FTP server. @@ -34,6 +35,7 @@ def download_test_data(stem): ftp.retrbinary(f"RETR {name}", open(data_directory / name, "wb").write) return catalog_root.resolve() + def plugin(json, pngs_directory="pngs"): """Run the plugin to create the figure. @@ -48,13 +50,14 @@ def plugin(json, pngs_directory="pngs"): def test_freanalysis_clouds(): - with TemporaryDirectory() as tmp: chdir(Path(tmp)) path = download_test_data(stem=tmp) yaml = Path(__file__).resolve().parent / "mdtf_timeslice_catalog.yaml" - outputpath = Path(__file__).resolve().parent / "data-catalog" - #Creates data catalog using the scripts in catalogbuilder - csv, json = gen_intake_gfdl.create_catalog(input_path=str(path),output_path=outputpath,config=str(yaml)) + outputpath = Path(__file__).resolve().parent / "data-catalog" + # Creates data catalog using the scripts in catalogbuilder + csv, json = gen_intake_gfdl.create_catalog(input_path=str(path), + output_path=outputpath, + config=str(yaml)) print(json,csv) plugin(json) diff --git a/freanalysis_land/README.md b/user-analysis-scripts/freanalysis_land/README.md similarity index 100% rename from freanalysis_land/README.md rename to user-analysis-scripts/freanalysis_land/README.md diff --git a/freanalysis_land/freanalysis_land/__init__.py b/user-analysis-scripts/freanalysis_land/freanalysis_land/__init__.py similarity index 100% rename from freanalysis_land/freanalysis_land/__init__.py rename to user-analysis-scripts/freanalysis_land/freanalysis_land/__init__.py diff --git a/freanalysis_land/freanalysis_land/land.py b/user-analysis-scripts/freanalysis_land/freanalysis_land/land.py similarity index 100% rename from freanalysis_land/freanalysis_land/land.py rename to user-analysis-scripts/freanalysis_land/freanalysis_land/land.py diff --git a/freanalysis_land/pyproject.toml b/user-analysis-scripts/freanalysis_land/pyproject.toml similarity index 100% rename from freanalysis_land/pyproject.toml rename to user-analysis-scripts/freanalysis_land/pyproject.toml diff --git a/tests/test_freanalysis_land.py b/user-analysis-scripts/freanalysis_land/tests/test_freanalysis_land.py similarity index 70% rename from tests/test_freanalysis_land.py rename to user-analysis-scripts/freanalysis_land/tests/test_freanalysis_land.py index 99c778c..19225fa 100644 --- a/tests/test_freanalysis_land.py +++ b/user-analysis-scripts/freanalysis_land/tests/test_freanalysis_land.py @@ -1,10 +1,10 @@ - - from freanalysis_land.land import LandAnalysisScript + def test_land_analysis_script(): land = LandAnalysisScript() - land.run_analysis("/work/a2p/lm4p2sc_GSWP3_hist_irr_catalog.json","/work/a2p/") - + land.run_analysis("/work/a2p/lm4p2sc_GSWP3_hist_irr_catalog.json", "/work/a2p/") + -test_land_analysis_script() +if __name_ == "__main__": + test_land_analysis_script() diff --git a/freanalysis_radiation/README.md b/user-analysis-scripts/freanalysis_radiation/README.md similarity index 100% rename from freanalysis_radiation/README.md rename to user-analysis-scripts/freanalysis_radiation/README.md diff --git a/freanalysis_radiation/freanalysis_radiation/__init__.py b/user-analysis-scripts/freanalysis_radiation/freanalysis_radiation/__init__.py similarity index 100% rename from freanalysis_radiation/freanalysis_radiation/__init__.py rename to user-analysis-scripts/freanalysis_radiation/freanalysis_radiation/__init__.py diff --git a/freanalysis_radiation/pyproject.toml b/user-analysis-scripts/freanalysis_radiation/pyproject.toml similarity index 100% rename from freanalysis_radiation/pyproject.toml rename to user-analysis-scripts/freanalysis_radiation/pyproject.toml