From c0c1bdd6abc80619130af06f94efffe96d08c354 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 2 Mar 2023 14:42:15 -0300 Subject: [PATCH 1/3] remove StreamConfig --- docs/source/examples/usage/StreamConfig.ipynb | 121 ------------------ docs/source/usage.rst | 29 ----- 2 files changed, 150 deletions(-) delete mode 100644 docs/source/examples/usage/StreamConfig.ipynb diff --git a/docs/source/examples/usage/StreamConfig.ipynb b/docs/source/examples/usage/StreamConfig.ipynb deleted file mode 100644 index 5afc4726..00000000 --- a/docs/source/examples/usage/StreamConfig.ipynb +++ /dev/null @@ -1,121 +0,0 @@ -{ - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2-final" - }, - "orig_nbformat": 2, - "kernelspec": { - "name": "python_defaultSpec_1597863786755", - "display_name": "Python 3.8.2 64-bit ('ioosqc38': conda)" - } - }, - "nbformat": 4, - "nbformat_minor": 2, - "cells": [ - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Install QC library\n", - "#!pip install git+git://github.com/ioos/ioos_qc.git\n", - "\n", - "# # Alternative installation (install specific branch):\n", - "# !pip uninstall -y ioos_qc\n", - "# !pip install git+git://github.com/ioos/ioos_qc.git@BRANCHNAME\n", - "\n", - "# Alternative installation (run with local updates):\n", - "#!pip uninstall -y ioos_qc\n", - "import sys\n", - "from pathlib import Path\n", - "basedir = Path().absolute()\n", - "libdir = basedir.parent.parent.parent.parent\n", - "sys.path.append(str(libdir))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Load a StreamConfig object from a Python dictionary" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": "OrderedDict([('qartod',\n {'aggregate': {},\n 'gross_range_test': {'suspect_span': [1, 11],\n 'fail_span': [0, 12]}})])" - }, - "metadata": {}, - "execution_count": 63 - } - ], - "source": [ - "from ioos_qc.config import StreamConfig\n", - "config = {\n", - " 'qartod': {\n", - " 'aggregate': {},\n", - " 'gross_range_test': {\n", - " 'suspect_span': [1, 11],\n", - " 'fail_span': [0, 12],\n", - " }\n", - " }\n", - "}\n", - "c = StreamConfig(config)\n", - "c.config" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Load a StreamConfig object from a YAML string" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": "OrderedDict([('qartod',\n {'aggregate': None,\n 'gross_range_test': {'suspect_span': [1, 11],\n 'fail_span': [0, 12]}})])" - }, - "metadata": {}, - "execution_count": 62 - } - ], - "source": [ - "from ioos_qc.config import StreamConfig\n", - "config = \"\"\"\n", - " qartod:\n", - " aggregate:\n", - " gross_range_test:\n", - " suspect_span: [1, 11]\n", - " fail_span: [0, 12]\n", - "\"\"\"\n", - "c = StreamConfig(config)\n", - "c.config" - ] - } - ] -} \ No newline at end of file diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 4d0aa726..d28303f1 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -69,7 +69,6 @@ Configurations Configuration objects represent a collection of quality control tests to run and the parameters for each one. There are three main types of `Config` objects: -- StreamConfig_: This configures QC tests for a single stream of data like a ``list``, ``tuple``, ``numpy.ndarray``, ``dask.array``, ``pandas.Series``, ``netCDF4.Variable``, or ``xarray.DataArray``. This can be used standalone, or as a building block for the following more complex configs. - ContextConfig_: This defines a collection of ``StreamConfig`` objects. These can be applied to multiple variables provided in a ``pandas.DataFrame``, ``dask.DataFrame``, ``netCDF4.Dataset``, or ``xarray.Dataset``. Optionally, these configs can be constrained to specific time domains (``windows``) -- and/or spatial domains (``regions``). - Config_: A collection of ``ContextConfig`` objects, suitable for configuring a single input dataset to be broken up by region and time window before having QC checks applied. @@ -83,34 +82,6 @@ In addition, the ``ContextConfig`` and ``Config`` objects can be initialized wit - netCDF4/xarray filepath (``str`` or ``Path`` object) or ``Dataset`` -StreamConfig -~~~~~~~~~~~~ -A ``StreamConfig`` object defines a specific `ioos_qc` test module and test function along with the configuration parameters in which to run it with. - -.. note:: - - In earlier versions, ``StreamConfig`` was known as ``QcConfig``. - -Usage -^^^^^ - -.. code-block:: python - :linenos: - :caption: A basic ``StreamConfig`` object - - from ioos_qc.config import StreamConfig - - config = { - 'qartod': { - 'gross_range_test': { - 'suspect_span': [1, 11], - 'fail_span': [0, 12], - } - } - } - c = StreamConfig(config) - - ContextConfig ~~~~~~~~~~~~~ A ``ContextConfig`` object defines multiple ``StreamConfig`` objects as well as optional `region` and `window` objects. From 291ed693d1746fd05f4ee4420182341494314357 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 15 Mar 2024 14:12:48 -0300 Subject: [PATCH 2/3] use ConfigTypes instead --- docs/source/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index d28303f1..66590238 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -69,7 +69,7 @@ Configurations Configuration objects represent a collection of quality control tests to run and the parameters for each one. There are three main types of `Config` objects: -- ContextConfig_: This defines a collection of ``StreamConfig`` objects. These can be applied to multiple variables provided in a ``pandas.DataFrame``, ``dask.DataFrame``, ``netCDF4.Dataset``, or ``xarray.Dataset``. Optionally, these configs can be constrained to specific time domains (``windows``) -- and/or spatial domains (``regions``). +- ContextConfig_: This defines a collection of ``ConfigTypes`` objects. These can be applied to multiple variables provided in a ``pandas.DataFrame``, ``dask.DataFrame``, ``netCDF4.Dataset``, or ``xarray.Dataset``. Optionally, these configs can be constrained to specific time domains (``windows``) -- and/or spatial domains (``regions``). - Config_: A collection of ``ContextConfig`` objects, suitable for configuring a single input dataset to be broken up by region and time window before having QC checks applied. Each configuration type can be initialized through Python objects or from files and can be represented in the following ways: @@ -84,7 +84,7 @@ In addition, the ``ContextConfig`` and ``Config`` objects can be initialized wit ContextConfig ~~~~~~~~~~~~~ -A ``ContextConfig`` object defines multiple ``StreamConfig`` objects as well as optional `region` and `window` objects. +A ``ContextConfig`` object defines multiple ``ConfigTypes`` objects as well as optional `region` and `window` objects. region ^^^^^^ From a8e78aabbeed906dac91041d52b8b00674afd822 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 15 Mar 2024 14:13:28 -0300 Subject: [PATCH 3/3] don't mentioned StreamConfig --- ioos_qc/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ioos_qc/config.py b/ioos_qc/config.py index 28f5f8b9..496841b9 100644 --- a/ioos_qc/config.py +++ b/ioos_qc/config.py @@ -261,7 +261,7 @@ def __init__(self, source, version=None, default_stream_key='_stream'): # Return a list with just one ContextConfig self._calls += list(ContextConfig(self.config).calls) elif dict_depth(self.config) >= 4: - # This is a StreamConfig + # This is a Config self._calls += list(ContextConfig(odict(streams=self.config)).calls) else: # This is a QcConfig @@ -353,7 +353,7 @@ def add(self, source) -> None: class ContextConfig: - """A collection of a Region, a TimeWindow and a list of StreamConfig objects + """A collection of a Region, a TimeWindow and a list of Config objects Defines a set of quality checks to run against multiple input streams. This can include a region and a time window to subset any DataStreams by before running checks. @@ -366,11 +366,11 @@ class ContextConfig: ending: 2020-04-01T00:00:00Z streams: variable1: # stream_id - qartod: # StreamConfig + qartod: # Config location_test: bbox: [-80, 40, -70, 60] variable2: # stream_id - qartod: # StreamConfig + qartod: # Config gross_range_test: suspect_span: [1, 11] fail_span: [0, 12] @@ -382,7 +382,7 @@ class ContextConfig: config (odict): dict representation of the parsed ContextConfig source region (GeometryCollection): A `shapely` object representing the valid geographic region window (namedtuple): A TimeWindow object representing the valid time period - streams (odict): dict representation of the parsed StreamConfig objects + streams (odict): dict representation of the parsed Config objects """ def __init__(self, source: ConfigTypes):