diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8483669..537f1c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout source diff --git a/ci/environment-3.9.yml b/ci/environment-3.9.yml deleted file mode 100644 index b7eb73f..0000000 --- a/ci/environment-3.9.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: access-nri-intake-test -channels: - - conda-forge - - accessnri - - defaults -dependencies: - - python>=3.9.2 # see https://github.com/dask/distributed/issues/7956 - - cftime - - ecgtools>=2023.7.13 - - intake>=0.7.0 - - intake-dataframe-catalog>=0.2.4 - - intake-esm>=2023.11.10 - - jsonschema - - pooch - - pre-commit - - pytest - - xarray - - pip - - pip: - - codecov - - pytest-cov diff --git a/src/access_nri_intake/catalog/manager.py b/src/access_nri_intake/catalog/manager.py index b6a4e9b..e7adb05 100644 --- a/src/access_nri_intake/catalog/manager.py +++ b/src/access_nri_intake/catalog/manager.py @@ -4,7 +4,6 @@ """Manager for adding/updating intake sources in an intake-dataframe-catalog like the ACCESS-NRI catalog""" import os -from typing import Optional, Union import intake from intake_dataframe_catalog.core import DfFileCatalog, DfFileCatalogError @@ -63,10 +62,10 @@ def build_esm( name: str, description: str, builder, - path: Union[str, list[str]], + path: str | list[str], translator=DefaultTranslator, - metadata: Optional[dict] = None, - directory: Optional[str] = None, + metadata: dict | None = None, + directory: str | None = None, overwrite: bool = False, **kwargs, ): @@ -131,7 +130,7 @@ def load( path: str, driver: str = "esm_datastore", translator=DefaultTranslator, - metadata: Optional[dict] = None, + metadata: dict | None = None, **kwargs, ): """ diff --git a/src/access_nri_intake/catalog/translators.py b/src/access_nri_intake/catalog/translators.py index 4e77e51..84f215c 100644 --- a/src/access_nri_intake/catalog/translators.py +++ b/src/access_nri_intake/catalog/translators.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from functools import partial -from typing import Callable, Optional +from typing import Callable import pandas as pd import tlz @@ -120,7 +120,7 @@ def _default_translator(self, column: str) -> pd.Series: return pd.Series([val] * len_df) - def translate(self, groupby: Optional[list[str]] = None) -> pd.DataFrame: + def translate(self, groupby: list[str] | None = None) -> pd.DataFrame: """ Return the translated :py:class:`~pandas.DataFrame` of metadata and merge into set of set of rows with unique values of the columns specified. @@ -168,7 +168,7 @@ def _unique_values(series): return df[self.columns] # Preserve ordering def set_dispatch( - self, core_colname: str, func: Callable, input_name: Optional[str] = None + self, core_colname: str, func: Callable, input_name: str | None = None ): """ Set a dispatch function for a column. Typically only required when either: @@ -549,10 +549,10 @@ class _DispatchKeys: Data class to store the keys for the dispatch dictionary in the Translator classes """ - model: Optional[str] = None - realm: Optional[str] = None - frequency: Optional[str] = None - variable: Optional[str] = None + model: str | None = None + realm: str | None = None + frequency: str | None = None + variable: str | None = None def _cmip_realm_translator(series) -> pd.Series: diff --git a/src/access_nri_intake/source/builders.py b/src/access_nri_intake/source/builders.py index 70c7cf9..12dcfbc 100644 --- a/src/access_nri_intake/source/builders.py +++ b/src/access_nri_intake/source/builders.py @@ -7,7 +7,6 @@ import re import traceback from pathlib import Path -from typing import Optional, Union import xarray as xr from ecgtools.builder import INVALID_ASSET, TRACEBACK, Builder @@ -58,14 +57,14 @@ class BaseBuilder(Builder): def __init__( self, - path: Union[str, list[str]], + path: str | list[str], depth: int = 0, - exclude_patterns: Optional[list[str]] = None, - include_patterns: Optional[list[str]] = None, + exclude_patterns: list[str] | None = None, + include_patterns: list[str] | None = None, data_format: str = "netcdf", - groupby_attrs: Optional[list[str]] = None, - aggregations: Optional[list[dict]] = None, - storage_options: Optional[dict] = None, + groupby_attrs: list[str] | None = None, + aggregations: list[dict] | None = None, + storage_options: dict | None = None, joblib_parallel_kwargs: dict = {"n_jobs": multiprocessing.cpu_count()}, ): """ @@ -120,7 +119,7 @@ def parse(self): self._parse() return self - def _save(self, name: str, description: str, directory: Optional[str]): + def _save(self, name: str, description: str, directory: str | None): super().save( name=name, path_column_name=PATH_COLUMN, @@ -135,9 +134,7 @@ def _save(self, name: str, description: str, directory: Optional[str]): to_csv_kwargs={"compression": "gzip"}, ) - def save( - self, name: str, description: str, directory: Optional[str] = None - ) -> None: + def save(self, name: str, description: str, directory: str | None = None) -> None: """ Save datastore contents to a file. @@ -221,10 +218,10 @@ def parser(file): def parse_access_filename( cls, filename: str, - patterns: Optional[list[str]] = None, + patterns: list[str] | None = None, frequencies: dict = FREQUENCIES, redaction_fill: str = "X", - ) -> tuple[str, Optional[str], Optional[str]]: + ) -> tuple[str, str | None, str | None]: """ Parse an ACCESS model filename and return a file id and any time information diff --git a/src/access_nri_intake/source/utils.py b/src/access_nri_intake/source/utils.py index 4d0ab76..bb4b903 100644 --- a/src/access_nri_intake/source/utils.py +++ b/src/access_nri_intake/source/utils.py @@ -7,7 +7,6 @@ from dataclasses import asdict, dataclass, field from datetime import timedelta from pathlib import Path -from typing import Optional, Union import cftime import xarray as xr @@ -30,10 +29,10 @@ class _AccessNCFileInfo: more explicit tests are probably more important than the slight redundancy. """ - filename: Union[str, Path] + filename: str | Path file_id: str path: str - filename_timestamp: Optional[str] + filename_timestamp: str | None frequency: str start_date: str end_date: str @@ -43,7 +42,7 @@ class _AccessNCFileInfo: variable_cell_methods: list[str] variable_units: list[str] - def to_dict(self) -> dict[str, Union[str, list[str]]]: + def to_dict(self) -> dict[str, str | list[str]]: """ Return a dictionary representation of the NcFileInfo object """ @@ -139,7 +138,7 @@ def _guess_start_end_dates(ts, te, frequency): def get_timeinfo( ds: xr.Dataset, - filename_frequency: Optional[str], + filename_frequency: str | None, time_dim: str, ) -> tuple[str, str, str]: """ @@ -177,7 +176,7 @@ def _todate(t): time_format = "%Y-%m-%d, %H:%M:%S" ts = None te = None - frequency: Union[str, tuple[Optional[int], str]] = "fx" + frequency: str | tuple[int | None, str] = "fx" has_time = time_dim in ds if has_time: