From 58b1553573d9d918ca89c601ddf7542862d7c7c5 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Tue, 31 Oct 2023 15:17:44 -0400 Subject: [PATCH 01/14] initial commit of Harmony service code (cli, adapter, downloader,utils) --- concatenator/harmony/__init__.py | 0 concatenator/harmony/cli.py | 33 ++++++ concatenator/harmony/download_worker.py | 113 +++++++++++++++++++++ concatenator/harmony/service_adapter.py | 128 ++++++++++++++++++++++++ concatenator/harmony/util.py | 98 ++++++++++++++++++ 5 files changed, 372 insertions(+) create mode 100644 concatenator/harmony/__init__.py create mode 100644 concatenator/harmony/cli.py create mode 100644 concatenator/harmony/download_worker.py create mode 100644 concatenator/harmony/service_adapter.py create mode 100644 concatenator/harmony/util.py diff --git a/concatenator/harmony/__init__.py b/concatenator/harmony/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/concatenator/harmony/cli.py b/concatenator/harmony/cli.py new file mode 100644 index 0000000..3329095 --- /dev/null +++ b/concatenator/harmony/cli.py @@ -0,0 +1,33 @@ +"""A Harmony CLI wrapper around the concatenate-batcher""" +from argparse import ArgumentParser + +import harmony + +from concatenator.harmony.service_adapter import StitcheeAdapter as HarmonyAdapter + + +def main(config: harmony.util.Config = None) -> None: + """Parse command line arguments and invoke the service to respond to them. + + Parameters + ---------- + config : harmony.util.Config + harmony.util.Config is injectable for tests + + Returns + ------- + None + """ + parser = ArgumentParser( + prog="Stitchee", description="Run the STITCH by Extending a dimEnsion service" + ) + harmony.setup_cli(parser) + args = parser.parse_args() + if harmony.is_harmony_cli(args): + harmony.run_cli(parser, args, HarmonyAdapter, cfg=config) + else: + parser.error("Only --harmony CLIs are supported") + + +if __name__ == "__main__": + main() diff --git a/concatenator/harmony/download_worker.py b/concatenator/harmony/download_worker.py new file mode 100644 index 0000000..0c177af --- /dev/null +++ b/concatenator/harmony/download_worker.py @@ -0,0 +1,113 @@ +"""A utility for downloading multiple granules simultaneously""" + +import queue +import re +from copy import deepcopy +from multiprocessing import Manager, Process +from os import cpu_count +from pathlib import Path +from urllib.parse import urlparse + +from harmony.logging import build_logger +from harmony.util import download + + +def multi_core_download(urls, destination_dir, access_token, cfg, process_count=None): + """ + A method which automagically scales downloads to the number of CPU + cores. For further explaination, see documentation on "multi-track + drifting" + + Parameters + ---------- + urls : list + list of urls to download + destination_dir : str + output path for downloaded files + access_token : str + access token as provided in Harmony input + cfg : dict + Harmony configuration information + process_count : int + Number of worker processes to run (expected >= 1) + + Returns + ------- + list + list of downloaded files as pathlib.Path objects + """ + + if process_count is None: + process_count = cpu_count() + + with Manager() as manager: + url_queue = manager.Queue(len(urls)) + path_list = manager.list() + + for url in urls: + url_queue.put(url) + + # Spawn worker processes + processes = [] + for _ in range(process_count): + download_process = Process( + target=_download_worker, + args=(url_queue, path_list, destination_dir, access_token, cfg), + ) + processes.append(download_process) + download_process.start() + + # Ensure worker processes exit successfully + for process in processes: + process.join() + if process.exitcode != 0: + raise RuntimeError(f"Download failed - exit code: {process.exitcode}") + + process.close() + + path_list = deepcopy(path_list) # ensure GC can cleanup multiprocessing + + return [Path(path) for path in path_list] + + +def _download_worker(url_queue, path_list, destination_dir, access_token, cfg): + """ + A method to be executed in a separate process which processes the url_queue + and places paths to completed downloads into the path_list. Downloads are + handled by harmony.util.download + + Parameters + ---------- + url_queue : queue.Queue + URLs to process - should be filled from start and only decreases + path_list : list + paths to completed file downloads + destination_dir : str + output path for downloaded files + access_token : str + access token as provided in Harmony input + cfg : dict + Harmony configuration information + """ + + logger = build_logger(cfg) + + while not url_queue.empty(): + try: + url = url_queue.get_nowait() + except queue.Empty: + break + + path = Path( + download(url, destination_dir, logger=logger, access_token=access_token, cfg=cfg) + ) + filename_match = re.match(r".*\/(.+\..+)", urlparse(url).path) + + if filename_match is not None: + filename = filename_match.group(1) + dest_path = path.parent.joinpath(filename) + path = path.rename(dest_path) + else: + logger.warning("Origin filename could not be assertained - %s", url) + + path_list.append(str(path)) diff --git a/concatenator/harmony/service_adapter.py b/concatenator/harmony/service_adapter.py new file mode 100644 index 0000000..5905229 --- /dev/null +++ b/concatenator/harmony/service_adapter.py @@ -0,0 +1,128 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import uuid4 + +import pystac +from harmony.adapter import BaseHarmonyAdapter +from harmony.util import bbox_to_geometry +from pystac import Item +from pystac.item import Asset + +from concatenator.harmony.download_worker import multi_core_download +from concatenator.harmony.util import ( + _get_netcdf_urls, + _get_output_bounding_box, + _get_output_date_range, +) +from concatenator.stitchee import stitchee + + +class StitcheeAdapter(BaseHarmonyAdapter): + """ + A harmony-service-lib wrapper around the concatenate-batcher module. + This wrapper does not support Harmony calls that do not have STAC catalogs + as support for this behavior is being depreciated in harmony-service-lib + """ + + def __init__(self, message, catalog=None, config=None): + """ + Constructs the adapter + + Parameters + ---------- + message : harmony.Message + The Harmony input which needs acting upon + catalog : pystac.Catalog + A STAC catalog containing the files on which to act + config : harmony.util.Config + The configuration values for this runtime environment. + """ + super().__init__(message, catalog=catalog, config=config) + + def invoke(self): + """ + Primary entrypoint into the service wrapper. Overrides BaseHarmonyAdapter.invoke + """ + if not self.catalog: + # Message-only support is being depreciated in Harmony, so we should expect to + # only see requests with catalogs when invoked with a newer Harmony instance + # https://github.com/nasa/harmony-service-lib-py/blob/21bcfbda17caf626fb14d2ac4f8673be9726b549/harmony/adapter.py#L71 + raise RuntimeError("Invoking Batchee without a STAC catalog is not supported") + + return self.message, self.process_catalog(self.catalog) + + def process_catalog(self, catalog: pystac.Catalog): + """Converts a list of STAC catalogs into a list of lists of STAC catalogs.""" + self.logger.info("process_catalog() started.") + try: + result = catalog.clone() + result.id = str(uuid4()) + result.clear_children() + + # Get all the items from the catalog, including from child or linked catalogs + items = list(self.get_all_catalog_items(catalog)) + + self.logger.info(f"length of items==={len(items)}.") + + # Quick return if catalog contains no items + if len(items) == 0: + return result + + # # --- Get granule filepaths (urls) --- + netcdf_urls: list[str] = _get_netcdf_urls(items) + self.logger.info(f"netcdf_urls==={netcdf_urls}.") + + # -- Process metadata -- + bounding_box: list | None = _get_output_bounding_box(items) + datetimes = _get_output_date_range(items) + + # Items did not have a bbox; valid under spec + if bounding_box and len(bounding_box) == 0: + bounding_box = None + + # -- Perform merging -- + collection = self._get_item_source(items[0]).collection + filename = f"{collection}_merged.nc4" + + with TemporaryDirectory() as temp_dir: + self.logger.info("Starting granule downloads") + input_files = multi_core_download( + netcdf_urls, temp_dir, self.message.accessToken, self.config + ) + self.logger.info("Finished granule downloads") + + output_path = str(Path(temp_dir).joinpath(filename).resolve()) + + # # --- Run STITCHEE --- + stitchee( + input_files, + output_path, + write_tmp_flat_concatenated=False, + keep_tmp_files=False, + concat_dim="mirror_step", # This is currently set only for TEMPO + logger=self.logger, + ) + staged_url = self._stage(output_path, filename, "application/x-netcdf4") + + # -- Output to STAC catalog -- + result.clear_items() + properties = dict( + start_datetime=datetimes["start_datetime"], end_datetime=datetimes["end_datetime"] + ) + + item = Item( + str(uuid4()), bbox_to_geometry(bounding_box), bounding_box, None, properties + ) + asset = Asset( + staged_url, title=filename, media_type="application/x-netcdf4", roles=["data"] + ) + item.add_asset("data", asset) + result.add_item(item) + + self.logger.info("STAC catalog creation complete.") + + return result + + except Exception as service_exception: + self.logger.error(service_exception, exc_info=1) + raise service_exception diff --git a/concatenator/harmony/util.py b/concatenator/harmony/util.py new file mode 100644 index 0000000..f77aeb9 --- /dev/null +++ b/concatenator/harmony/util.py @@ -0,0 +1,98 @@ +"""Misc utility functions""" +from datetime import datetime + +from pystac import Asset, Item + +VALID_EXTENSIONS = (".nc4", ".nc") +VALID_MEDIA_TYPES = ["application/x-netcdf", "application/x-netcdf4"] + + +def _is_netcdf_asset(asset: Asset) -> bool: + """Check that a `pystac.Asset` is a valid NetCDF-4 granule. This can be + ascertained via either the media type or by checking the extension of + granule itself if that media type is absent. + + """ + return asset.media_type in VALID_MEDIA_TYPES or ( + asset.media_type is None and asset.href.lower().endswith(VALID_EXTENSIONS) + ) + + +def _get_item_url(item: Item) -> str | None: + """Check the `pystac.Item` for the first asset with the `data` role and a + valid input format. If there are no matching assets, return None + + """ + return next( + ( + asset.href + for asset in item.assets.values() + if "data" in (asset.roles or []) and _is_netcdf_asset(asset) + ), + None, + ) + + +def _get_netcdf_urls(items: list[Item]) -> list[str]: + """Iterate through a list of `pystac.Item` instances, from the input + `pystac.Catalog`. Extract the `pystac.Asset.href` for the first asset + of each item that has a role of "data". If there are any items that do + not have a data asset, then raise an exception. + + """ + catalog_urls = [_get_item_url(item) for item in items] + + if None in catalog_urls: + raise RuntimeError("Some input granules do not have NetCDF-4 assets.") + + return catalog_urls # type: ignore[return-value] + + +def _get_output_bounding_box(input_items: list[Item]) -> list[float]: + """Create a bounding box that is the maximum combined extent of all input + `pystac.Item` bounding box extents. + + """ + bounding_box = input_items[0].bbox + + for item in input_items: + bounding_box[0] = min(bounding_box[0], item.bbox[0]) + bounding_box[1] = min(bounding_box[1], item.bbox[1]) + bounding_box[2] = max(bounding_box[2], item.bbox[2]) + bounding_box[3] = max(bounding_box[3], item.bbox[3]) + + return bounding_box + + +def _get_output_date_range(input_items: list[Item]) -> dict[str, str]: + """Create a dictionary of start and end datetime, which encompasses the + full temporal range of all input `pystac.Item` instances. This output + dictionary will be used for the `properties` of the output Zarr store + `pystac.Item`. + + """ + start_datetime, end_datetime = _get_item_date_range(input_items[0]) + + for item in input_items: + new_start_datetime, new_end_datetime = _get_item_date_range(item) + start_datetime = min(start_datetime, new_start_datetime) + end_datetime = max(end_datetime, new_end_datetime) + + return {"start_datetime": start_datetime.isoformat(), "end_datetime": end_datetime.isoformat()} + + +def _get_item_date_range(item: Item) -> tuple[datetime, datetime]: + """A helper function to retrieve the temporal range from a `pystac.Item` + instance. If the `pystac.Item.datetime` property exists, there is a + single datetime associated with the granule, otherwise there will be a + start and end time contained within the `pystac.Item` metadata. + + """ + if item.datetime is None: + start_datetime = item.common_metadata.start_datetime + end_datetime = item.common_metadata.end_datetime + else: + start_datetime = item.datetime + end_datetime = item.datetime + + return start_datetime, end_datetime From db16b9f70332238ecc0cd6b9ceacc194e5bae763 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Tue, 31 Oct 2023 15:18:23 -0400 Subject: [PATCH 02/14] update poetry lock --- poetry.lock | 315 ++++++++++++++++++++++++++++------------------------ 1 file changed, 168 insertions(+), 147 deletions(-) diff --git a/poetry.lock b/poetry.lock index b306a7c..705cafe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "black" @@ -20,7 +20,6 @@ files = [ {file = "black-23.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:840015166dbdfbc47992871325799fd2dc0dcf9395e401ada6d88fe11498abad"}, {file = "black-23.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:037e9b4664cafda5f025a1728c50a9e9aedb99a759c89f760bd83730e76ba884"}, {file = "black-23.10.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:7cb5936e686e782fddb1c73f8aa6f459e1ad38a6a7b0e54b403f1f05a1507ee9"}, - {file = "black-23.10.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:7670242e90dc129c539e9ca17665e39a146a761e681805c54fbd86015c7c84f7"}, {file = "black-23.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed45ac9a613fb52dad3b61c8dea2ec9510bf3108d4db88422bacc7d1ba1243d"}, {file = "black-23.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:6d23d7822140e3fef190734216cefb262521789367fbdc0b3f22af6744058982"}, {file = "black-23.10.1-py3-none-any.whl", hash = "sha256:d431e6739f727bb2e0495df64a6c7a5310758e87505f5f8cde9ff6c0f2d7e4fe"}, @@ -44,17 +43,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.28.60" +version = "1.28.74" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.28.60-py3-none-any.whl", hash = "sha256:d5f270c2c9a051f78c308cbba4268458e8df441057b73ba140742707ac1bc7ea"}, - {file = "boto3-1.28.60.tar.gz", hash = "sha256:dccb49cc10b31314b8553c6c9614c44b2249e0d0285d73f608a5d2010f6e1d82"}, + {file = "boto3-1.28.74-py3-none-any.whl", hash = "sha256:a12619d23d50523bdfa6bd2d0c124a9567a5434ec63e47df71c3e88fc9d482f6"}, + {file = "boto3-1.28.74.tar.gz", hash = "sha256:c4d89ff98cf53e1e74ed8c1c1bb13a7bd7d909e1c0d9ad25792ccfe2cb1575bc"}, ] [package.dependencies] -botocore = ">=1.31.60,<1.32.0" +botocore = ">=1.31.74,<1.32.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.7.0,<0.8.0" @@ -63,19 +62,19 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.31.60" +version = "1.31.74" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.31.60-py3-none-any.whl", hash = "sha256:b6de7a6a03ca3da18b78615a2cb5221c9fdb9483d3f50cb4281ae038b3f22d9f"}, - {file = "botocore-1.31.60.tar.gz", hash = "sha256:578470a15a5bd64f67437a81f23feccba85084167acf63c56acada2c1c1d95d8"}, + {file = "botocore-1.31.74-py3-none-any.whl", hash = "sha256:3eef070a8d8c4240aad07e5c89395002b140af213bcfc4dcd1d441e3ee4b3bee"}, + {file = "botocore-1.31.74.tar.gz", hash = "sha256:2549ca7dc0f9227d8692a56a133c1346642efb38736c7c3f1330724d937ad6ec"}, ] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" python-dateutil = ">=2.1,<3.0.0" -urllib3 = ">=1.25.4,<1.27" +urllib3 = {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""} [package.extras] crt = ["awscrt (==0.16.26)"] @@ -188,103 +187,135 @@ files = [ [package.dependencies] numpy = ">1.13.3" +[[package]] +name = "cftime" +version = "1.6.3" +description = "Time-handling functionality from netcdf4-python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "cftime-1.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b62d42546fa5c914dfea5b15a9aaed2087ea1211cc36d08c374502ef95892038"}, + {file = "cftime-1.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb6dd70b2ccabfe1a14b7fbb0bbdce0418e71697094373c0d573c880790fa291"}, + {file = "cftime-1.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9878bfd8c1c3f24184ecbd528f739ba46ebaceaf1c8a24d348d7befb117a285"}, + {file = "cftime-1.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:3cf6e216a4c06f9a628cdf8e9c9d5e8097fb3eb02dd087dd14ab3b18478a7271"}, + {file = "cftime-1.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d2c01456d9d7b46aa710a41d1c711a50d5ea259aff4a987d0e973d1093bc922"}, + {file = "cftime-1.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:80eb1170ce1639016f55760847f4aadd04b0312496c5bac2797e930914bba48d"}, + {file = "cftime-1.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d87dadd0824262bdd7493babd2a44447da0a22175ded8ae9e060a3aebec7c5d7"}, + {file = "cftime-1.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:0a38eb9f5c733a23e1714bd3ef2762ed5acee34f127670f8fb4ad6464946f6b3"}, + {file = "cftime-1.6.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2d113a01ab924445e61d65c26bbd95bc08e4a22878d3b947064bba056c884c4a"}, + {file = "cftime-1.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f11685663a6af97418908060492a07663c16d42519c139ca03c2ffb1377fd25"}, + {file = "cftime-1.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a98abb1d46d118e52b0611ce668a0b714b407be26177ef0581ecf5e95f894725"}, + {file = "cftime-1.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:4d6fbd5f41b322cfa7b0ac3aaadeceb4450100a164b5bccbbb9e7c5048489a88"}, + {file = "cftime-1.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bedb577bc8b8f3f10f5336c0792e5dae88605781890f50f36b45bb46907968e8"}, + {file = "cftime-1.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:022dabf1610cdd04a693e730fa8f71d307059717f29dba921e7486e553412bb4"}, + {file = "cftime-1.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbf782ab4ac0605bdec2b941952c897595613203942b7f8c2fccd17efa5147df"}, + {file = "cftime-1.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:9eb177a02db7cd84aa6962278e4bd2d3106a545de82e6aacd9404f1e153661db"}, + {file = "cftime-1.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b86be8c2f254147be4ba88f12099466dde457a4a3a21de6c69d52a7224c13ae"}, + {file = "cftime-1.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:523b9a6bf03f5e36407979e248381d0fcab2d225b915bbde77d00c6dde192b90"}, + {file = "cftime-1.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a14d2c7d22fd2a6dfa6ad563283b6d6679f1df95e0ed8d14b8f284dad402887"}, + {file = "cftime-1.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:d9b00c2844c7a1701d8ede5336b6321dfee256ceab81a34a1aff0483d56891a6"}, +] + +[package.dependencies] +numpy = {version = ">1.13.3", markers = "python_version < \"3.12.0.rc1\""} + [[package]] name = "charset-normalizer" -version = "3.3.0" +version = "3.3.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.3.0.tar.gz", hash = "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-win32.whl", hash = "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-win32.whl", hash = "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-win32.whl", hash = "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-win32.whl", hash = "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-win32.whl", hash = "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884"}, - {file = "charset_normalizer-3.3.0-py3-none-any.whl", hash = "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2"}, + {file = "charset-normalizer-3.3.1.tar.gz", hash = "sha256:d9137a876020661972ca6eec0766d81aef8a5627df628b664b234b73396e727e"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8aee051c89e13565c6bd366813c386939f8e928af93c29fda4af86d25b73d8f8"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:352a88c3df0d1fa886562384b86f9a9e27563d4704ee0e9d56ec6fcd270ea690"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:223b4d54561c01048f657fa6ce41461d5ad8ff128b9678cfe8b2ecd951e3f8a2"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f861d94c2a450b974b86093c6c027888627b8082f1299dfd5a4bae8e2292821"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1171ef1fc5ab4693c5d151ae0fdad7f7349920eabbaca6271f95969fa0756c2d"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28f512b9a33235545fbbdac6a330a510b63be278a50071a336afc1b78781b147"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0e842112fe3f1a4ffcf64b06dc4c61a88441c2f02f373367f7b4c1aa9be2ad5"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f9bc2ce123637a60ebe819f9fccc614da1bcc05798bbbaf2dd4ec91f3e08846"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f194cce575e59ffe442c10a360182a986535fd90b57f7debfaa5c845c409ecc3"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9a74041ba0bfa9bc9b9bb2cd3238a6ab3b7618e759b41bd15b5f6ad958d17605"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b578cbe580e3b41ad17b1c428f382c814b32a6ce90f2d8e39e2e635d49e498d1"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6db3cfb9b4fcecb4390db154e75b49578c87a3b9979b40cdf90d7e4b945656e1"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:debb633f3f7856f95ad957d9b9c781f8e2c6303ef21724ec94bea2ce2fcbd056"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-win32.whl", hash = "sha256:87071618d3d8ec8b186d53cb6e66955ef2a0e4fa63ccd3709c0c90ac5a43520f"}, + {file = "charset_normalizer-3.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:e372d7dfd154009142631de2d316adad3cc1c36c32a38b16a4751ba78da2a397"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae4070f741f8d809075ef697877fd350ecf0b7c5837ed68738607ee0a2c572cf"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58e875eb7016fd014c0eea46c6fa92b87b62c0cb31b9feae25cbbe62c919f54d"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dbd95e300367aa0827496fe75a1766d198d34385a58f97683fe6e07f89ca3e3c"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de0b4caa1c8a21394e8ce971997614a17648f94e1cd0640fbd6b4d14cab13a72"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:985c7965f62f6f32bf432e2681173db41336a9c2611693247069288bcb0c7f8b"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a15c1fe6d26e83fd2e5972425a772cca158eae58b05d4a25a4e474c221053e2d"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae55d592b02c4349525b6ed8f74c692509e5adffa842e582c0f861751701a673"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be4d9c2770044a59715eb57c1144dedea7c5d5ae80c68fb9959515037cde2008"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:851cf693fb3aaef71031237cd68699dded198657ec1e76a76eb8be58c03a5d1f"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:31bbaba7218904d2eabecf4feec0d07469284e952a27400f23b6628439439fa7"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:871d045d6ccc181fd863a3cd66ee8e395523ebfbc57f85f91f035f50cee8e3d4"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:501adc5eb6cd5f40a6f77fbd90e5ab915c8fd6e8c614af2db5561e16c600d6f3"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f5fb672c396d826ca16a022ac04c9dce74e00a1c344f6ad1a0fdc1ba1f332213"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-win32.whl", hash = "sha256:bb06098d019766ca16fc915ecaa455c1f1cd594204e7f840cd6258237b5079a8"}, + {file = "charset_normalizer-3.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:8af5a8917b8af42295e86b64903156b4f110a30dca5f3b5aedea123fbd638bff"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:7ae8e5142dcc7a49168f4055255dbcced01dc1714a90a21f87448dc8d90617d1"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5b70bab78accbc672f50e878a5b73ca692f45f5b5e25c8066d748c09405e6a55"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5ceca5876032362ae73b83347be8b5dbd2d1faf3358deb38c9c88776779b2e2f"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34d95638ff3613849f473afc33f65c401a89f3b9528d0d213c7037c398a51296"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9edbe6a5bf8b56a4a84533ba2b2f489d0046e755c29616ef8830f9e7d9cf5728"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6a02a3c7950cafaadcd46a226ad9e12fc9744652cc69f9e5534f98b47f3bbcf"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10b8dd31e10f32410751b3430996f9807fc4d1587ca69772e2aa940a82ab571a"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edc0202099ea1d82844316604e17d2b175044f9bcb6b398aab781eba957224bd"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b891a2f68e09c5ef989007fac11476ed33c5c9994449a4e2c3386529d703dc8b"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:71ef3b9be10070360f289aea4838c784f8b851be3ba58cf796262b57775c2f14"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:55602981b2dbf8184c098bc10287e8c245e351cd4fdcad050bd7199d5a8bf514"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:46fb9970aa5eeca547d7aa0de5d4b124a288b42eaefac677bde805013c95725c"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:520b7a142d2524f999447b3a0cf95115df81c4f33003c51a6ab637cbda9d0bf4"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-win32.whl", hash = "sha256:8ec8ef42c6cd5856a7613dcd1eaf21e5573b2185263d87d27c8edcae33b62a61"}, + {file = "charset_normalizer-3.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:baec8148d6b8bd5cee1ae138ba658c71f5b03e0d69d5907703e3e1df96db5e41"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63a6f59e2d01310f754c270e4a257426fe5a591dc487f1983b3bbe793cf6bac6"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d6bfc32a68bc0933819cfdfe45f9abc3cae3877e1d90aac7259d57e6e0f85b1"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f3100d86dcd03c03f7e9c3fdb23d92e32abbca07e7c13ebd7ddfbcb06f5991f"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39b70a6f88eebe239fa775190796d55a33cfb6d36b9ffdd37843f7c4c1b5dc67"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e12f8ee80aa35e746230a2af83e81bd6b52daa92a8afaef4fea4a2ce9b9f4fa"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b6cefa579e1237ce198619b76eaa148b71894fb0d6bcf9024460f9bf30fd228"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:61f1e3fb621f5420523abb71f5771a204b33c21d31e7d9d86881b2cffe92c47c"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4f6e2a839f83a6a76854d12dbebde50e4b1afa63e27761549d006fa53e9aa80e"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:1ec937546cad86d0dce5396748bf392bb7b62a9eeb8c66efac60e947697f0e58"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:82ca51ff0fc5b641a2d4e1cc8c5ff108699b7a56d7f3ad6f6da9dbb6f0145b48"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:633968254f8d421e70f91c6ebe71ed0ab140220469cf87a9857e21c16687c034"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-win32.whl", hash = "sha256:c0c72d34e7de5604df0fde3644cc079feee5e55464967d10b24b1de268deceb9"}, + {file = "charset_normalizer-3.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:63accd11149c0f9a99e3bc095bbdb5a464862d77a7e309ad5938fbc8721235ae"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5a3580a4fdc4ac05f9e53c57f965e3594b2f99796231380adb2baaab96e22761"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2465aa50c9299d615d757c1c888bc6fef384b7c4aec81c05a0172b4400f98557"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb7cd68814308aade9d0c93c5bd2ade9f9441666f8ba5aa9c2d4b389cb5e2a45"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91e43805ccafa0a91831f9cd5443aa34528c0c3f2cc48c4cb3d9a7721053874b"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:854cc74367180beb327ab9d00f964f6d91da06450b0855cbbb09187bcdb02de5"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c15070ebf11b8b7fd1bfff7217e9324963c82dbdf6182ff7050519e350e7ad9f"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c4c99f98fc3a1835af8179dcc9013f93594d0670e2fa80c83aa36346ee763d2"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fb765362688821404ad6cf86772fc54993ec11577cd5a92ac44b4c2ba52155b"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dced27917823df984fe0c80a5c4ad75cf58df0fbfae890bc08004cd3888922a2"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a66bcdf19c1a523e41b8e9d53d0cedbfbac2e93c649a2e9502cb26c014d0980c"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ecd26be9f112c4f96718290c10f4caea6cc798459a3a76636b817a0ed7874e42"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3f70fd716855cd3b855316b226a1ac8bdb3caf4f7ea96edcccc6f484217c9597"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:17a866d61259c7de1bdadef418a37755050ddb4b922df8b356503234fff7932c"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-win32.whl", hash = "sha256:548eefad783ed787b38cb6f9a574bd8664468cc76d1538215d510a3cd41406cb"}, + {file = "charset_normalizer-3.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:45f053a0ece92c734d874861ffe6e3cc92150e32136dd59ab1fb070575189c97"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bc791ec3fd0c4309a753f95bb6c749ef0d8ea3aea91f07ee1cf06b7b02118f2f"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8c61fb505c7dad1d251c284e712d4e0372cef3b067f7ddf82a7fa82e1e9a93"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2c092be3885a1b7899cd85ce24acedc1034199d6fca1483fa2c3a35c86e43041"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2000c54c395d9e5e44c99dc7c20a64dc371f777faf8bae4919ad3e99ce5253e"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4cb50a0335382aac15c31b61d8531bc9bb657cfd848b1d7158009472189f3d62"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c30187840d36d0ba2893bc3271a36a517a717f9fd383a98e2697ee890a37c273"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe81b35c33772e56f4b6cf62cf4aedc1762ef7162a31e6ac7fe5e40d0149eb67"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0bf89afcbcf4d1bb2652f6580e5e55a840fdf87384f6063c4a4f0c95e378656"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:06cf46bdff72f58645434d467bf5228080801298fbba19fe268a01b4534467f5"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3c66df3f41abee950d6638adc7eac4730a306b022570f71dd0bd6ba53503ab57"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd805513198304026bd379d1d516afbf6c3c13f4382134a2c526b8b854da1c2e"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:9505dc359edb6a330efcd2be825fdb73ee3e628d9010597aa1aee5aa63442e97"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:31445f38053476a0c4e6d12b047b08ced81e2c7c712e5a1ad97bc913256f91b2"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-win32.whl", hash = "sha256:bd28b31730f0e982ace8663d108e01199098432a30a4c410d06fe08fdb9e93f4"}, + {file = "charset_normalizer-3.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:555fe186da0068d3354cdf4bbcbc609b0ecae4d04c921cc13e209eece7720727"}, + {file = "charset_normalizer-3.3.1-py3-none-any.whl", hash = "sha256:800561453acdecedaac137bf09cd719c7a440b6800ec182f077bb8e7025fb708"}, ] [[package]] @@ -303,13 +334,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cloudpickle" -version = "2.2.1" -description = "Extended pickling support for Python objects" +version = "3.0.0" +description = "Pickler class to extend the standard pickle.Pickler functionality" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "cloudpickle-2.2.1-py3-none-any.whl", hash = "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f"}, - {file = "cloudpickle-2.2.1.tar.gz", hash = "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"}, + {file = "cloudpickle-3.0.0-py3-none-any.whl", hash = "sha256:246ee7d0c295602a036e86369c77fecda4ab17b506496730f2f576d9016fd9c7"}, + {file = "cloudpickle-3.0.0.tar.gz", hash = "sha256:996d9a482c6fb4f33c1a35335cf8afd065d2a56e973270364840712d9131a882"}, ] [[package]] @@ -446,13 +477,13 @@ test = ["pytest (>=6)"] [[package]] name = "fsspec" -version = "2023.9.2" +version = "2023.10.0" description = "File-system specification" optional = false python-versions = ">=3.8" files = [ - {file = "fsspec-2023.9.2-py3-none-any.whl", hash = "sha256:603dbc52c75b84da501b9b2ec8c11e1f61c25984c4a0dda1f129ef391fbfc9b4"}, - {file = "fsspec-2023.9.2.tar.gz", hash = "sha256:80bfb8c70cc27b2178cc62a935ecf242fc6e8c3fb801f9c571fc01b1e715ba7d"}, + {file = "fsspec-2023.10.0-py3-none-any.whl", hash = "sha256:346a8f024efeb749d2a5fca7ba8854474b1ff9af7c3faaf636a4548781136529"}, + {file = "fsspec-2023.10.0.tar.gz", hash = "sha256:330c66757591df346ad3091a53bd907e15348c2ba17d63fd54f5c39c4457d2a5"}, ] [package.extras] @@ -783,42 +814,42 @@ xml = ["lxml (>=4.8.0)"] [[package]] name = "pandas" -version = "2.1.1" +version = "2.1.2" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" files = [ - {file = "pandas-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58d997dbee0d4b64f3cb881a24f918b5f25dd64ddf31f467bb9b67ae4c63a1e4"}, - {file = "pandas-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02304e11582c5d090e5a52aec726f31fe3f42895d6bfc1f28738f9b64b6f0614"}, - {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffa8f0966de2c22de408d0e322db2faed6f6e74265aa0856f3824813cf124363"}, - {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1f84c144dee086fe4f04a472b5cd51e680f061adf75c1ae4fc3a9275560f8f4"}, - {file = "pandas-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ce97667d06d69396d72be074f0556698c7f662029322027c226fd7a26965cb"}, - {file = "pandas-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:4c3f32fd7c4dccd035f71734df39231ac1a6ff95e8bdab8d891167197b7018d2"}, - {file = "pandas-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e2959720b70e106bb1d8b6eadd8ecd7c8e99ccdbe03ee03260877184bb2877d"}, - {file = "pandas-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25e8474a8eb258e391e30c288eecec565bfed3e026f312b0cbd709a63906b6f8"}, - {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8bd1685556f3374520466998929bade3076aeae77c3e67ada5ed2b90b4de7f0"}, - {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc3657869c7902810f32bd072f0740487f9e030c1a3ab03e0af093db35a9d14e"}, - {file = "pandas-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:05674536bd477af36aa2effd4ec8f71b92234ce0cc174de34fd21e2ee99adbc2"}, - {file = "pandas-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:b407381258a667df49d58a1b637be33e514b07f9285feb27769cedb3ab3d0b3a"}, - {file = "pandas-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c747793c4e9dcece7bb20156179529898abf505fe32cb40c4052107a3c620b49"}, - {file = "pandas-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3bcad1e6fb34b727b016775bea407311f7721db87e5b409e6542f4546a4951ea"}, - {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5ec7740f9ccb90aec64edd71434711f58ee0ea7f5ed4ac48be11cfa9abf7317"}, - {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29deb61de5a8a93bdd033df328441a79fcf8dd3c12d5ed0b41a395eef9cd76f0"}, - {file = "pandas-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f99bebf19b7e03cf80a4e770a3e65eee9dd4e2679039f542d7c1ace7b7b1daa"}, - {file = "pandas-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:84e7e910096416adec68075dc87b986ff202920fb8704e6d9c8c9897fe7332d6"}, - {file = "pandas-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:366da7b0e540d1b908886d4feb3d951f2f1e572e655c1160f5fde28ad4abb750"}, - {file = "pandas-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9e50e72b667415a816ac27dfcfe686dc5a0b02202e06196b943d54c4f9c7693e"}, - {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1ab6a25da197f03ebe6d8fa17273126120874386b4ac11c1d687df288542dd"}, - {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0dbfea0dd3901ad4ce2306575c54348d98499c95be01b8d885a2737fe4d7a98"}, - {file = "pandas-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0489b0e6aa3d907e909aef92975edae89b1ee1654db5eafb9be633b0124abe97"}, - {file = "pandas-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:4cdb0fab0400c2cb46dafcf1a0fe084c8bb2480a1fa8d81e19d15e12e6d4ded2"}, - {file = "pandas-2.1.1.tar.gz", hash = "sha256:fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b"}, + {file = "pandas-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:24057459f19db9ebb02984c6fdd164a970b31a95f38e4a49cf7615b36a1b532c"}, + {file = "pandas-2.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a6cf8fcc8a63d333970b950a7331a30544cf59b1a97baf0a7409e09eafc1ac38"}, + {file = "pandas-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ae6ffbd9d614c20d028c7117ee911fc4e266b4dca2065d5c5909e401f8ff683"}, + {file = "pandas-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eff794eeb7883c5aefb1ed572e7ff533ae779f6c6277849eab9e77986e352688"}, + {file = "pandas-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02954e285e8e2f4006b6f22be6f0df1f1c3c97adbb7ed211c6b483426f20d5c8"}, + {file = "pandas-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:5b40c9f494e1f27588c369b9e4a6ca19cd924b3a0e1ef9ef1a8e30a07a438f43"}, + {file = "pandas-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:08d287b68fd28906a94564f15118a7ca8c242e50ae7f8bd91130c362b2108a81"}, + {file = "pandas-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bbd98dcdcd32f408947afdb3f7434fade6edd408c3077bbce7bd840d654d92c6"}, + {file = "pandas-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e90c95abb3285d06f6e4feedafc134306a8eced93cb78e08cf50e224d5ce22e2"}, + {file = "pandas-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52867d69a54e71666cd184b04e839cff7dfc8ed0cd6b936995117fdae8790b69"}, + {file = "pandas-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8d0382645ede2fde352da2a885aac28ec37d38587864c0689b4b2361d17b1d4c"}, + {file = "pandas-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:65177d1c519b55e5b7f094c660ed357bb7d86e799686bb71653b8a4803d8ff0d"}, + {file = "pandas-2.1.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5aa6b86802e8cf7716bf4b4b5a3c99b12d34e9c6a9d06dad254447a620437931"}, + {file = "pandas-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d594e2ce51b8e0b4074e6644758865dc2bb13fd654450c1eae51201260a539f1"}, + {file = "pandas-2.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3223f997b6d2ebf9c010260cf3d889848a93f5d22bb4d14cd32638b3d8bba7ad"}, + {file = "pandas-2.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4944dc004ca6cc701dfa19afb8bdb26ad36b9bed5bcec617d2a11e9cae6902"}, + {file = "pandas-2.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3f76280ce8ec216dde336e55b2b82e883401cf466da0fe3be317c03fb8ee7c7d"}, + {file = "pandas-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:7ad20d24acf3a0042512b7e8d8fdc2e827126ed519d6bd1ed8e6c14ec8a2c813"}, + {file = "pandas-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:021f09c15e1381e202d95d4a21ece8e7f2bf1388b6d7e9cae09dfe27bd2043d1"}, + {file = "pandas-2.1.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e7f12b2de0060b0b858cfec0016e7d980ae5bae455a1746bfcc70929100ee633"}, + {file = "pandas-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c166b9bb27c1715bed94495d9598a7f02950b4749dba9349c1dd2cbf10729d"}, + {file = "pandas-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25c9976c17311388fcd953cb3d0697999b2205333f4e11e669d90ff8d830d429"}, + {file = "pandas-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:851b5afbb0d62f6129ae891b533aa508cc357d5892c240c91933d945fff15731"}, + {file = "pandas-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:e78507adcc730533619de07bfdd1c62b2918a68cd4419ea386e28abf7f6a1e5c"}, + {file = "pandas-2.1.2.tar.gz", hash = "sha256:52897edc2774d2779fbeb6880d2cfb305daa0b1a29c16b91f531a18918a6e0f3"}, ] [package.dependencies] numpy = [ - {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -1031,7 +1062,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1039,15 +1069,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1064,7 +1087,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1072,7 +1094,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, From 883bfdfdb6603a8998b5135c1be12cb8b8b0bb2d Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Tue, 31 Oct 2023 16:50:41 -0400 Subject: [PATCH 03/14] remove dockerfilelint --- .pre-commit-config.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2aa0f30..ba7dee6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,9 +38,3 @@ repos: - id: yamllint args: ["-d {extends: relaxed, rules: {line-length: {max: 120}}}"] stages: [commit, push] - - - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks - rev: v0.1.0 - hooks: - - id: dockerfilelint - stages: [commit, push] From 7b3ed1d7db65c0af1929c40f941f73caa9447099 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Tue, 31 Oct 2023 16:51:06 -0400 Subject: [PATCH 04/14] initial commit of Docker files --- .dockerignore | 103 +++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 53 ++++++++++++++++++++++ build-service | 3 ++ docker-entrypoint.sh | 10 +++++ 4 files changed, 169 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 build-service create mode 100644 docker-entrypoint.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a38a8b4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,103 @@ +.travis.yaml +.swagger-codegen-ignore +README.md +tox.ini +git_push.sh +test-requirements.txt +setup.py + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +venv/ +.python-version + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +#Ipython Notebook +.ipynb_checkpoints + +# Intellij project settings +.idea + +# VSCode project settings +.vscode + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# ruff +.ruff_cache/ +# pytest +.pytest_cache/ + +# Pyre type checker +.pyre/ + +# Github integration settings +.github + +# .dockerignore, Dockerfile and builder script themselves are not needed by +# docker build. +.dockerignore +Dockerfile* +build-service diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ee4a61c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ + +FROM python:3.10-slim + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + gcc \ + libnetcdf-dev \ + #libhdf5-dev \ + #hdf5-helpers \ + && pip3 install --upgrade pip \ + && pip3 install cython \ + && pip3 install poetry \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + + +# Create a new user +RUN adduser --quiet --disabled-password --shell /bin/sh --home /home/dockeruser --gecos "" --uid 1000 dockeruser +USER dockeruser +ENV HOME /home/dockeruser +ENV PYTHONPATH "${PYTHONPATH}:/home/dockeruser/.local/bin" +ENV PATH="/home/dockeruser/.local/bin:${PATH}" + +# The 'SOURCE' argument is what will be used in 'pip install'. +ARG SOURCE + +# Set this argument if running the pip install on a local directory, so +# the local dist files are copied into the container. +ARG DIST_PATH + +USER root +RUN mkdir -p /worker && chown dockeruser /worker +COPY pyproject.toml /worker +# COPY ../pyproject.toml /worker +USER dockeruser + +WORKDIR /worker + +# ENV PYTHONPATH=${PYTHONPATH}:${PWD} + +COPY --chown=dockeruser $DIST_PATH $DIST_PATH +USER dockeruser +#RUN pip3 install --no-cache-dir --force --user --index-url https://pypi.org/simple/ --extra-index-url https://test.pypi.org/simple/ $SOURCE \ +# && rm -rf $DIST_PATH + +#install poetry as root +USER root +RUN poetry config virtualenvs.create false +RUN poetry install --no-dev + +USER dockeruser +COPY --chown=dockeruser ./docker-entrypoint.sh docker-entrypoint.sh +# Run the service +ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/build-service b/build-service new file mode 100755 index 0000000..1026e26 --- /dev/null +++ b/build-service @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build -t "asdc-trade/stitchee:${VERSION-latest}" . diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 0000000..000ebbe --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +if [ "$1" = 'stitchee' ]; then + exec stitchee "$@" +elif [ "$1" = 'stitchee_harmony' ]; then + exec stitchee_harmony "$@" +else + exec stitchee_harmony "$@" +fi From d062a4a53aa1e34092aa1966c6843efb2828b26a Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Tue, 31 Oct 2023 17:16:15 -0400 Subject: [PATCH 05/14] update .dockerignore --- .dockerignore | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index a38a8b4..3207eef 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,10 @@ +TEMPO_*.nc +tests/*data*/ +.git +.gitignore .travis.yaml .swagger-codegen-ignore -README.md tox.ini -git_push.sh -test-requirements.txt -setup.py # Byte-compiled / optimized / DLL files __pycache__/ @@ -96,8 +96,7 @@ dmypy.json # Github integration settings .github -# .dockerignore, Dockerfile and builder script themselves are not needed by +# .dockerignore and builder script themselves are not needed by # docker build. .dockerignore -Dockerfile* build-service From f8585822476f8e6b1bb8f26179bb2c818230ad2e Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Tue, 31 Oct 2023 17:16:26 -0400 Subject: [PATCH 06/14] update Dockerfile --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee4a61c..b2a3501 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,21 +31,16 @@ USER root RUN mkdir -p /worker && chown dockeruser /worker COPY pyproject.toml /worker # COPY ../pyproject.toml /worker -USER dockeruser WORKDIR /worker - # ENV PYTHONPATH=${PYTHONPATH}:${PWD} - COPY --chown=dockeruser $DIST_PATH $DIST_PATH -USER dockeruser #RUN pip3 install --no-cache-dir --force --user --index-url https://pypi.org/simple/ --extra-index-url https://test.pypi.org/simple/ $SOURCE \ # && rm -rf $DIST_PATH #install poetry as root -USER root RUN poetry config virtualenvs.create false -RUN poetry install --no-dev +RUN poetry install --only main USER dockeruser COPY --chown=dockeruser ./docker-entrypoint.sh docker-entrypoint.sh From ae72c7db47f1ce0f3b69bbb91703519d677dfcfe Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Tue, 31 Oct 2023 17:16:44 -0400 Subject: [PATCH 07/14] add stitchee_harmony poetry script entry --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 46fc7d5..82535b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ coverage = "^7.3.2" nco = "^1.1.0" [tool.poetry.scripts] +stitchee_harmony = 'concatenator.harmony.cli:main' stitchee = 'concatenator.run_stitchee:main' [build-system] From 7e6a7646eee8678693af1780fee06f403009bb09 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Tue, 31 Oct 2023 17:21:14 -0400 Subject: [PATCH 08/14] update permissions on docker-entrypoint.sh --- docker-entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 docker-entrypoint.sh diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh old mode 100644 new mode 100755 From b9d8a8a259b08a7cedac54bdca2b1b627dedd408 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 1 Nov 2023 11:05:08 -0400 Subject: [PATCH 09/14] remove comment --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b2a3501..eedcd8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,6 @@ ARG DIST_PATH USER root RUN mkdir -p /worker && chown dockeruser /worker COPY pyproject.toml /worker -# COPY ../pyproject.toml /worker WORKDIR /worker # ENV PYTHONPATH=${PYTHONPATH}:${PWD} From aa3baaed4413640a6337e53df1cb412505cf1a2b Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Wed, 1 Nov 2023 12:01:15 -0400 Subject: [PATCH 10/14] exclude all tests and pytest config from docker build --- .dockerignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 3207eef..3c94215 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ TEMPO_*.nc -tests/*data*/ +tests/ +pytest.ini .git .gitignore .travis.yaml From 439cd5bf54d616cd5de30fe199921e132a557640 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 3 Nov 2023 12:45:38 -0400 Subject: [PATCH 11/14] add missing file staging method, and improve logging --- concatenator/harmony/service_adapter.py | 57 +++++++++++++++++++++++-- concatenator/harmony/util.py | 12 ++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/concatenator/harmony/service_adapter.py b/concatenator/harmony/service_adapter.py index 5905229..f3eb3ec 100644 --- a/concatenator/harmony/service_adapter.py +++ b/concatenator/harmony/service_adapter.py @@ -1,10 +1,12 @@ from pathlib import Path +from shutil import copyfile from tempfile import TemporaryDirectory +from urllib.parse import urlsplit from uuid import uuid4 import pystac from harmony.adapter import BaseHarmonyAdapter -from harmony.util import bbox_to_geometry +from harmony.util import bbox_to_geometry, stage from pystac import Item from pystac.item import Asset @@ -13,6 +15,7 @@ _get_netcdf_urls, _get_output_bounding_box, _get_output_date_range, + sizeof_fmt, ) from concatenator.stitchee import stitchee @@ -51,7 +54,7 @@ def invoke(self): return self.message, self.process_catalog(self.catalog) - def process_catalog(self, catalog: pystac.Catalog): + def process_catalog(self, catalog: pystac.Catalog) -> pystac.Catalog: """Converts a list of STAC catalogs into a list of lists of STAC catalogs.""" self.logger.info("process_catalog() started.") try: @@ -83,14 +86,20 @@ def process_catalog(self, catalog: pystac.Catalog): # -- Perform merging -- collection = self._get_item_source(items[0]).collection filename = f"{collection}_merged.nc4" + self.logger.info(f"Merged filename will be === {filename}.") with TemporaryDirectory() as temp_dir: self.logger.info("Starting granule downloads") input_files = multi_core_download( netcdf_urls, temp_dir, self.message.accessToken, self.config ) - self.logger.info("Finished granule downloads") + self.logger.info("Finished granule downloads.") + for file_count, file in enumerate(input_files): + file_size = sizeof_fmt(file.stat().st_size) + self.logger.info(f"File {file_count} is size <{file_size}>. Path={file}") + + self.logger.info("Running Stitchee..") output_path = str(Path(temp_dir).joinpath(filename).resolve()) # # --- Run STITCHEE --- @@ -102,7 +111,9 @@ def process_catalog(self, catalog: pystac.Catalog): concat_dim="mirror_step", # This is currently set only for TEMPO logger=self.logger, ) + self.logger.info("Stitchee completed.") staged_url = self._stage(output_path, filename, "application/x-netcdf4") + self.logger.info("Staging completed.") # -- Output to STAC catalog -- result.clear_items() @@ -126,3 +137,43 @@ def process_catalog(self, catalog: pystac.Catalog): except Exception as service_exception: self.logger.error(service_exception, exc_info=1) raise service_exception + + def _stage(self, local_filename: str, remote_filename: str, mime: str) -> str: + """ + Stages a local file to either to S3 (utilizing harmony.util.stage) or to + the local filesystem by performing a file copy. Staging location is + determined by message.stagingLocation or the --harmony-data-location + CLI argument override + + Parameters + ---------- + local_filename : string + A path and filename to the local file that should be staged + remote_filename : string + The basename to give to the remote file + mime : string + The mime type to apply to the staged file for use when it is served, e.g. "application/x-netcdf4" + + Returns + ------- + url : string + A URL to the staged file + """ + url_components = urlsplit(self.message.stagingLocation) + scheme = url_components.scheme + + if scheme == "file": + dest_path = Path(url_components.path).joinpath(remote_filename) + self.logger.info("Staging to local filesystem: '%s'", str(dest_path)) + + copyfile(local_filename, dest_path) + return dest_path.as_uri() + + return stage( + local_filename, + remote_filename, + mime, + logger=self.logger, + location=self.message.stagingLocation, + cfg=self.config, + ) diff --git a/concatenator/harmony/util.py b/concatenator/harmony/util.py index f77aeb9..a972aaf 100644 --- a/concatenator/harmony/util.py +++ b/concatenator/harmony/util.py @@ -96,3 +96,15 @@ def _get_item_date_range(item: Item) -> tuple[datetime, datetime]: end_datetime = item.datetime return start_datetime, end_datetime + + +def sizeof_fmt(num: float, suffix="B") -> str: + """Get a human-readable version of file size + + From: https://stackoverflow.com/a/1094933 + """ + for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"): + if abs(num) < 1024.0: + return f"{num:3.1f}{unit}{suffix}" + num /= 1024.0 + return f"{num:.1f}Yi{suffix}" From c6a9b546ffa5c3883517caca597bb57f587baab5 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 3 Nov 2023 12:47:18 -0400 Subject: [PATCH 12/14] put concatenation in a try except block --- concatenator/stitchee.py | 153 +++++++++++++++++++++------------------ 1 file changed, 81 insertions(+), 72 deletions(-) diff --git a/concatenator/stitchee.py b/concatenator/stitchee.py index 103e3b6..3e2f061 100644 --- a/concatenator/stitchee.py +++ b/concatenator/stitchee.py @@ -52,84 +52,93 @@ def stitchee( logger.info("No non-empty netCDF files found. Exiting.") return "" - logger.debug("Flattening all input files...") + logger.info("Flattening all input files...") xrdataset_list = [] - for i, filepath in enumerate(input_files): - # The group structure is flattened. + + try: + for i, filepath in enumerate(input_files): + # The group structure is flattened. + start_time = time.time() + logger.info(" ..file %03d/%03d <%s>..", i + 1, num_input_files, filepath) + flat_dataset, coord_vars, _ = flatten_grouped_dataset( + nc.Dataset(filepath, "r"), filepath, ensure_all_dims_are_coords=True + ) + + logger.info("Removing duplicate dimensions") + flat_dataset = remove_duplicate_dims(flat_dataset) + + logger.info("Opening flattened file with xarray.") + xrds = xr.open_dataset( + xr.backends.NetCDF4DataStore(flat_dataset), + decode_times=False, + decode_coords=False, + drop_variables=coord_vars, + ) + + benchmark_log["flattening"] = time.time() - start_time + + # The flattened file is written to disk. + # flat_file_path = add_label_to_path(filepath, label="_flat_intermediate") + # xrds.to_netcdf(flat_file_path, encoding={v_name: {'dtype': 'str'} for v_name in string_vars}) + # intermediate_flat_filepaths.append(flat_file_path) + # xrdataset_list.append(xr.open_dataset(flat_file_path)) + xrdataset_list.append(xrds) + + # Flattened files are concatenated together (Using XARRAY). start_time = time.time() - logger.debug(" ..file %03d/%03d <%s>..", i + 1, num_input_files, filepath) - flat_dataset, coord_vars, _ = flatten_grouped_dataset( - nc.Dataset(filepath, "r"), filepath, ensure_all_dims_are_coords=True + logger.info("Concatenating flattened files...") + # combined_ds = xr.open_mfdataset(intermediate_flat_filepaths, + # decode_times=False, + # decode_coords=False, + # data_vars='minimal', + # coords='minimal', + # compat='override') + + if concat_kwargs is None: + concat_kwargs = {} + + combined_ds = xr.concat( + xrdataset_list, + dim=GROUP_DELIM + concat_dim, + data_vars="minimal", + coords="minimal", + **concat_kwargs, ) - flat_dataset = remove_duplicate_dims(flat_dataset) + benchmark_log["concatenating"] = time.time() - start_time - xrds = xr.open_dataset( - xr.backends.NetCDF4DataStore(flat_dataset), - decode_times=False, - decode_coords=False, - drop_variables=coord_vars, - ) + if write_tmp_flat_concatenated: + logger.info("Writing concatenated flattened temporary file to disk...") + # Concatenated, yet still flat, file is written to disk for debugging. + tmp_flat_concatenated_path = add_label_to_path(output_file, label="_flat_intermediate") + combined_ds.to_netcdf(tmp_flat_concatenated_path, format="NETCDF4") + else: + tmp_flat_concatenated_path = None - benchmark_log["flattening"] = time.time() - start_time - - # The flattened file is written to disk. - # flat_file_path = add_label_to_path(filepath, label="_flat_intermediate") - # xrds.to_netcdf(flat_file_path, encoding={v_name: {'dtype': 'str'} for v_name in string_vars}) - # intermediate_flat_filepaths.append(flat_file_path) - # xrdataset_list.append(xr.open_dataset(flat_file_path)) - xrdataset_list.append(xrds) - - # Flattened files are concatenated together (Using XARRAY). - start_time = time.time() - logger.debug("Concatenating flattened files...") - # combined_ds = xr.open_mfdataset(intermediate_flat_filepaths, - # decode_times=False, - # decode_coords=False, - # data_vars='minimal', - # coords='minimal', - # compat='override') - - if concat_kwargs is None: - concat_kwargs = {} - - combined_ds = xr.concat( - xrdataset_list, - dim=GROUP_DELIM + concat_dim, - data_vars="minimal", - coords="minimal", - **concat_kwargs, - ) - - benchmark_log["concatenating"] = time.time() - start_time - - if write_tmp_flat_concatenated: - logger.debug("Writing concatenated flattened temporary file to disk...") - # Concatenated, yet still flat, file is written to disk for debugging. - tmp_flat_concatenated_path = add_label_to_path(output_file, label="_flat_intermediate") - combined_ds.to_netcdf(tmp_flat_concatenated_path, format="NETCDF4") - else: - tmp_flat_concatenated_path = None - - # The group hierarchy of the concatenated file is reconstructed (using XARRAY). - start_time = time.time() - logger.debug("Reconstructing groups within concatenated file...") - regroup_flattened_dataset(combined_ds, output_file) - benchmark_log["reconstructing_groups"] = time.time() - start_time - - logger.info("--- Benchmark results ---") - total_time = 0.0 - for k, v in benchmark_log.items(): - logger.info("%s: %f", k, v) - total_time += v - logger.info("-- total processing time: %f", total_time) - - # If requested, remove temporary intermediate files. - if not keep_tmp_files: - for file in intermediate_flat_filepaths: - os.remove(file) - if tmp_flat_concatenated_path: - os.remove(tmp_flat_concatenated_path) + # The group hierarchy of the concatenated file is reconstructed (using XARRAY). + start_time = time.time() + logger.info("Reconstructing groups within concatenated file...") + regroup_flattened_dataset(combined_ds, output_file) + benchmark_log["reconstructing_groups"] = time.time() - start_time + + logger.info("--- Benchmark results ---") + total_time = 0.0 + for k, v in benchmark_log.items(): + logger.info("%s: %f", k, v) + total_time += v + logger.info("-- total processing time: %f", total_time) + + # If requested, remove temporary intermediate files. + if not keep_tmp_files: + for file in intermediate_flat_filepaths: + os.remove(file) + if tmp_flat_concatenated_path: + os.remove(tmp_flat_concatenated_path) + + except Exception as err: + logger.info("Stitchee encountered an error!") + logger.error(err) + raise err return output_file From 4ce3bd7a94cec406addb0e867bfb98f77feee90a Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 3 Nov 2023 12:53:36 -0400 Subject: [PATCH 13/14] add type annotations --- concatenator/harmony/download_worker.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/concatenator/harmony/download_worker.py b/concatenator/harmony/download_worker.py index 0c177af..3f502e3 100644 --- a/concatenator/harmony/download_worker.py +++ b/concatenator/harmony/download_worker.py @@ -12,7 +12,9 @@ from harmony.util import download -def multi_core_download(urls, destination_dir, access_token, cfg, process_count=None): +def multi_core_download( + urls: list, destination_dir: str, access_token: str, cfg: dict, process_count: int | None = None +) -> list[Path]: """ A method which automagically scales downloads to the number of CPU cores. For further explaination, see documentation on "multi-track @@ -28,7 +30,7 @@ def multi_core_download(urls, destination_dir, access_token, cfg, process_count= access token as provided in Harmony input cfg : dict Harmony configuration information - process_count : int + process_count : int, optional Number of worker processes to run (expected >= 1) Returns @@ -39,6 +41,8 @@ def multi_core_download(urls, destination_dir, access_token, cfg, process_count= if process_count is None: process_count = cpu_count() + if process_count is None: + raise RuntimeError("cannot determine number of cpus") with Manager() as manager: url_queue = manager.Queue(len(urls)) @@ -70,7 +74,9 @@ def multi_core_download(urls, destination_dir, access_token, cfg, process_count= return [Path(path) for path in path_list] -def _download_worker(url_queue, path_list, destination_dir, access_token, cfg): +def _download_worker( + url_queue: queue.Queue, path_list: list, destination_dir: str, access_token: str, cfg: dict +) -> None: """ A method to be executed in a separate process which processes the url_queue and places paths to completed downloads into the path_list. Downloads are From a833be28200dbe1564cecac12f6a0a4f8d5717c5 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Tue, 7 Nov 2023 11:25:05 -0500 Subject: [PATCH 14/14] update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a69d564..59e7003 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - [PR #1](https://github.com/danielfromearth/stitchee/pull/1): An initial GitHub Actions workflow + - [Issue #8](https://github.com/danielfromearth/stitchee/issues/8): Create working Docker image + - [Issue #10](https://github.com/danielfromearth/stitchee/issues/10): Add code necessary to communicate with Harmony ### Changed - [PR #12](https://github.com/danielfromearth/stitchee/pull/12): Changed name to "stitchee" - [PR #15](https://github.com/danielfromearth/stitchee/pull/15): Use ruff+black chain for pre-commit lint & format