From 86b70e2742ad420b0057dc9caeac7c0377b5f8b9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 4 Sep 2020 09:29:05 +0200 Subject: [PATCH 1/7] Allow conversion tools to be called with dir This allows the calling code to determine where a temporary directory should be created for the files. --- conda_package/mpas_tools/mesh/conversion.py | 43 +++++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/conda_package/mpas_tools/mesh/conversion.py b/conda_package/mpas_tools/mesh/conversion.py index 3c188fe56..0ecb073d3 100644 --- a/conda_package/mpas_tools/mesh/conversion.py +++ b/conda_package/mpas_tools/mesh/conversion.py @@ -1,17 +1,14 @@ -from __future__ import absolute_import, division, print_function, \ - unicode_literals - import os import xarray import subprocess -from backports.tempfile import TemporaryDirectory +from tempfile import TemporaryDirectory import shutil from mpas_tools.io import write_netcdf -def convert(dsIn, graphInfoFileName=None, logger=None): - ''' +def convert(dsIn, graphInfoFileName=None, logger=None, dir=None): + """ Use ``MpasMeshConverter.x`` to convert an input mesh to a valid MPAS mesh that is fully compliant with the MPAS mesh specification. https://mpas-dev.github.io/files/documents/MPAS-MeshSpec.pdf @@ -29,13 +26,16 @@ def convert(dsIn, graphInfoFileName=None, logger=None): logger : ``logging.Logger``, optional A logger for the output if not stdout + dir : str, optional + A directory in which a temporary directory will be added with files + produced during conversion and then deleted upon completion. + Returns ------- dsOut : ``xarray.Dataset`` The MPAS mesh - ''' - - with TemporaryDirectory() as tempdir: + """ + with TemporaryDirectory(dir=dir) as tempdir: inFileName = '{}/mesh_in.nc'.format(tempdir) write_netcdf(dsIn, inFileName) @@ -64,8 +64,8 @@ def convert(dsIn, graphInfoFileName=None, logger=None): def cull(dsIn, dsMask=None, dsInverse=None, dsPreserve=None, - graphInfoFileName=None, logger=None): - ''' + graphInfoFileName=None, logger=None, dir=None): + """ Use ``MpasCellCuller.x`` to cull cells from a mesh based on the ``cullCell`` field in the input file or DataSet and/or the provided masks. ``cullCell``, dsMask and dsInverse are merged together so that the final @@ -98,14 +98,18 @@ def cull(dsIn, dsMask=None, dsInverse=None, dsPreserve=None, logger : ``logging.Logger``, optional A logger for the output if not stdout + dir : str, optional + A directory in which a temporary directory will be added with files + produced during cell culling and then deleted upon completion. + Returns ------- dsOut : ``xarray.Dataset`` The culled mesh - ''' + """ - with TemporaryDirectory() as tempdir: + with TemporaryDirectory(dir=dir) as tempdir: inFileName = '{}/ds_in.nc'.format(tempdir) write_netcdf(dsIn, inFileName) outFileName = '{}/ds_out.nc'.format(tempdir) @@ -158,8 +162,9 @@ def cull(dsIn, dsMask=None, dsInverse=None, dsPreserve=None, return dsOut -def mask(dsMesh, fcMask=None, fcSeed=None, positiveLon=False, logger=None): - ''' +def mask(dsMesh, fcMask=None, fcSeed=None, positiveLon=False, logger=None, + dir=None): + """ Use ``MpasMaskCreator.x`` to create a set of region masks either from mask feature collecitons or from seed points to be used to flood fill @@ -178,14 +183,18 @@ def mask(dsMesh, fcMask=None, fcSeed=None, positiveLon=False, logger=None): logger : ``logging.Logger``, optional A logger for the output if not stdout + dir : str, optional + A directory in which a temporary directory will be added with files + produced during mask creation and then deleted upon completion. + Returns ------- dsMask : ``xarray.Dataset`` The masks - ''' + """ - with TemporaryDirectory() as tempdir: + with TemporaryDirectory(dir=dir) as tempdir: inFileName = '{}/mesh_in.nc'.format(tempdir) write_netcdf(dsMesh, inFileName) outFileName = '{}/mesh_out.nc'.format(tempdir) From c761dfd6df8f7144283bfead0a735ce0fa884e08 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 4 Sep 2020 09:29:55 +0200 Subject: [PATCH 2/7] Update the recipe to remove backports.tempdir Update the version of mpas_tools to 0.0.13 --- conda_package/docs/versions.rst | 3 +++ conda_package/mpas_tools/__init__.py | 2 +- conda_package/recipe/meta.yaml | 3 +-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conda_package/docs/versions.rst b/conda_package/docs/versions.rst index dfa5ecaff..052fa9ab6 100644 --- a/conda_package/docs/versions.rst +++ b/conda_package/docs/versions.rst @@ -9,6 +9,7 @@ Documentation On GitHub `stable`_ `master`_ `v0.0.11`_ `0.0.11`_ `v0.0.12`_ `0.0.12`_ +`v0.0.13`_ `0.0.13`_ ================ =============== .. _`stable`: ../stable/index.html @@ -17,3 +18,5 @@ Documentation On GitHub .. _`0.0.11`: https://github.com/MPAS-Dev/MPAS-Analysis/tree/0.0.11 .. _`v0.0.12`: ../0.0.12/index.html .. _`0.0.12`: https://github.com/MPAS-Dev/MPAS-Analysis/tree/0.0.12 +.. _`v0.0.13`: ../0.0.13/index.html +.. _`0.0.13`: https://github.com/MPAS-Dev/MPAS-Analysis/tree/0.0.13 diff --git a/conda_package/mpas_tools/__init__.py b/conda_package/mpas_tools/__init__.py index 225b5f6d6..e78543431 100644 --- a/conda_package/mpas_tools/__init__.py +++ b/conda_package/mpas_tools/__init__.py @@ -1,2 +1,2 @@ -__version_info__ = (0, 0, 12) +__version_info__ = (0, 0, 13) __version__ = '.'.join(str(vi) for vi in __version_info__) diff --git a/conda_package/recipe/meta.yaml b/conda_package/recipe/meta.yaml index ece711440..fbc79329b 100644 --- a/conda_package/recipe/meta.yaml +++ b/conda_package/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "mpas_tools" %} -{% set version = "0.0.12" %} +{% set version = "0.0.13" %} package: name: {{ name|lower }} @@ -45,7 +45,6 @@ requirements: - geometric_features - pyevtk - future - - backports.tempfile - jigsaw >=0.9.12 - jigsawpy >=0.2.1 - pyflann From e93db2b20cbeee4a059df7786e6f8a080e24a6e6 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 4 Sep 2020 11:48:41 +0200 Subject: [PATCH 3/7] Update MOC tools to provide a temp directory --- conda_package/mpas_tools/ocean/moc.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/conda_package/mpas_tools/ocean/moc.py b/conda_package/mpas_tools/ocean/moc.py index 4e0fd8db5..6a8a48c34 100755 --- a/conda_package/mpas_tools/ocean/moc.py +++ b/conda_package/mpas_tools/ocean/moc.py @@ -18,7 +18,8 @@ def make_moc_basins_and_transects(gf, mesh_filename, mask_and_transect_filename, geojson_filename=None, mask_filename=None, - logger=None): + logger=None, + dir=None): """ Builds features defining the ocean basins and southern transects used in computing the meridional overturning circulation (MOC) @@ -42,6 +43,10 @@ def make_moc_basins_and_transects(gf, mesh_filename, logger : ``logging.Logger``, optional A logger for the output if not stdout + dir : str, optional + A directory in which a temporary directory will be added with files + produced during conversion and then deleted upon completion. + Returns ------- fc : ``FeatureCollection`` @@ -58,7 +63,7 @@ def make_moc_basins_and_transects(gf, mesh_filename, dsMesh = xarray.open_dataset(mesh_filename) dsMasks = mpas_tools.mesh.conversion.mask(dsMesh=dsMesh, fcMask=fcMOC, - logger=logger) + logger=logger, dir=dir) if mask_filename is not None: write_netcdf(dsMasks, mask_filename) From fa651889ee65d352cdf661d644f017448fdd7aec Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 4 Sep 2020 11:49:10 +0200 Subject: [PATCH 4/7] Update paraview extractor to cull and mask in temp dir --- conda_package/mpas_tools/viz/paraview_extractor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conda_package/mpas_tools/viz/paraview_extractor.py b/conda_package/mpas_tools/viz/paraview_extractor.py index 62a207675..f203d9b9d 100644 --- a/conda_package/mpas_tools/viz/paraview_extractor.py +++ b/conda_package/mpas_tools/viz/paraview_extractor.py @@ -2142,11 +2142,13 @@ def _cull_files(fc_region_mask, temp_dir, mesh_filename, time_file_names, with xarray.open_dataset(mesh_filename) as ds_mesh: ds_mesh = ds_mesh[mesh_vars] print('Making a region mask file') - ds_mask = mask(dsMesh=ds_mesh, fcMask=fc_region_mask, logger=logger) + ds_mask = mask(dsMesh=ds_mesh, fcMask=fc_region_mask, logger=logger, + dir=temp_dir) write_netcdf(ds_mask, '{}/mask.nc'.format(temp_dir)) print('Cropping mesh to region') out_mesh_filename = '{}/mesh.nc'.format(temp_dir) - ds_culled = cull(dsIn=ds_mesh, dsInverse=ds_mask, logger=logger) + ds_culled = cull(dsIn=ds_mesh, dsInverse=ds_mask, logger=logger, + dir=temp_dir) write_netcdf(ds_culled, out_mesh_filename) region_masks = dict() From 69da3bc8322408cc3c858814ba2125997282f6f1 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 4 Sep 2020 11:50:01 +0200 Subject: [PATCH 5/7] Update build_mesh to use CWD as the default temp dir --- conda_package/mpas_tools/mesh/creation/build_mesh.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/conda_package/mpas_tools/mesh/creation/build_mesh.py b/conda_package/mpas_tools/mesh/creation/build_mesh.py index 381756464..d264fa470 100644 --- a/conda_package/mpas_tools/mesh/creation/build_mesh.py +++ b/conda_package/mpas_tools/mesh/creation/build_mesh.py @@ -17,7 +17,8 @@ def build_spherical_mesh(cellWidth, lon, lat, earth_radius, - out_filename='base_mesh.nc', plot_cellWidth=True): + out_filename='base_mesh.nc', plot_cellWidth=True, + dir='./'): """ Build an MPAS mesh using JIGSAW with the given cell sizes as a function of latitude and longitude. @@ -46,6 +47,10 @@ def build_spherical_mesh(cellWidth, lon, lat, earth_radius, plot_cellWidth : bool, optional Whether to produce a plot of ``cellWidth``. If so, it will be written to ``cellWidthGlobal.png``. + + dir : str, optional + A directory in which a temporary directory will be added with files + produced during mesh conversion and then deleted upon completion. """ da = xarray.DataArray(cellWidth, @@ -92,7 +97,7 @@ def build_spherical_mesh(cellWidth, lon, lat, earth_radius, sphere_radius=earth_radius) print('Step 3. Convert from triangles to MPAS mesh') - write_netcdf(convert(xarray.open_dataset('mesh_triangles.nc')), + write_netcdf(convert(xarray.open_dataset('mesh_triangles.nc'), dir=dir), out_filename) From 63ef928fedc96e68a80ac8d8867901fdef3bd631 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 5 Sep 2020 16:05:27 +0200 Subject: [PATCH 6/7] Make temp directory an absolute path if it is supplied --- conda_package/mpas_tools/mesh/conversion.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conda_package/mpas_tools/mesh/conversion.py b/conda_package/mpas_tools/mesh/conversion.py index 0ecb073d3..b55179761 100644 --- a/conda_package/mpas_tools/mesh/conversion.py +++ b/conda_package/mpas_tools/mesh/conversion.py @@ -35,6 +35,8 @@ def convert(dsIn, graphInfoFileName=None, logger=None, dir=None): dsOut : ``xarray.Dataset`` The MPAS mesh """ + if dir is not None: + dir = os.path.abspath(dir) with TemporaryDirectory(dir=dir) as tempdir: inFileName = '{}/mesh_in.nc'.format(tempdir) write_netcdf(dsIn, inFileName) @@ -108,7 +110,8 @@ def cull(dsIn, dsMask=None, dsInverse=None, dsPreserve=None, The culled mesh """ - + if dir is not None: + dir = os.path.abspath(dir) with TemporaryDirectory(dir=dir) as tempdir: inFileName = '{}/ds_in.nc'.format(tempdir) write_netcdf(dsIn, inFileName) @@ -193,7 +196,8 @@ def mask(dsMesh, fcMask=None, fcSeed=None, positiveLon=False, logger=None, The masks """ - + if dir is not None: + dir = os.path.abspath(dir) with TemporaryDirectory(dir=dir) as tempdir: inFileName = '{}/mesh_in.nc'.format(tempdir) write_netcdf(dsMesh, inFileName) From d81d308335dec94134c4ec0e053edf1fe7f5bfd4 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 5 Sep 2020 17:14:40 +0200 Subject: [PATCH 7/7] Bump compass version number Pin new mpas_tools --- compass/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compass/meta.yaml b/compass/meta.yaml index 6586de39e..38f5b3516 100644 --- a/compass/meta.yaml +++ b/compass/meta.yaml @@ -1,5 +1,5 @@ {% set name = "compass" %} -{% set version = "0.1.10" %} +{% set version = "0.1.11" %} {% set build = 0 %} {% if mpi == "nompi" %} @@ -31,7 +31,7 @@ requirements: run: - python - geometric_features 0.1.11 - - mpas_tools 0.0.12 + - mpas_tools 0.0.13 - jigsaw 0.9.12 - jigsawpy 0.2.1 - metis