Process typing

examples/clip/clip.mapchete

@@ -0,0 +1,16 @@
+process: mapchete.processes.clip
+zoom_levels:
+    min: 0
+    max: 8
+pyramid:
+    grid: geodetic
+    metatiling: 1
+input:
+    inp: ../../test/testdata/cleantopo_br.tif
+    clip: ../../test/testdata/aoi_br.geojson
+
+output:
+    path: output
+    format: GTiff
+    dtype: uint8
+    bands: 1

examples/contours/contours.mapchete

@@ -0,0 +1,19 @@
+process: mapchete.processes.contours
+zoom_levels:
+    min: 0
+    max: 8
+pyramid:
+    grid: geodetic
+    metatiling: 1
+input:
+    dem: ../../test/testdata/cleantopo_tl.tif
+output:
+    format: GeoJSON
+    path: output
+    schema:
+        properties:
+            elevation: float
+        geometry: LineString
+process_parameters:
+    field: "elevation"
+    interval: 100

examples/custom_grid/custom_grid.mapchete

@@ -0,0 +1,29 @@
+process: mapchete.processes.convert
+input:
+  inp: ../../test/testdata/custom_grid_points.geojson
+output:
+  format: GeoJSON
+  path: output
+  schema:
+    properties:
+      name: str
+    geometry: Unknown
+pyramid:
+  grid:
+    shape:
+    - 1525
+    - 125
+    bounds:
+    - 166020
+    - 0
+    - 934020
+    - 9369600
+    is_global: false
+    proj: +proj=utm +zone=32 +datum=WGS84 +units=m +no_defs
+  metatiling: 4
+zoom_levels: 3
+bounds:
+- 166020
+- 0
+- 934020
+- 9369600

examples/file_groups/file_groups.mapchete

@@ -0,0 +1,20 @@
+process: file_groups.py
+zoom_levels:
+    min: 0
+    max: 5
+pyramid:
+    grid: geodetic
+    pixelbuffer: 10
+    metatiling: 8
+input:
+    group1:
+        file1: ../../test/testdata/cleantopo_tl.tif
+        file2: ../../test/testdata/cleantopo_br.tif
+    group2:
+        file1: ../../test/testdata/dummy1.tif
+        file2: ../../test/testdata/dummy2.tif
+output:
+    dtype: uint16
+    bands: 1
+    format: GTiff
+    path: output

examples/file_groups/file_groups.py

@@ -0,0 +1,21 @@
+"""Example process file."""
+
+import numpy.ma as ma
+
+from mapchete import RasterInputGroup
+
+
+def execute(group1: RasterInputGroup, group2: RasterInputGroup) -> ma.MaskedArray:
+    """User defined process."""
+
+    # read band 1 and get mean of group1
+    group1 = ma.mean(
+        ma.stack([raster_input.read(1) for _, raster_input in group1]), axis=0
+    )
+
+    # read band 1 and get mean of group1
+    group2 = ma.mean(
+        ma.stack([raster_input.read(1) for _, raster_input in group2]), axis=0
+    )
+
+    return ma.mean(ma.stack([group1, group2]), axis=0)

examples/hillshade/hillshade.mapchete

@@ -0,0 +1,21 @@
+process: mapchete.processes.hillshade
+zoom_levels:
+    min: 0
+    max: 8
+pyramid:
+    grid: geodetic
+    metatiling: 1
+    pixelbuffer: 2
+input:
+    dem: ../../test/testdata/cleantopo_tl.tif
+output:
+    path: output
+    format: GTiff
+    dtype: uint8
+    bands: 1
+process_parameters:
+    resampling: "nearest"
+    azimuth: 315.0
+    altitude: 45.0
+    z: 1.0
+    scale: 1.0

mapchete/__init__.py

@@ -1,12 +1,19 @@
 import logging
 import os
-from typing import Optional, Union
+from typing import Union
 
 from fsspec import AbstractFileSystem
 
 from mapchete.config import MapcheteConfig
+from mapchete.errors import Empty, MapcheteNodataTile
 from mapchete.executor import Executor, MFuture
 from mapchete.formats import read_output_metadata
+from mapchete.formats.protocols import (
+    RasterInput,
+    RasterInputGroup,
+    VectorInput,
+    VectorInputGroup,
+)
 from mapchete.path import MPath, fs_from_path
 from mapchete.processing import Mapchete, MapcheteProcess
 from mapchete.tile import count_tiles
@@ -19,7 +26,13 @@
     "MapcheteProcess",
     "Timer",
     "Executor",
+    "Empty",
+    "MapcheteNodataTile",
     "MFuture",
+    "RasterInput",
+    "RasterInputGroup",
+    "VectorInput",
+    "VectorInputGroup",
 ]
 __version__ = "2024.5.0"
 
@@ -30,8 +43,6 @@
 def open(
     some_input: Union[MPathLike, dict, MapcheteConfig],
     with_cache: bool = False,
-    fs: Optional[AbstractFileSystem] = None,
-    fs_kwargs: Optional[dict] = None,
     **kwargs,
 ) -> Mapchete:
     """
@@ -72,11 +83,9 @@ def open(
     # for TileDirectory inputs
     if isinstance(some_input, MPath) and some_input.suffix == "":
         logger.debug("assuming TileDirectory")
-        metadata_json = MPath.from_inp(some_input).joinpath("metadata.json")
-        fs_kwargs = fs_kwargs or {}
-        fs = fs or fs_from_path(metadata_json, **fs_kwargs)
+        metadata_json = MPath.from_inp(some_input) / "metadata.json"
         logger.debug("read metadata.json")
-        metadata = read_output_metadata(metadata_json, fs=fs)
+        metadata = read_output_metadata(metadata_json)
         config = dict(
             process=None,
             input=None,
@@ -88,8 +97,6 @@ def open(
                     if k not in ["delimiters", "mode"]
                 },
                 path=some_input,
-                fs=fs,
-                fs_kwargs=fs_kwargs,
                 **kwargs,
             ),
             config_dir=os.getcwd(),

mapchete/cli/default/cp.py

@@ -1,5 +1,4 @@
 import click
-import tqdm
 
 from mapchete import commands
 from mapchete.cli import options

mapchete/cli/default/rm.py

@@ -4,7 +4,7 @@
 from mapchete import commands
 from mapchete.cli import options
 from mapchete.cli.progress_bar import PBar
-from mapchete.commands._rm import existing_paths
+from mapchete.commands.rm import existing_paths
 
 
 @click.command(help="Remove tiles from TileDirectory.")

mapchete/cli/options.py

@@ -31,8 +31,8 @@ def _validate_zoom(ctx, param, zoom):
             if len(zoom_levels) > 2:
                 raise ValueError("zooms can be maximum two items")
             return ZoomLevels.from_inp(zoom_levels)
-        except Exception as e:
-            raise click.BadParameter(e)
+        except Exception as exc:
+            raise click.BadParameter(str(exc))
 
 
 def _validate_bounds(ctx, param, bounds):

mapchete/commands/__init__.py

@@ -2,10 +2,11 @@
 This package contains easy to access functions which otherwise would have to be called via the CLI.
 This should make the use from within other scripts, notebooks, etc. easier.
 """
-from mapchete.commands._convert import convert
-from mapchete.commands._cp import cp
-from mapchete.commands._execute import execute
-from mapchete.commands._index import index
-from mapchete.commands._rm import rm
+
+from mapchete.commands.convert import convert
+from mapchete.commands.cp import cp
+from mapchete.commands.execute import execute
+from mapchete.commands.index import index
+from mapchete.commands.rm import rm
 
 __all__ = ["convert", "cp", "execute", "index", "rm"]

mapchete/commands/_convert.py → mapchete/commands/convert.py

@@ -3,15 +3,15 @@
 from contextlib import AbstractContextManager
 from multiprocessing import cpu_count
 from pprint import pformat
-from typing import List, NoReturn, Optional, Tuple, Type, Union
+from typing import List, Optional, Tuple, Type, Union
 
 import tilematrix
 from rasterio.crs import CRS
 from rasterio.enums import Resampling
 from shapely.geometry import box
 from shapely.geometry.base import BaseGeometry
 
-from mapchete.commands._execute import execute
+from mapchete.commands.execute import execute
 from mapchete.commands.observer import ObserverProtocol, Observers
 from mapchete.commands.parser import InputInfo, OutputInfo
 from mapchete.config import DaskSettings
@@ -43,7 +43,7 @@ def convert(
     dask_settings: DaskSettings = DaskSettings(),
     workers: Optional[int] = None,
     clip_geometry: Optional[str] = None,
-    bidx: Optional[List[int]] = None,
+    bidx: Optional[Union[List[int], int]] = None,
     output_pyramid: Optional[Union[str, dict, MPathLike]] = None,
     output_metatiling: Optional[int] = None,
     output_format: Optional[str] = None,
@@ -63,7 +63,7 @@ def convert(
     retry_on_exception: Tuple[Type[Exception], Type[Exception]] = Exception,
     cancel_on_exception: Type[Exception] = JobCancelledError,
     retries: int = 0,
-) -> NoReturn:
+) -> None:
     """
     Convert mapchete outputs or other geodata.
 
@@ -121,9 +121,11 @@ def convert(
                 ),
             )
             if output_pyramid
-            else input_info.output_pyramid.to_dict()
-            if input_info.output_pyramid
-            else None
+            else (
+                input_info.output_pyramid.to_dict()
+                if input_info.output_pyramid
+                else None
+            )
         ),
         output=dict(
             {
@@ -139,9 +141,11 @@ def convert(
             ),
             dtype=output_dtype or input_info.output_params.get("dtype"),
             **creation_options,
-            **dict(overviews=True, overviews_resampling=overviews_resampling_method)
-            if overviews
-            else dict(),
+            **(
+                dict(overviews=True, overviews_resampling=overviews_resampling_method)
+                if overviews
+                else dict()
+            ),
         ),
         config_dir=os.getcwd(),
         zoom_levels=zoom or input_info.zoom_levels,