diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1fe980..4c4b4b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,24 +1,69 @@ -# This workflow will install Conda environment, run tests and lint with a single version of Python. - name: Continuous Integration on: + schedule: + # Run this workflow at 00:00 on every Monday + - cron: "0 0 * * MON" push: - branches: [ main, master ] + branches: [ master ] pull_request: - branches: [ main, master ] + branches: [ master ] workflow_dispatch: jobs: - ci: - name: CI + build-test-taswira: + name: Build and Test Taswira runs-on: ubuntu-latest - env: - DOCKER_BUILDKIT: "1" + steps: - name: Checkout code uses: actions/checkout@v2 - - name: Lint using pylint - run: docker build . --target lint - - name: Test using pytest - run: docker build . --target unit-test + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Setup Conda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.7 + mamba-version: "*" + channels: conda-forge + channel-priority: true + activate-environment: taswira + + - name: Cache conda + uses: actions/cache@v2 + env: + # Increase this value to reset cache if environment.yml has not changed + CACHE_NUMBER: 0 + with: + # Cache the path where dependencies are installed with a key unique to the existing environment.yml + path: /usr/local/miniconda/envs/taswira + key: + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }} + id: envcache + + - name: Update Conda Environment + # Setup (or update) the environment for faster installs + run: mamba env update -n taswira -f environment.yml + + - name: Install Python package + shell: bash -l {0} + run: | + conda activate taswira + python -m pip install -e . + + - name: Run Black + run: | + python -m pip install black + black src --diff + black --check src + + - name: Test + shell: bash -l {0} + # Test the package + run: | + conda activate taswira + python -m pytest diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..bb5e9ca --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,58 @@ +name: Docker CI/CD + +on: + schedule: + # Run this workflow at 00:00 on every Monday + - cron: "0 0 * * MON" + pull_request: + branches: [ master ] + push: + branches: [ master ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + +jobs: + build: + name: Build and publish Taswira + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into GitHub Container Registry + if: ${{ github.event_name != 'pull_request' }} + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + # It will push the image to ghcr.io/moja-global/taswira:master on push + with: + images: ${{ env.REGISTRY }}/moja-global/taswira + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + # Prevent push to GHCR on pull requests + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # Implement Docker build caching for faster builds + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/environment.yml b/environment.yml index 92a8ec2..1affecc 100644 --- a/environment.yml +++ b/environment.yml @@ -15,3 +15,5 @@ dependencies: - pylint>=2.5.2 - dash==1.13.3 - dash-leaflet==0.0.19 + - markupsafe<2.1 + - jinja2 diff --git a/src/taswira/app.py b/src/taswira/app.py index e121d99..bca814f 100644 --- a/src/taswira/app.py +++ b/src/taswira/app.py @@ -9,8 +9,10 @@ from dash.dependencies import Input, Output, State from terracotta.handlers.colormap import colormap as get_colormap -BASE_MAP_ATTRIBUTION = ('© ' - 'OpenStreetMap contributors') +BASE_MAP_ATTRIBUTION = ( + '© ' + "OpenStreetMap contributors" +) N_COLORBAR_ROWS = 6 @@ -60,16 +62,18 @@ def get_colorbar(stretch_range, colormap): """ ctg = [ f'{cmap["value"]:.3f}+' - for cmap in get_colormap(stretch_range=stretch_range, - colormap=colormap, - num_values=N_COLORBAR_ROWS) + for cmap in get_colormap( + stretch_range=stretch_range, colormap=colormap, num_values=N_COLORBAR_ROWS + ) ] - return dlx.categorical_colorbar(categories=ctg, - colorscale=colormap, - width=20, - height=100, - position="bottomright") + return dlx.categorical_colorbar( + categories=ctg, + colorscale=colormap, + width=20, + height=100, + position="bottomright", + ) def get_app(): @@ -84,81 +88,91 @@ def get_app(): # pylint: disable=unused-variable data = _get_data() app = dash.Dash(__name__, server=False) - app.title = 'Taswira' - options = [{'label': k, 'value': k} for k in list(data)] + app.title = "Taswira" + options = [{"label": k, "value": k} for k in list(data)] app.layout = html.Div( [ - dcc.Store(id='raster-layers-store'), - dcc.Dropdown(id='title-dropdown', - clearable=False, - options=options, - value=options[0]['value'], - style={ - 'position': 'relative', - 'top': '5px', - 'zIndex': '500', - 'height': '0', - 'maxWidth': '200px', - 'marginLeft': 'auto', - 'marginRight': '10px' - }), - html.Div(dl.Map([ - dl.TileLayer(attribution=BASE_MAP_ATTRIBUTION), - dl.LayerGroup(id='raster-layers'), - dl.LayerGroup(id='colorbar-layer') - ], - id='main-map'), - id='main-map-div', - style={ - 'position': 'relative', - 'width': '100%', - 'height': '70%', - 'top': '0', - 'left': '0' - }), + dcc.Store(id="raster-layers-store"), + dcc.Dropdown( + id="title-dropdown", + clearable=False, + options=options, + value=options[0]["value"], + style={ + "position": "relative", + "top": "5px", + "zIndex": "500", + "height": "0", + "maxWidth": "200px", + "marginLeft": "auto", + "marginRight": "10px", + }, + ), + html.Div( + dl.Map( + [ + dl.TileLayer(attribution=BASE_MAP_ATTRIBUTION), + dl.LayerGroup(id="raster-layers"), + dl.LayerGroup(id="colorbar-layer"), + ], + id="main-map", + ), + id="main-map-div", + style={ + "position": "relative", + "width": "100%", + "height": "70%", + "top": "0", + "left": "0", + }, + ), html.Div( [ - html.Button(id='animation-btn'), - dcc.Interval(id='animation-interval', disabled=True) + html.Button(id="animation-btn"), + dcc.Interval(id="animation-interval", disabled=True), ], style={ - 'position': 'relative', - 'top': '-50px', - 'left': '10px', - 'zIndex': '500', - 'height': '0', + "position": "relative", + "top": "-50px", + "left": "10px", + "zIndex": "500", + "height": "0", }, - id="animation-control"), + id="animation-control", + ), html.Div( - [dcc.Slider( - id='year-slider', - step=None, - value=0, - )], + [ + dcc.Slider( + id="year-slider", + step=None, + value=0, + ) + ], style={ - 'position': 'relative', - 'top': '-50px', - 'left': '60px', - 'zIndex': '500', - 'height': '0', - 'marginRight': '9em' + "position": "relative", + "top": "-50px", + "left": "60px", + "zIndex": "500", + "height": "0", + "marginRight": "9em", }, - id='year-slider-div'), - dcc.Graph(id='indicator-change-graph', - responsive=True, - style={ - 'width': '100%', - 'height': '30%' - }) + id="year-slider-div", + ), + dcc.Graph( + id="indicator-change-graph", + responsive=True, + style={"width": "100%", "height": "30%"}, + ), ], style={ - 'position': 'absolute', - 'width': '100%', - 'height': '100%', - 'top': '0', - 'left': '0', - 'fontFamily': 'sans-serif' - }) + "position": "absolute", + "width": "100%", + "height": "100%", + "top": "0", + "left": "0", + "fontFamily": "sans-serif", + }, + ) app.clientside_callback( """ @@ -170,68 +184,75 @@ def get_app(): return l; }); } - """, Output('raster-layers', 'children'), - [Input('year-slider', 'value'), - Input('raster-layers-store', 'data')], - [State('raster-layers', 'children')]) - - @app.callback([ - Output('raster-layers-store', 'data'), - Output('colorbar-layer', 'children'), - Output('main-map', 'bounds') - ], [Input('title-dropdown', 'value')]) + """, + Output("raster-layers", "children"), + [Input("year-slider", "value"), Input("raster-layers-store", "data")], + [State("raster-layers", "children")], + ) + + @app.callback( + [ + Output("raster-layers-store", "data"), + Output("colorbar-layer", "children"), + Output("main-map", "bounds"), + ], + [Input("title-dropdown", "value")], + ) def update_raster_layers_colobar_map_bounds(title): - ranges = [data[title][year]['range'] for year in data[title].keys()] + ranges = [data[title][year]["range"] for year in data[title].keys()] lowers, uppers = list(zip(*ranges)) stretch_range = [min(lowers), max(uppers)] - xyz = '{z}/{x}/{y}' + xyz = "{z}/{x}/{y}" layers = [] for year in data[title]: raster_data = data[title][year] - colormap = raster_data['metadata']['colormap'] - bounds = format_bounds(raster_data['bounds']) + colormap = raster_data["metadata"]["colormap"] + bounds = format_bounds(raster_data["bounds"]) layers.append( dl.TileLayer( - url= - f'/singleband/{title}/{year}/{xyz}.png?colormap={colormap}', + url=f"/singleband/{title}/{year}/{xyz}.png?colormap={colormap}", opacity=0, - id=year)) + id=year, + ) + ) colorbar = get_colorbar(stretch_range, colormap) return layers, [colorbar], bounds - @app.callback([ - Output('year-slider', 'marks'), - Output('year-slider', 'min'), - Output('year-slider', 'max'), - ], [Input('title-dropdown', 'value')]) + @app.callback( + [ + Output("year-slider", "marks"), + Output("year-slider", "min"), + Output("year-slider", "max"), + ], + [Input("title-dropdown", "value")], + ) def update_slider(title): - mark_style = {'color': '#fff', 'textShadow': '1px 1px 2px #000'} - marks = { - int(k): dict(label=k, style=mark_style) - for k in data[title].keys() - } + mark_style = {"color": "#fff", "textShadow": "1px 1px 2px #000"} + marks = {int(k): dict(label=k, style=mark_style) for k in data[title].keys()} min_value = min(marks.keys()) max_value = max(marks.keys()) return marks, min_value, max_value - @app.callback(Output('year-slider', 'value'), [ - Input('year-slider', 'marks'), - Input('animation-interval', 'n_intervals') - ], [State('year-slider', 'value')]) - def update_slider_value(marks, n_intervals, current_value): # pylint: disable=unused-argument + @app.callback( + Output("year-slider", "value"), + [Input("year-slider", "marks"), Input("animation-interval", "n_intervals")], + [State("year-slider", "value")], + ) + def update_slider_value( + marks, n_intervals, current_value + ): # pylint: disable=unused-argument ctx = dash.callback_context min_value = min(marks.keys()) if ctx.triggered: - trigger = ctx.triggered[0]['prop_id'].split('.')[0] - trigger_value = ctx.triggered[0]['value'] - if trigger == 'animation-interval' and trigger_value: - new_value = get_element_after(str(current_value), - iter(marks.keys())) + trigger = ctx.triggered[0]["prop_id"].split(".")[0] + trigger_value = ctx.triggered[0]["value"] + if trigger == "animation-interval" and trigger_value: + new_value = get_element_after(str(current_value), iter(marks.keys())) if new_value is not None: return int(new_value) elif current_value: @@ -239,49 +260,59 @@ def update_slider_value(marks, n_intervals, current_value): # pylint: disable=u return int(min_value) - @app.callback(Output('indicator-change-graph', 'figure'), - [Input('title-dropdown', 'value')]) + @app.callback( + Output("indicator-change-graph", "figure"), [Input("title-dropdown", "value")] + ) def update_graph(title): fig = go.Figure() x_marks = [] y_margs = [] for year, meta in data[title].items(): x_marks.append(year) - y_margs.append(meta['metadata']['indicator_value']) - fig.add_trace(go.Scatter(x=x_marks, y=y_margs, mode='lines+markers')) + y_margs.append(meta["metadata"]["indicator_value"]) + fig.add_trace(go.Scatter(x=x_marks, y=y_margs, mode="lines+markers")) - unit = '' + unit = "" for _, meta in data[title].items(): - unit = meta['metadata']['unit'] + unit = meta["metadata"]["unit"] break - fig.update_layout(autosize=False, - xaxis_title='Year', - yaxis_title=f'{title} ({unit})', - xaxis_type='category', - height=150, - margin=dict(t=10, b=0)) + fig.update_layout( + autosize=False, + xaxis_title="Year", + yaxis_title=f"{title} ({unit})", + xaxis_type="category", + height=150, + margin=dict(t=10, b=0), + ) return fig - @app.callback(Output('animation-control', 'children'), - [Input('animation-btn', 'n_clicks')], [ - State('animation-btn', 'value'), - ]) - def update_animation_control(n_clicks, current_value): # pylint: disable=unused-argument - new_value = 'pause' if current_value == 'play' else 'play' - btn = html.Button(new_value.capitalize(), - value=new_value, - id='animation-btn', - style={ - 'height': '30px', - 'backgroundColor': '#fff', - 'textAlign': 'center', - 'borderRadius': '4px', - 'border': '2px solid rgba(0,0,0,0.2)', - 'fontWeight': 'bold' - }) - is_paused = (new_value == 'play') - interval = dcc.Interval(id='animation-interval', disabled=is_paused) + @app.callback( + Output("animation-control", "children"), + [Input("animation-btn", "n_clicks")], + [ + State("animation-btn", "value"), + ], + ) + def update_animation_control( + n_clicks, current_value + ): # pylint: disable=unused-argument + new_value = "pause" if current_value == "play" else "play" + btn = html.Button( + new_value.capitalize(), + value=new_value, + id="animation-btn", + style={ + "height": "30px", + "backgroundColor": "#fff", + "textAlign": "center", + "borderRadius": "4px", + "border": "2px solid rgba(0,0,0,0.2)", + "fontWeight": "bold", + }, + ) + is_paused = new_value == "play" + interval = dcc.Interval(id="animation-interval", disabled=is_paused) return [btn, interval] return app diff --git a/src/taswira/scripts/arg_types.py b/src/taswira/scripts/arg_types.py index 80b7ae3..4b86833 100644 --- a/src/taswira/scripts/arg_types.py +++ b/src/taswira/scripts/arg_types.py @@ -39,7 +39,8 @@ def indicator_file(path): for key in INDICATOR_REQUIRED_KEYS: if not key in indicator: raise argparse.ArgumentTypeError( - f"Required key `{key}` missing in config element {i}.") + f"Required key `{key}` missing in config element {i}." + ) return config diff --git a/src/taswira/scripts/console.py b/src/taswira/scripts/console.py index cbb0763..adbe208 100644 --- a/src/taswira/scripts/console.py +++ b/src/taswira/scripts/console.py @@ -25,45 +25,53 @@ def start_servers(dbpath, port): dbpath: Path to a Terracota-generated DB. port: Port number for Terracotta server. """ + def handler(signum, frame): # pylint: disable=unused-argument sys.exit(0) signal.signal(signal.SIGINT, handler) - tc.update_settings(DRIVER_PATH=dbpath, DRIVER_PROVIDER='sqlite') + tc.update_settings(DRIVER_PATH=dbpath, DRIVER_PROVIDER="sqlite") app = get_app() app.init_app(tc_app) def open_browser(): - webbrowser.open(f'http://localhost:{port}') + webbrowser.open(f"http://localhost:{port}") threading.Timer(2, open_browser).start() - if 'DEBUG' in os.environ: + if "DEBUG" in os.environ: app.run_server(port=port, threaded=False, debug=True) else: - print('Starting Taswira...') - run_simple('localhost', port, app.server) + print("Starting Taswira...") + run_simple("localhost", port, app.server) def console(): """The command-line interface for Taswira""" parser = argparse.ArgumentParser( - description="Interactive visualization tool for GCBM") + description="Interactive visualization tool for GCBM" + ) parser.add_argument( "config", type=arg_types.indicator_file, help="path to JSON config file", ) - parser.add_argument("spatial_results", - type=arg_types.spatial_results, - help="path to GCBM spatial output directory") - parser.add_argument("db_results", - type=arg_types.db_results, - help="path to compiled GCBM results database") - parser.add_argument("--allow-unoptimized", - action="store_true", - help="allow processing unoptimized raster files") + parser.add_argument( + "spatial_results", + type=arg_types.spatial_results, + help="path to GCBM spatial output directory", + ) + parser.add_argument( + "db_results", + type=arg_types.db_results, + help="path to compiled GCBM results database", + ) + parser.add_argument( + "--allow-unoptimized", + action="store_true", + help="allow processing unoptimized raster files", + ) args = parser.parse_args() update_config(args.config) @@ -71,14 +79,19 @@ def console(): with tempfile.TemporaryDirectory() as tmpdirname: try: if args.allow_unoptimized: - warnings.simplefilter('ignore') # Supress Terracotta warnings - - dbpath = ingest(args.spatial_results, args.db_results, tmpdirname, - args.allow_unoptimized) + warnings.simplefilter("ignore") # Supress Terracotta warnings + + dbpath = ingest( + args.spatial_results, + args.db_results, + tmpdirname, + args.allow_unoptimized, + ) port = get_free_port() start_servers(dbpath, port) except UnoptimizedRaster: - sys.exit("""\ + sys.exit( + """\ Found a raster file that is not a valid cloud-optimized GeoTIFFs. This tool wasn't designed to work with such files. You can try continuing anyway by passing the `--allow-unoptimized` flag but it's not recommended. @@ -87,6 +100,7 @@ def console(): the following GDAL parameters: BIGTIFF=YES, TILED=YES, COMPRESS=ZSTD, ZSTD_LEVEL=1 -""") +""" + ) except KeyboardInterrupt: sys.exit("Raster loading was interrupted") diff --git a/src/taswira/scripts/ingestion.py b/src/taswira/scripts/ingestion.py index c853e64..f40a831 100644 --- a/src/taswira/scripts/ingestion.py +++ b/src/taswira/scripts/ingestion.py @@ -11,12 +11,12 @@ from . import get_config from .metadata import get_metadata -DB_NAME = 'terracotta.sqlite' -GCBM_RASTER_NAME_PATTERN = r'.*_(?P\d{4}).tiff' -GCBM_RASTER_KEYS = ('title', 'year') +DB_NAME = "terracotta.sqlite" +GCBM_RASTER_NAME_PATTERN = r".*_(?P\d{4}).tiff" +GCBM_RASTER_KEYS = ("title", "year") GCBM_RASTER_KEYS_DESCRIPTION = { - 'title': 'Name of indicator', - 'year': 'Year of raster data', + "title": "Name of indicator", + "year": "Year of raster data", } @@ -24,10 +24,9 @@ def _find_raster_year(raster_path): raster_filename = os.path.basename(raster_path) match = re.match(GCBM_RASTER_NAME_PATTERN, raster_filename) if match is None: - raise ValueError( - f'Input file {raster_filename} does not match raster pattern') + raise ValueError(f"Input file {raster_filename} does not match raster pattern") - return match.group('year') + return match.group("year") class UnoptimizedRaster(Exception): @@ -46,33 +45,32 @@ def ingest(rasterdir, db_results, outputdir, allow_unoptimized=False): Returns: Path to generated DB. """ - driver = get_driver(os.path.join(outputdir, DB_NAME), provider='sqlite') + driver = get_driver(os.path.join(outputdir, DB_NAME), provider="sqlite") driver.create(GCBM_RASTER_KEYS, GCBM_RASTER_KEYS_DESCRIPTION) - progress = tqdm.tqdm(get_config(), desc='Searching raster files') + progress = tqdm.tqdm(get_config(), desc="Searching raster files") raster_files = [] for config in progress: - for file in glob.glob(rasterdir + os.sep + config['file_pattern']): + for file in glob.glob(rasterdir + os.sep + config["file_pattern"]): if not is_valid_cog(file) and not allow_unoptimized: raise UnoptimizedRaster raster_files.append(dict(path=file, **config)) with driver.connect(): metadata = get_metadata(db_results) - progress = tqdm.tqdm(raster_files, desc='Processing raster files') + progress = tqdm.tqdm(raster_files, desc="Processing raster files") for raster in progress: - title = raster.get('title', raster['database_indicator']) - year = _find_raster_year(raster['path']) - unit = find_units(raster.get('graph_units')) + title = raster.get("title", raster["database_indicator"]) + year = _find_raster_year(raster["path"]) + unit = find_units(raster.get("graph_units")) computed_metadata = driver.compute_metadata( - raster['path'], + raster["path"], extra_metadata={ - 'indicator_value': str(metadata[title][year]), - 'colormap': raster.get('palette').lower(), - 'unit': unit.value[2] - }) - driver.insert((title, year), - raster['path'], - metadata=computed_metadata) + "indicator_value": str(metadata[title][year]), + "colormap": raster.get("palette").lower(), + "unit": unit.value[2], + }, + ) + driver.insert((title, year), raster["path"], metadata=computed_metadata) return driver.path diff --git a/src/taswira/scripts/metadata.py b/src/taswira/scripts/metadata.py index b8e0d12..455eeaf 100644 --- a/src/taswira/scripts/metadata.py +++ b/src/taswira/scripts/metadata.py @@ -14,16 +14,16 @@ def _get_simulation_years(conn): - years = conn.execute( - "SELECT MIN(year), MAX(year) from v_age_indicators").fetchone() + years = conn.execute("SELECT MIN(year), MAX(year) from v_age_indicators").fetchone() return years def _find_indicator_table(conn, indicator): for table, value_col in RESULTS_TABLES.items(): - if conn.execute(f"SELECT 1 FROM {table} WHERE indicator = ?", - [indicator]).fetchone(): + if conn.execute( + f"SELECT 1 FROM {table} WHERE indicator = ?", [indicator] + ).fetchone(): return table, value_col return None, None @@ -34,7 +34,8 @@ def _get_annual_result(conn, indicator, units=Units.Tc): _, units_tc, _ = units.value start_year, end_year = _get_simulation_years(conn) - db_result = conn.execute(f""" + db_result = conn.execute( + f""" SELECT years.year, COALESCE(SUM(i.{value_col}), 0) / {units_tc} AS value FROM (SELECT DISTINCT year FROM v_age_indicators ORDER BY year) AS years LEFT JOIN {table} i @@ -43,7 +44,8 @@ def _get_annual_result(conn, indicator, units=Units.Tc): AND (years.year BETWEEN {start_year} AND {end_year}) GROUP BY years.year ORDER BY years.year - """).fetchall() + """ + ).fetchall() data = OrderedDict() for year, value in db_result: @@ -64,7 +66,7 @@ def get_metadata(db_results): metadata = {} conn = sqlite3.connect(db_results) for config in get_config(): - indicator = config['database_indicator'] - title = config.get('title', indicator) + indicator = config["database_indicator"] + title = config.get("title", indicator) metadata[title] = _get_annual_result(conn, indicator) return metadata diff --git a/src/taswira/units.py b/src/taswira/units.py index 1a3f821..f647280 100644 --- a/src/taswira/units.py +++ b/src/taswira/units.py @@ -4,6 +4,7 @@ class Units(Enum): """Enum of units containing tuple (IsPerHa, Mult, Label)""" + Blank = False, 1, "" Tc = False, 1, "tC" Ktc = False, 1e3, "KtC"