first commit

.editorconfig

@@ -0,0 +1,24 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+indent_style = space
+indent_size = 4
+max_line_length = 88
+
+[*.{toml,yaml,yml}]
+indent_size = 2
+
+[*.{json,geojson,jsonl,js,jsx,ts,tsx,css,less,scss,html,xml}]
+indent_size = 2
+
+[*.md]
+max_line_length = off
+
+[Makefile]
+indent_style = tab
+
+### Extra lines below are preserved ###

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+
+### Extra lines below are preserved ###

.github/workflows/docker.yml

@@ -0,0 +1,28 @@
+name: docker
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  docker:
+    name: docker
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          push: ${{ github.ref == 'refs/heads/main' }}
+          tags: ghcr.io/nens/prefect-kghub-example-task:main

.github/workflows/nens-meta.yml

@@ -0,0 +1,32 @@
+# Generated by nens-meta.
+# See https://nens-meta.readthedocs.io/en/latest/config-files.html for info.
+# If you want this file to be left alone, add "nens_meta_leave_alone" in
+# all caps somewhere in this file in a comment.
+#
+
+name: nens-meta
+on:
+  push:
+    branches:
+      - master
+      - main
+  pull_request:
+    branches:
+      - master
+      - main
+
+  workflow_dispatch:
+
+jobs:
+  nens-meta:
+    name: nens-meta
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - uses: pre-commit/[email protected]
+
+### Extra lines below are preserved ###

.gitignore

@@ -0,0 +1,44 @@
+# Pyc/pyo/backup files.
+*.py[co]
+*~
+*.swp
+*.pyc
+.DS_Store
+
+# Packages/buildout/pip/pipenv
+*.egg
+*.egg-info
+sdist
+pip-log.txt
+bower_components/
+node_modules/
+doc/build/
+ansible/*.retry
+.venv
+venv
+bin/
+lib/
+pyvenv.cfg
+dist/
+var/
+*.suggestion
+
+# Our generated files
+for_step_summary.md
+
+# Unit test / coverage reports
+.coverage
+.tox
+htmlcov
+.codeintel
+coverage.*
+.pytest_cache
+
+# Pycharm, visual studio
+.idea
+.vscode
+
+# Docker
+docker-compose.override.yml
+
+### Extra lines below are preserved ###

.nens.toml

@@ -0,0 +1,9 @@
+[meta]
+uses_ansible = false
+uses_python = true
+meta_version = "0.7.dev0"
+project_name = "prefect-kghub-example-task"
+
+
+[money]
+project_number = "w0289"

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+default_language_version:
+  python: python3
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+        args: [--allow-multiple-documents]
+      - id: check-toml
+      - id: check-added-large-files
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.1.13
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: ["--fix"]
+      # Run the formatter.
+      - id: ruff-format
+
+### Extra lines below are preserved ###

Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.12.3-bookworm
+WORKDIR /code
+LABEL project_name="prefect-kghub-example-task"
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+CMD ["python", "src/server.py"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Nelen & Schuurmans
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,61 @@
+# prefect-kghub-example-task prefect task
+
+## Project Documentation
+
+#TODO: add the documentation of your code here
+
+## Prefect flow development
+
+- Create a virtual environment and install the requirements.txt:
+
+        $ python3 -m venv .venv            # Only needed once
+        $ .venv\Scripts\activate           # Activate on windows
+        $ source .venv/bin/activate        # Activate on linux
+        $ pip install -r requirements.txt  # Whenever requirements.txt changes
+
+- Write your script in the `src` folder, with your 'main' script in `flows.py`, and tasks in `tasks.py`. Feel free to add new folders or files in the `src` folder.
+
+- Create a deployment in `python src/server.py`.
+
+- To test your flow locally, start a prefect instance within your venv:
+
+        $ prefect server start
+
+- Run `python src/flows.py` to run your script, or `python src/server.py` to test the deployment.
+
+## Deploying your flow to production
+
+- Create the repo: https://github.com/nens/prefect-kghub-example-task
+
+- Start a local git project:
+
+        $ git init
+
+- To keep code readable and maintainable, pre-commit is installed. If you have never used it, install globally on your device with:
+
+        $ pip install pre-commit
+
+- Install the pre-commit for this git repo:
+
+        $ pre-commit install
+
+- Commit your code and push it to your new repo. If you have troubles with pre-commit, you can always run it manually with:
+
+        $ pre-commit run --all
+
+> **_NOTE:_**  You need to fix all the pre-commit problems if it doesn't fix them itself. If pre-commit fails, the docker image build will also fail, and your flow will not be deployed.
+
+- Add 2 new teams to your github repo: Adviseurs (write) and Nelen & Schuurmans pull only (admin)
+
+- Ask Florian or Reinout to register your new flow.
+
+
+## Optional (but definitely recommended):
+
+If you use vscode and did the `.venv` thingy above, the python plugin will detect your code and prefect. So you'll have proper code completion! And type hints become more useful. Tip: also install [the vscode editorconfig plugin](https://marketplace.visualstudio.com/items?itemName=EditorConfig.EditorConfig) because that will automatically handle unneeded spaces at the end of lines and other minutia.
+
+There are even tests files (see https://docs.pytest.org/ for instructions), you can use them to test calculations. Ask Reinout or Florian for tips. Don't test whether an ftp download can work, but *do* test when you do some real programming work.
+
+Once the virtualenv is activated, you can run the tests simply with:
+
+        $ pytest

pyproject.toml

@@ -0,0 +1,14 @@
+[tool.ruff]
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "I", "UP"]
+
+[tool.zest-releaser]
+release = false
+
+[tool.pyright]
+# Out-of-the-box vscode support, provided you created .venv as mentioned in the readme.
+venvPath = "."
+venv = ".venv"
+include = "src"

requirements.txt

@@ -0,0 +1,2 @@
+prefect<3
+pytest

src/flows.py

@@ -0,0 +1,20 @@
+from prefect import flow
+from tasks import uppercase_the_text
+
+
+@flow(
+    name="Clear name of your flow",
+    flow_run_name = "kghub_example_task Flow run",
+    description= "Short description of what the flow does.",
+    retries=0, # If wanted, place your retries count here,
+    retry_delay_seconds=10,
+    log_prints=True
+)
+def kghub_example_task_flow(text: str = "Hi"):
+    uppercase_text = uppercase_the_text(text)
+    print(f"Turned {text} into {uppercase_text}")
+
+
+if __name__ == "__main__":
+    print("Not serving flows, but running them locally for testing")
+    kghub_example_task_flow()

src/server.py

@@ -0,0 +1,15 @@
+from flows import kghub_example_task_flow
+from prefect import serve
+
+if __name__ == "__main__":
+
+    kghub_example_task_deployment = kghub_example_task_flow.to_deployment(
+        name="Clear name of your deployment",
+        interval=60, # alternative: cron="5 4 * * *". Check crontab.guru to create the correct schedule expression
+        parameters={"text": "Python is absolutely fabulous"}, # Input needs to be convertable to json
+        description="Code found at: https://github.com/nens/prefect-kghub-example-task", # Place this in EACH deployment, so that it its clear which repo serves which deployment
+        tags=["Your name", "Project keyword"] # Add the name of the first point of contact, and one or more keywords of the project
+    )
+
+
+    serve(kghub_example_task_deployment)  # It is possible to create multiple deployments and serve them all.

src/tasks.py

@@ -0,0 +1,70 @@
+from prefect import task, get_run_logger
+import requests
+import pandas as pd
+from datetime import datetime
+import json
+
+# Constants
+uuid_gpm_ts = "fb384ac4-c8e8-452e-83d3-30f23c522577"
+uuid_gpm_rast = "f23e174e-0636-45f1-86fd-e543a7aa49ac"
+
+lizard_api = 'https://weact.lizard.net/api/v4'
+kghub_api = 'http://kghub.caiag.kg/api'
+
+
+lizard_headers = {
+    "username": "__key__",
+    "password": "yLMMChiw.5inIEGlTVGsebmhAEQWSyFjoTVXsm2ln",
+    "Content-Type": "application/json",
+}
+
+
+
+bearer_token = 'testing'
+
+kghub_headers = {
+    'Authorization': f'Bearer {bearer_token}',
+    'Content-Type': 'application/json'  # Specify that the data is in JSON format
+}
+
+@task(retries=2)
+def get_precipitation(uuid, start, end, coordinates):
+    logger = get_run_logger()
+    url = f"{lizard_api}/rasters/{uuid}/point/"
+    lat = coordinates[0]
+    lon = coordinates[1]
+    params = {"geom": 'POINT (74.590958 42.871773)',
+              "start": start.isoformat() + "Z",
+              "stop": end.isoformat() + "Z"}
+
+    r = requests.get(url, params=params, headers=lizard_headers)
+    r.raise_for_status()
+    df = pd.DataFrame(r.json()["results"])
+    if df.empty:
+        logger.error("No data retrieved")
+    else:
+        logger.info(f"Retrieved {len(df)} data points")
+    return df
+
+
+def post_timeseries(ts_uuid, df_data, json_headers):
+    events_url = f"{kghub_api}/timeseries/{ts_uuid}/events/"
+    events = df_data[['time', 'value']].to_dict(orient='records')
+    print((events))
+    print(json_headers)
+    # POST each event to the server
+    response = requests.post(
+        url=events_url,
+        data=json.dumps(events),
+        headers=json_headers
+    )
+
+    return response.json()
+
+
+start = datetime(2024,11,1,0,0,0)
+end =  datetime(2024,11,6,0,0,0)
+coordinates = (74.590958, 42.871773, 0.0)
+
+df = get_precipitation(uuid_gpm_rast, start, end, coordinates)
+post_timeseries(uuid_gpm_ts, df, kghub_headers)