Merge main branch to dev

.github/workflows/push.yaml

@@ -14,11 +14,9 @@ jobs:
   install:
     strategy:
       matrix:
-        K3S_VERSION: ["v1.27.14+k3s1", "v1.28.10+k3s1", "v1.29.5+k3s1", "v1.30.1+k3s1"]
+        K3S_VERSION: ["v1.30.6+k3s1", "v1.31.2+k3s1"]
         exclude:
-          - K3S_VERSION: ${{ github.base_ref != 'refs/heads/main' && 'v1.27.14+k3s1' }}
-          - K3S_VERSION: ${{ github.base_ref != 'refs/heads/main' && 'v1.28.10+k3s1' }}
-          - K3S_VERSION: ${{ github.base_ref != 'refs/heads/main' && 'v1.29.5+k3s1' }}
+          - K3S_VERSION: ${{ github.base_ref != 'refs/heads/main' && 'v1.30.6+k3s1' }}
     runs-on: self-hosted
     steps:
       - name: Cleanup
@@ -33,10 +31,10 @@ jobs:
 
       - name: Install dependencies
         env:
-          HELM_VERSION: "v3.11.3"
-          HELM_DIFF_VERSION: "3.6.0"
-          HELMFILE_VERSION: "v0.152.0"
-          YQ_VERSION: "v4.33.3"
+          HELM_VERSION: "v3.16.3"
+          HELM_DIFF_VERSION: "3.9.12"
+          HELMFILE_VERSION: "v0.169.1"
+          YQ_VERSION: "v4.44.3"
         run: |
           ./.github/ci_config/bin/install-dependencies
 

CONTRIBUTING.md

@@ -110,6 +110,41 @@ Then you can make a new fork or branch and make your changes there and after you
 If you're changing an existing code, make sure that it is either backwards compatible or the documentation shows a clear path of applying the changes without breaking the existing installations.
 
 
+#### Development automation
+
+This repository can be used for development automation for instance on a k3s or k3d (dockerized k3s) cluster. The example below shows how to deploy on a k3d cluster.
+
+1. Install k3d (see [here](https://github.com/k3d-io/k3d#get))
+2. Create a k3d cluster that is configured to run RADAR-base
+
+```shell
+k3d cluster create my-test-cluster --port '80:80@loadbalancer' --config=.github/ci_config/k3d-config.yaml
+```
+
+This example creates a cluster named `my-test-cluster` with a load balancer that forwards local port 80 to the cluster. The
+configuration file `.github/ci_config/k3d-config.yaml` is used to configure the cluster. This cluster will be accessible
+in _kubectl_ with context name _k3d-my-test-cluster_.
+
+3. Initialize the RADAR-Kubernetes deployment. Run:
+
+```shell
+./bin/init
+```
+
+4. In file _etc/production.yaml_:
+
+- set _kubeContext_ to _k3d-my-test-cluster_
+- set _dev_deployment_ to _true_
+- (optional) enable/disable components as needed with the __install_ fields
+
+5. Install RADAR-Kubernetes on the k3d cluster:
+
+```shell
+helmfile sync
+```
+
+When installation is complete, you can access the applications at `http://localhost`.
+
 #### Adding a new component to RADAR-Kuberentes
 In order to add a new component you first need to add its helm chart to [radar-helm-charts)](https://github.com/RADAR-base/radar-helm-charts) repository. Refer to contributing guidelines of that repository for more information. Once the chart has been added you need to:
 - Add a helmfile for it in `helmfile.d` directory. The helmfiles are seperated in a modular way in order to avoid having a huge file and also installing certain components in order. Have a look at the current helmfiles and if your component is related to one of them add your component in that file other file create a new file. If your component is a dependency to other components, like Kafka or PostgreSQL prefix the file name with a smaller number so it will be installed first, but if it's a standalone component, the prefix number can be higher.

bin/generate-secrets

@@ -64,8 +64,8 @@ nginx_auth_password=$(generate_secret)
 secret="thehyve:$(echo $nginx_auth_password | openssl passwd -apr1 -stdin)" insert_secret ".kube_prometheus_stack.nginx_auth"
 comment="username: thehyve, password: $nginx_auth_password" yq -i ".kube_prometheus_stack.nginx_auth line_comment |= strenv(comment)" etc/secrets.yaml
 
-# Generate secrets for all remaining fields with value 'secret'.
-replacements=$(yq e '.. | select(. == "secret") | [(path | "."+join("."))] | join(" ")' etc/secrets.yaml);
+# Generate secrets for all remaining fields with value 'secret' or 'secret-secret' (if the component has a min length requirement).
+replacements=$(yq e '.. | select(. == "secret" or . == "secret-secret") | [(path | "."+join("."))] | join(" ")' etc/secrets.yaml);
 for key in $replacements; do
   insert_secret $key
 done

docs/upgrade.md

@@ -0,0 +1,203 @@
+# Upgrade instructions
+
+Run the following instructions to upgrade an existing RADAR-Kubernetes cluster.
+
+| :exclamation: Note                                                                                                                                                                                                                                |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Upgrading the major version of a PostgreSQL image is not supported. If necessary, we propose to use a `pg_dump` to dump the current data and a `pg_restore` to restore that data on a newer version. Please find instructions for this elsewhere. |
+
+## Upgrade to RADAR-Kubernetes version 1.2.0
+
+### Update `production.yaml` file
+
+1. Remove any line beginning with `_chart_version:`.
+2. Remove any line beginning with `imageTag:`.
+3. Add email server config to `management_portal` and `radar-appserver` sections analogous to:
+
+```yaml
+    management_portal:
+        smtp:
+            enabled: true
+            host: smtp
+            port: 25
+            from: [email protected]
+            starttls: false
+            auth: true
+```
+
+```yaml
+    radar_appserver:
+        smtp:
+            enabled: true
+            host: smtp
+            port: 25
+            from: [email protected]
+            starttls: false
+            auth: true
+```
+
+4. Update _timescaledb_ database configuration:
+
+- Rename `timescaledb_username` to `grafana_metrics_db_username`
+- Remove `grafana_metrics_username` and `timescaledb_db_name` variables.
+- When using _realtime-dashboard_, add `realtime_dashboard_db_username` that points to the current value of `timescaledb_username`. 
+
+5. For databases where data should persist after the update uncomment the respective `existingClaim` field. Example:
+
+```yaml
+realtime_dashboard_timescaledb:
+  postgresql:
+    primary:
+      persistence:
+        existingClaim: "data-timescaledb-postgresql-0"
+```
+
+### Update `secrets.yaml` file
+
+1. Add the following new secrets to the `secrets.yaml` file to correctponding sections:
+
+```yaml
+management_portal:
+  oauth_clients:
+    radar_data_dashboard_backend:
+      client_secret: <add your own random secret here>
+```
+
+```yaml
+radar_appserver:
+  smtp:
+    username: <your smtp username>
+    password: <your smtp password>
+```
+
+```yaml
+data_dashboard_db_password: <same password as timescaledb_password>
+realtime_dashboard_db_password: <same password as timescaledb_password>
+```
+
+2. Rename the `grafana_metrics_password` secret to `grafana_metrics_db_password` and `timescaledb_password` to `data_dashboard_db_password`.
+
+
+### MongoDB
+MongoDB has been updated to a new version and it's not compatible with the current version that has been installed in the cluster. There are two pathes forward:
+- Deleting the MongoDB and its volumes and then installing it and configuring Graylog again. This is the recommeneded approach since usually there is no important data is stored in MongoDB and the Graylog stack will be replaced in the next release.
+- Upgrading MongoDB cluster. If you have configured Graylog significantly, then it might be better to upgrade the MongoDB instead of reinstalling it. Options are:
+  - Manually exporting the databases and doing a clean reinstall as stated in the last step and the importing the data again.
+  - Following official MongoDB upgrade instructions for version [5.0](https://www.mongodb.com/docs/manual/release-notes/5.0-upgrade-replica-set/), [6.0](https://www.mongodb.com/docs/manual/release-notes/6.0-upgrade-replica-set/) and [7.0](https://www.mongodb.com/docs/manual/release-notes/7.0-upgrade-replica-set/).
+
+## Upgrade to RADAR-Kubernetes version 1.1.x
+Before running the upgrade make sure to copy `environments.yaml.tmpl` to `environments.yaml` and if you've previously changed `environments.yaml` apply the changes again. This is necessary due to addition of `helmDefaults` and `repositories` configurations to this file.
+
+## Upgrade to RADAR-Kubernetes version 1.0.0
+
+Before running the upgrade, compare `etc/base.yaml` and `etc/base.yaml.gotmpl` with their `production.yaml` counterparts. Please ensure that all properties in `etc/base.yaml` are overridden in your `production.yaml` or that the `base.yaml` default value is fine, in which case no value needs to be provided in `production.yaml`.
+
+To upgrade the initial services, run
+
+```shell
+kubectl delete -n monitoring deployments kube-prometheus-stack-kube-state-metrics
+helm -n graylog uninstall mongodb
+kubectl delete -n graylog pvc datadir-mongodb-0 datadir-mongodb-1
+```
+
+Note that this will remove your graylog settings but not your actual logs. This step is unfortunately needed to enable credentials on the Graylog database hosted by the mongodb chart. You will need to recreate the GELF TCP input source as during install.
+
+Then run
+
+```shell
+helmfile -f helmfile.d/00-init.yaml apply --concurrency 1
+helmfile -f helmfile.d/10-base.yaml --selector name=cert-manager-letsencrypt apply
+```
+
+To update the Kafka stack, run:
+
+```shell
+helmfile -f helmfile.d/10-base.yaml apply --concurrency 1
+```
+
+After this has succeeded, edit your `production.yaml` and change the `cp_kafka.customEnv.KAFKA_INTER_BROKER_PROTOCOL_VERSION` to the corresponding version documented in the [Confluent upgrade instructions](https://docs.confluent.io/platform/current/installation/upgrade.html) of your Kafka installation. Find the currently installed version of Kafka with `kubectl exec cp-kafka-0 -c cp-kafka-broker -- kafka-topics --version`.
+When the `cp_kafka.customEnv.KAFKA_INTER_BROKER_PROTOCOL_VERSION` is updated, again run
+
+```shell
+helmfile -f helmfile.d/10-base.yaml apply
+```
+
+To upgrade to the latest PostgreSQL helm chart, in `production.yaml`, uncomment the line `postgresql.primary.persistence.existingClaim: "data-postgresql-postgresql-0"` to use the same data storage as previously. Then run
+
+```shell
+kubectl delete secrets postgresql
+kubectl delete statefulsets postgresql-postgresql
+helmfile -f helmfile.d/10-managementportal.yaml apply
+```
+
+If installed, `radar-appserver-postgresql`, uncomment the `production.yaml` line `radar_appserver_postgresql.primary.existingClaim: "data-radar-appserver-postgresql-postgresql-0"`. Then run
+
+```shell
+kubectl delete secrets radar-appserver-postgresql
+kubectl delete statefulsets radar-appserver-postgresql-postgresql
+helmfile -f helmfile.d/20-appserver.yaml apply
+```
+
+If installed, to upgrade `timescaledb`, uncomment the `production.yaml` line `timescaledb.primary.existingClaim: "data-timescaledb-postgresql-0"`. Then run
+
+```shell
+kubectl delete secrets timescaledb-postgresql
+kubectl delete statefulsets timescaledb-postgresql
+helmfile -f helmfile.d/20-grafana.yaml apply
+```
+
+If installed, to upgrade `radar-upload-postgresql`, uncomment the `production.yaml` line `radar_upload_postgresql.primary.existingClaim: "data-radar-upload-postgresql-postgresql-0"`. Then run
+
+```shell
+kubectl delete secrets radar-upload-postgresql
+kubectl delete statefulsets radar-upload-postgresql-postgresql
+helmfile -f helmfile.d/20-upload.yaml apply
+```
+
+If minio is installed, upgrade it with the following instructions:
+
+```shell
+# get minio PV and PVC
+kubectl get pv | grep export-minio- | tr -s ' ' | cut -d ' ' -f 1,6 | tr '/' ' ' | cut -d ' ' -f 1,3 | tee minio-pv.list
+# Uninstall the minio statefulset
+helm uninstall minio
+# Associate PV with the new PVC name
+while read -r pv pvc
+do
+  # Don not delete PV
+  kubectl patch pv $pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
+  # Delete PVC
+  kubectl delete pvc $pvc
+  # Name of the new PVC
+  newpvc=$(echo $pvc | sed 's/export-/data-/')
+  # Associate PV with the new PVC name
+  kubectl patch pv $pv -p '{"spec":{"claimRef":{"name": "'$newpvc'", "namespace": "default", "uid": null}}}'
+  # Create new PVC
+  cat <<EOF | sed "s/data-minio-i/$newpvc/" | kubectl apply -f -
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    app.kubernetes.io/instance: minio
+    app.kubernetes.io/name: minio
+  name: data-minio-i
+  namespace: default
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi
+EOF
+done < minio-pv.list
+# Do the new helm install.
+helmfile -f helmfile.d/20-s3.yaml apply
+```
+
+Delete the redis stateful set (this will not delete the data on the volume)
+
+```shell
+kubectl delete statefulset redis-master
+helmfile -f helmfile.d/20-s3.yaml sync --concurrency 1
+```
+

etc/base-secrets.yaml

@@ -127,10 +127,10 @@ radar_rest_sources_backend:
   postgres:
     password: secret
 # --------------------------------------------------------- 20-dashboard.yaml ---------------------------------------------------------
-timescaledb_password: secret
 grafana_password: secret
-grafana_metrics_password: secret
-
+data_dashboard_db_password: secret
+grafana_metrics_db_password: secret
+realtime_dashboard_db_password: secret
 # --------------------------------------------------------- 20-redcap.yaml ---------------------------------------------------------
 radar_integration:
   projects: []
@@ -147,9 +147,10 @@ radar_integration:
 
 # --------------------------------------------------------- 20-s3-connector.yaml ---------------------------------------------------------
 # The access keys and secret keys of object storage services should match.
-# If AWS S3 is used as a storage medium instead of minio, then fill in those.
-s3_access_key: secret
-s3_secret_key: secret
+# If AWS S3 is used as a storage medium instead of minio, then enter the AWS-provided secrets here.
+# For minio the secrets must consist of at least 8 characters.
+s3_access_key: secret-secret
+s3_secret_key: secret-secret
 
 # --------------------------------------------------------- 20-upload.yaml ---------------------------------------------------------
 radar_upload_postgres_password: secret