From 0b49eea9e2189a439dc830262f23865b4457435a Mon Sep 17 00:00:00 2001 From: Zeid Adabel Date: Tue, 27 Feb 2024 10:53:27 +0100 Subject: [PATCH] Merge pull request #26 from Ortec-Finance/broker-scaledown-component Enabling Zone Redundant High Availability Component for Sailfish-broker --- CHANGELOG.md | 6 +++ docs/features/high-availability.md | 8 +++ .../sailfish-monitoring-dashboard.yaml | 4 +- k8s/sailfish/base/foundation/broker.yaml | 5 +- .../high-availability/kustomization.yaml | 53 +++++++++++++++++++ k8s/sailfish/overlay/kustomization.yaml | 1 + .../sailfish-py/gateway/gateway.py | 2 +- .../sailfish-py/run-manager/runmanager.py | 2 +- 8 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 docs/features/high-availability.md create mode 100644 k8s/sailfish/components/high-availability/kustomization.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d01ecc..75a1d4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ # Changelog + +## v0.14.0 +Added a new component that allows you to set the Sailfish Broker in High Availability +This will make your Queue Messages Zone redundant, which means if one zone goes down, the messages will be migrated to another. +Fixes to observability dashboard outofsync issues + ## v0.13.0 Removed `activeDeadlineSeconds` from base configuration as it does not comply with the Job Paradigm as we intend it. Added Documentation that explains how the Job Paradigm is used in Sailfish. diff --git a/docs/features/high-availability.md b/docs/features/high-availability.md new file mode 100644 index 0000000..4a30998 --- /dev/null +++ b/docs/features/high-availability.md @@ -0,0 +1,8 @@ +# High Availabilty +While Sailfish Runners and Managers already are resilient as they run across multiple availability zones and can spin back up upon failures, The broker does not. + +By enabling this component you instead spin up two to three instances of the Sailfish-Broker. The `messageMigration` feature is enabled which will handle the migration of messages in the case of a Broker shutting down in a certain zone. + +## Compatibility +This feature is not compatible with the `ephemeral-broker` as the `messageMigration` feature requires the use of PersistentVolumes. + diff --git a/k8s/observability/sailfish-monitoring-dashboard.yaml b/k8s/observability/sailfish-monitoring-dashboard.yaml index 8a118db..d5c3119 100644 --- a/k8s/observability/sailfish-monitoring-dashboard.yaml +++ b/k8s/observability/sailfish-monitoring-dashboard.yaml @@ -5,7 +5,7 @@ metadata: labels: app: grafana spec: - name: sailfish-monitoring configMapRef: name: sailfish-monitoring-configmap - key: dashboard.json \ No newline at end of file + key: dashboard.json + customFolderName: Sailfish dashboards \ No newline at end of file diff --git a/k8s/sailfish/base/foundation/broker.yaml b/k8s/sailfish/base/foundation/broker.yaml index 8a0d679..a6be10c 100644 --- a/k8s/sailfish/base/foundation/broker.yaml +++ b/k8s/sailfish/base/foundation/broker.yaml @@ -14,12 +14,11 @@ spec: console: expose: true deploymentPlan: - image: placeholder + persistenceEnabled: true + messageMigration: false jolokiaAgentEnabled: false journalType: nio managementRBACEnabled: true - messageMigration: false - persistenceEnabled: true storage: size: 4Gi storageClassName: azurefile-csi # is aro specific diff --git a/k8s/sailfish/components/high-availability/kustomization.yaml b/k8s/sailfish/components/high-availability/kustomization.yaml new file mode 100644 index 0000000..b2ad7ec --- /dev/null +++ b/k8s/sailfish/components/high-availability/kustomization.yaml @@ -0,0 +1,53 @@ +kind: Component + +patches: +- target: + group: broker.amq.io + version: v1beta1 + kind: ActiveMQArtemis + name: sailfish-broker + patch: |- + - op: replace + path: /spec/deploymentPlan/messageMigration + value: true + +- target: + group: broker.amq.io + version: v1beta1 + kind: ActiveMQArtemis + name: sailfish-broker + patch: |- + - op: replace + path: /spec/deploymentPlan/size + value: 2 + +- target: + group: keda.sh + version: v1alpha1 + kind: ScaledObject + name: sailfish-amq-broker-autoscaler + patch: |- + - op: replace + path: /spec/maxReplicaCount + value: 3 + +- target: + group: keda.sh + version: v1alpha1 + kind: ScaledObject + name: sailfish-amq-broker-autoscaler + patch: |- + - op: replace + path: /spec/minReplicaCount + value: 2 + +- target: + group: keda.sh + version: v1alpha1 + kind: ScaledObject + name: sailfish-amq-broker-autoscaler + patch: |- + - op: replace + path: /spec/idleReplicaCount + value: 0 + diff --git a/k8s/sailfish/overlay/kustomization.yaml b/k8s/sailfish/overlay/kustomization.yaml index 7efce95..be19b52 100644 --- a/k8s/sailfish/overlay/kustomization.yaml +++ b/k8s/sailfish/overlay/kustomization.yaml @@ -4,6 +4,7 @@ resources: components: - ../components/sailfish-gateway - ../components/broker-scale-to-zero + - ../components/high-availability diff --git a/sailfish-example/sailfish-py/gateway/gateway.py b/sailfish-example/sailfish-py/gateway/gateway.py index 09b7bfa..3207a25 100644 --- a/sailfish-example/sailfish-py/gateway/gateway.py +++ b/sailfish-example/sailfish-py/gateway/gateway.py @@ -16,7 +16,7 @@ Demonstrates how to create an amqp connection and a sender to publish messages. """ class Send(MessagingHandler): - def __init__(self, url, address, job, username, password, QoS=1): + def __init__(self, url, address, job, username, password, QoS=2): super(Send, self).__init__() # amqp broker host url diff --git a/sailfish-example/sailfish-py/run-manager/runmanager.py b/sailfish-example/sailfish-py/run-manager/runmanager.py index f8492b6..843f92c 100644 --- a/sailfish-example/sailfish-py/run-manager/runmanager.py +++ b/sailfish-example/sailfish-py/run-manager/runmanager.py @@ -70,7 +70,7 @@ def on_disconnected(self, event): Demonstrates how to create an amqp connection and a sender to publish messages. """ class Send(MessagingHandler): - def __init__(self, url, address, jobs, username, password, QoS=1): + def __init__(self, url, address, jobs, username, password, QoS=2): super(Send, self).__init__() # amqp broker host url