diff --git a/external/grafana/Chart.yaml b/external/grafana/Chart.yaml
index 99d765d9..3e7f2fd1 100644
--- a/external/grafana/Chart.yaml
+++ b/external/grafana/Chart.yaml
@@ -32,4 +32,4 @@ sources:
 - https://github.com/grafana/grafana
 - https://github.com/grafana/helm-charts
 type: application
-version: 8.3.2
+version: 8.3.7
diff --git a/external/grafana/ci/with-affinity-values.yaml b/external/grafana/ci/with-affinity-values.yaml
new file mode 100644
index 00000000..f5b9b53e
--- /dev/null
+++ b/external/grafana/ci/with-affinity-values.yaml
@@ -0,0 +1,16 @@
+affinity:
+  podAntiAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+      - podAffinityTerm:
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/instance: grafana-test
+              app.kubernetes.io/name: grafana
+          topologyKey: failure-domain.beta.kubernetes.io/zone
+        weight: 100
+    requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: grafana-test
+            app.kubernetes.io/name: grafana
+        topologyKey: kubernetes.io/hostname
diff --git a/external/grafana/ci/with-extraconfigmapmounts-values.yaml b/external/grafana/ci/with-extraconfigmapmounts-values.yaml
new file mode 100644
index 00000000..5cc44a05
--- /dev/null
+++ b/external/grafana/ci/with-extraconfigmapmounts-values.yaml
@@ -0,0 +1,7 @@
+extraConfigmapMounts:
+  - name: '{{ include "grafana.fullname" . }}'
+    configMap: '{{ include "grafana.fullname" . }}'
+    mountPath: /var/lib/grafana/dashboards/test-dashboard.json
+    # This is not a realistic test, but for this we only care about extraConfigmapMounts not being empty and pointing to an existing ConfigMap
+    subPath: grafana.ini
+    readOnly: true
diff --git a/external/grafana/ci/with-persistence.yaml b/external/grafana/ci/with-persistence.yaml
new file mode 100644
index 00000000..b92ca02c
--- /dev/null
+++ b/external/grafana/ci/with-persistence.yaml
@@ -0,0 +1,3 @@
+persistence:
+  type: pvc
+  enabled: true
diff --git a/external/grafana/ci/with-sidecars-envvaluefrom-values.yaml b/external/grafana/ci/with-sidecars-envvaluefrom-values.yaml
new file mode 100644
index 00000000..a6935e56
--- /dev/null
+++ b/external/grafana/ci/with-sidecars-envvaluefrom-values.yaml
@@ -0,0 +1,38 @@
+extraObjects:
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: '{{ include "grafana.fullname" . }}-test'
+    data:
+      var1: "value1"
+  - apiVersion: v1
+    kind: Secret
+    metadata:
+      name: '{{ include "grafana.fullname" . }}-test'
+    type: Opaque
+    data:
+      var2: "dmFsdWUy"
+
+sidecar:
+  dashboards:
+    enabled: true
+    envValueFrom:
+      VAR1:
+        configMapKeyRef:
+          name: '{{ include "grafana.fullname" . }}-test'
+          key: var1
+      VAR2:
+        secretKeyRef:
+          name: '{{ include "grafana.fullname" . }}-test'
+          key: var2
+  datasources:
+    enabled: true
+    envValueFrom:
+      VAR1:
+        configMapKeyRef:
+          name: '{{ include "grafana.fullname" . }}-test'
+          key: var1
+      VAR2:
+        secretKeyRef:
+          name: '{{ include "grafana.fullname" . }}-test'
+          key: var2
diff --git a/external/grafana/templates/_config.tpl b/external/grafana/templates/_config.tpl
new file mode 100644
index 00000000..b866217f
--- /dev/null
+++ b/external/grafana/templates/_config.tpl
@@ -0,0 +1,172 @@
+{{/*
+ Generate config map data
+ */}}
+{{- define "grafana.configData" -}}
+{{ include "grafana.assertNoLeakedSecrets" . }}
+{{- $files := .Files }}
+{{- $root := . -}}
+{{- with .Values.plugins }}
+plugins: {{ join "," . }}
+{{- end }}
+grafana.ini: |
+{{- range $elem, $elemVal := index .Values "grafana.ini" }}
+  {{- if not (kindIs "map" $elemVal) }}
+  {{- if kindIs "invalid" $elemVal }}
+  {{ $elem }} =
+  {{- else if kindIs "string" $elemVal }}
+  {{ $elem }} = {{ tpl $elemVal $ }}
+  {{- else }}
+  {{ $elem }} = {{ $elemVal }}
+  {{- end }}
+  {{- end }}
+{{- end }}
+{{- range $key, $value := index .Values "grafana.ini" }}
+  {{- if kindIs "map" $value }}
+  [{{ $key }}]
+  {{- range $elem, $elemVal := $value }}
+  {{- if kindIs "invalid" $elemVal }}
+  {{ $elem }} =
+  {{- else if kindIs "string" $elemVal }}
+  {{ $elem }} = {{ tpl $elemVal $ }}
+  {{- else }}
+  {{ $elem }} = {{ $elemVal }}
+  {{- end }}
+  {{- end }}
+  {{- end }}
+{{- end }}
+
+{{- range $key, $value := .Values.datasources }}
+{{- if not (hasKey $value "secret") }}
+{{ $key }}: |
+  {{- tpl (toYaml $value | nindent 2) $root }}
+{{- end }}
+{{- end }}
+
+{{- range $key, $value := .Values.notifiers }}
+{{- if not (hasKey $value "secret") }}
+{{ $key }}: |
+  {{- toYaml $value | nindent 2 }}
+{{- end }}
+{{- end }}
+
+{{- range $key, $value := .Values.alerting }}
+{{- if (hasKey $value "file") }}
+{{ $key }}:
+{{- toYaml ( $files.Get $value.file ) | nindent 2 }}
+{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}}
+{{/*  will be stored inside secret generated by "configSecret.yaml"*/}}
+{{- else }}
+{{ $key }}: |
+  {{- tpl (toYaml $value | nindent 2) $root }}
+{{- end }}
+{{- end }}
+
+{{- range $key, $value := .Values.dashboardProviders }}
+{{ $key }}: |
+  {{- toYaml $value | nindent 2 }}
+{{- end }}
+
+{{- if .Values.dashboards  }}
+download_dashboards.sh: |
+  #!/usr/bin/env sh
+  set -euf
+  {{- if .Values.dashboardProviders }}
+    {{- range $key, $value := .Values.dashboardProviders }}
+      {{- range $value.providers }}
+  mkdir -p {{ .options.path }}
+      {{- end }}
+    {{- end }}
+  {{- end }}
+{{ $dashboardProviders := .Values.dashboardProviders }}
+{{- range $provider, $dashboards := .Values.dashboards }}
+  {{- range $key, $value := $dashboards }}
+    {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }}
+  curl -skf \
+  --connect-timeout 60 \
+  --max-time 60 \
+    {{- if not $value.b64content }}
+      {{- if not $value.acceptHeader }}
+  -H "Accept: application/json" \
+      {{- else }}
+  -H "Accept: {{ $value.acceptHeader }}" \
+      {{- end }}
+      {{- if $value.token }}
+  -H "Authorization: token {{ $value.token }}" \
+      {{- end }}
+      {{- if $value.bearerToken }}
+  -H "Authorization: Bearer {{ $value.bearerToken }}" \
+      {{- end }}
+      {{- if $value.basic }}
+  -H "Authorization: Basic {{ $value.basic }}" \
+      {{- end }}
+      {{- if $value.gitlabToken }}
+  -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \
+      {{- end }}
+  -H "Content-Type: application/json;charset=UTF-8" \
+    {{- end }}
+  {{- $dpPath := "" -}}
+  {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }}
+    {{- if eq $kd.name $provider }}
+    {{- $dpPath = $kd.options.path }}
+    {{- end }}
+  {{- end }}
+  {{- if $value.url }}
+    "{{ $value.url }}" \
+  {{- else }}
+    "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \
+  {{- end }}
+  {{- if $value.datasource }}
+    {{- if kindIs "string" $value.datasource }}
+    | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \
+    {{- end }}
+    {{- if kindIs "slice" $value.datasource }}
+      {{- range $value.datasource }}
+        | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \
+      {{- end }}
+    {{- end }}
+  {{- end }}
+  {{- if $value.b64content }}
+    | base64 -d \
+  {{- end }}
+  > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json"
+    {{ end }}
+  {{- end }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{/*
+ Generate dashboard json config map data
+ */}}
+{{- define "grafana.configDashboardProviderData" -}}
+provider.yaml: |-
+  apiVersion: 1
+  providers:
+    - name: '{{ .Values.sidecar.dashboards.provider.name }}'
+      orgId: {{ .Values.sidecar.dashboards.provider.orgid }}
+      {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }}
+      folder: '{{ .Values.sidecar.dashboards.provider.folder }}'
+      folderUid: '{{ .Values.sidecar.dashboards.provider.folderUid }}'
+      {{- end }}
+      type: {{ .Values.sidecar.dashboards.provider.type }}
+      disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }}
+      allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }}
+      updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }}
+      options:
+        foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }}
+        path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}
+{{- end -}}
+
+{{- define "grafana.secretsData" -}}
+{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }}
+admin-user: {{ .Values.adminUser | b64enc | quote }}
+{{- if .Values.adminPassword }}
+admin-password: {{ .Values.adminPassword | b64enc | quote }}
+{{- else }}
+admin-password: {{ include "grafana.password" . }}
+{{- end }}
+{{- end }}
+{{- if not .Values.ldap.existingSecret }}
+ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }}
+{{- end }}
+{{- end -}}
diff --git a/external/grafana/templates/configSecret.yaml b/external/grafana/templates/configSecret.yaml
new file mode 100644
index 00000000..55574b9b
--- /dev/null
+++ b/external/grafana/templates/configSecret.yaml
@@ -0,0 +1,43 @@
+{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}}
+{{- if and .Values.createConfigmap $createConfigSecret }}
+{{- $files := .Files }}
+{{- $root := . -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "{{ include "grafana.fullname" . }}-config-secret"
+  namespace: {{ include "grafana.namespace" . }}
+  labels:
+      {{- include "grafana.labels" . | nindent 4 }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+data:
+{{- range $key, $value := .Values.alerting }}
+  {{- if (hasKey $value "secretFile") }}
+  {{- $key | nindent 2 }}:
+    {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}}
+  {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}}
+  {{- end }}
+{{- end }}
+stringData:
+{{- range $key, $value := .Values.datasources }}
+{{- if (hasKey $value "secret") }}
+{{- $key | nindent 2 }}: |
+  {{- tpl (toYaml $value.secret | nindent 4) $root }}
+{{- end }}
+{{- end }}
+{{- range $key, $value := .Values.notifiers }}
+{{- if (hasKey $value "secret") }}
+{{- $key | nindent 2 }}: |
+  {{- tpl (toYaml $value.secret | nindent 4) $root }}
+{{- end }}
+{{- end }}
+{{- range $key, $value := .Values.alerting }}
+{{ if (hasKey $value "secret") }}
+  {{- $key | nindent 2 }}: |
+    {{- tpl (toYaml $value.secret | nindent 4) $root }}
+  {{- end }}
+{{- end }}
+{{- end }}
diff --git a/external/grafana/templates/image-renderer-hpa.yaml b/external/grafana/templates/image-renderer-hpa.yaml
new file mode 100644
index 00000000..b0f0059b
--- /dev/null
+++ b/external/grafana/templates/image-renderer-hpa.yaml
@@ -0,0 +1,47 @@
+{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }}
+apiVersion: {{ include "grafana.hpa.apiVersion" . }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "grafana.fullname" . }}-image-renderer
+  namespace: {{ include "grafana.namespace" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer
+    helm.sh/chart: {{ include "grafana.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "grafana.fullname" . }}-image-renderer
+  minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.imageRenderer.autoscaling.targetMemory }}
+    - type: Resource
+      resource:
+        name: memory
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }}
+        {{- else }}
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }}
+        {{- end }}
+    {{- end }}
+    {{- if .Values.imageRenderer.autoscaling.targetCPU }}
+    - type: Resource
+      resource:
+        name: cpu
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }}
+        {{- else }}
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }}
+        {{- end }}
+    {{- end }}
+  {{- if .Values.imageRenderer.autoscaling.behavior }}
+  behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }}
+  {{- end }}
+{{- end }}
diff --git a/external/grafana/templates/image-renderer-network-policy.yaml b/external/grafana/templates/image-renderer-network-policy.yaml
index d1a0eb31..bcbd2497 100644
--- a/external/grafana/templates/image-renderer-network-policy.yaml
+++ b/external/grafana/templates/image-renderer-network-policy.yaml
@@ -69,7 +69,7 @@ spec:
       to:
         - namespaceSelector:
             matchLabels:
-              name: {{ include "grafana.namespace" . }}
+              kubernetes.io/metadata.name: {{ include "grafana.namespace" . }}
           podSelector:
             matchLabels:
               {{- include "grafana.selectorLabels" . | nindent 14 }}
diff --git a/external/grafana/templates/image-renderer-servicemonitor.yaml b/external/grafana/templates/image-renderer-servicemonitor.yaml
new file mode 100644
index 00000000..5d9f09d2
--- /dev/null
+++ b/external/grafana/templates/image-renderer-servicemonitor.yaml
@@ -0,0 +1,48 @@
+{{- if .Values.imageRenderer.serviceMonitor.enabled }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "grafana.fullname" . }}-image-renderer
+  {{- if .Values.imageRenderer.serviceMonitor.namespace }}
+  namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }}
+  {{- else }}
+  namespace: {{ include "grafana.namespace" . }}
+  {{- end }}
+  labels:
+    {{- include "grafana.imageRenderer.labels" . | nindent 4 }}
+    {{- with .Values.imageRenderer.serviceMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+  - port: {{ .Values.imageRenderer.service.portName }}
+    {{- with .Values.imageRenderer.serviceMonitor.interval }}
+    interval: {{ . }}
+    {{- end }}
+    {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ . }}
+    {{- end }}
+    honorLabels: true
+    path: {{ .Values.imageRenderer.serviceMonitor.path }}
+    scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }}
+    {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }}
+    tlsConfig:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.imageRenderer.serviceMonitor.relabelings }}
+    relabelings:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  jobLabel: "{{ .Release.Name }}-image-renderer"
+  selector:
+    matchLabels:
+      {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+      - {{ include "grafana.namespace" . }}
+  {{- with .Values.imageRenderer.serviceMonitor.targetLabels }}
+  targetLabels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
diff --git a/external/grafana/templates/pvc.yaml b/external/grafana/templates/pvc.yaml
index 0a4613ed..d1c4b2de 100644
--- a/external/grafana/templates/pvc.yaml
+++ b/external/grafana/templates/pvc.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}}
+{{- if and (not .Values.useStatefulSet) .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
diff --git a/external/grafana/templates/statefulset.yaml b/external/grafana/templates/statefulset.yaml
index e6c944a4..7546c188 100644
--- a/external/grafana/templates/statefulset.yaml
+++ b/external/grafana/templates/statefulset.yaml
@@ -39,7 +39,9 @@ spec:
       {{- include "grafana.pod" . | nindent 6 }}
   {{- if .Values.persistence.enabled}}
   volumeClaimTemplates:
-  - metadata:
+  - apiVersion: v1
+    kind: PersistentVolumeClaim
+    metadata:
       name: storage
     spec:
       accessModes: {{ .Values.persistence.accessModes }}
diff --git a/external/grafana/values.yaml b/external/grafana/values.yaml
index 0eb4dbb1..0390f78f 100644
--- a/external/grafana/values.yaml
+++ b/external/grafana/values.yaml
@@ -4,7 +4,7 @@ global:
 
   # To help compatibility with other charts which use global.imagePullSecrets.
   # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style).
-  # Can be tempalted.
+  # Can be templated.
   # global:
   #   imagePullSecrets:
   #   - name: pullSecret1
@@ -859,7 +859,7 @@ sidecar:
     # -- The Docker registry
     registry: quay.io
     repository: kiwigrid/k8s-sidecar
-    tag: 1.26.1
+    tag: 1.27.4
     sha: ""
   imagePullPolicy: IfNotPresent
   resources: {}