Auth to prometheus using token instead of basicauth

roles/servicetelemetry/tasks/component_grafana.yml

@@ -82,18 +82,18 @@
             namespace: '{{ ansible_operator_meta.namespace }}'
           register: serving_certs_ca
 
-        - name: Retrieve prometheus secret
+        - name: Retrieve prometheus reader token
           k8s_info:
             api_version: v1
             kind: Secret
             namespace: '{{ ansible_operator_meta.namespace }}'
-            name: '{{ ansible_operator_meta.name }}-prometheus-htpasswd'
-          register: prometheus_secret
+            name: stf-prometheus-reader-token
+          register: prometheus_reader_secret
 
-        - name: Decode prometheus password
+        - name: Decode prometheus reader token
           no_log: true
           set_fact:
-            prom_basicauth_passwd: '{{ prometheus_secret.resources[0].data.password | b64decode }}'
+            prometheus_reader_token: '{{ prometheus_reader_secret.resources[0].data.token | b64decode }}'
 
     # Lookup existing datasources
     - name: Remove legacy datasources

roles/servicetelemetry/tasks/component_prometheus.yml

@@ -173,46 +173,6 @@
         name: prometheus-k8s-{{ ansible_operator_meta.namespace }}
         namespace: '{{ ansible_operator_meta.namespace }}'
 
-- name: Check for existing prometheus htpasswd user secret
-  k8s_info:
-    api_version: v1
-    kind: Secret
-    namespace: '{{ ansible_operator_meta.namespace }}'
-    name: '{{ ansible_operator_meta.name }}-prometheus-htpasswd'
-  register: prometheus_htpasswd
-
-- name: Create a new prometheus password if it doesn't exist yet
-  when: prometheus_htpasswd.resources|length == 0
-  block:
-    - name: Set prometheus htpasswd
-      no_log: true
-      set_fact:
-        prom_basicauth_passwd: "{{ lookup('password', '/dev/null') }}"
-
-    - name: Create htpasswd secret  # Contains both the htpasswd version and plaintext for lookup
-      no_log: true
-      k8s:
-        definition:
-          api_version: v1
-          kind: Secret
-          metadata:
-            name: '{{ ansible_operator_meta.name }}-prometheus-htpasswd'
-            namespace: '{{ ansible_operator_meta.namespace }}'
-          type: Opaque
-          stringData:
-            auth: 'internal:{{ prom_basicauth_passwd | password_hash("bcrypt") | replace("$2b$","$2y$", 1)}}'
-            password: '{{ prom_basicauth_passwd }}'
-      tags:
-        - skip_ansible_lint
-
-    - name: Re-register new object for use in the annotation
-      k8s_info:
-        api_version: v1
-        kind: Secret
-        namespace: '{{ ansible_operator_meta.namespace }}'
-        name: '{{ ansible_operator_meta.name }}-prometheus-htpasswd'
-      register: prometheus_htpasswd
-
 - name: Lookup template
   debug:
     msg: "{{ lookup('template', './manifest_prometheus.j2') | from_yaml }}"

roles/servicetelemetry/tasks/component_prometheus_reader.yml

@@ -0,0 +1,58 @@
+- name: Create ServiceAccount/stf-prometheus-reader
+  k8s:
+    state: '{{ "present" if servicetelemetry_vars.backends.metrics.prometheus.enabled else "absent" }}'
+    definition:
+      apiVersion: v1
+      kind: ServiceAccount
+      metadata:
+        name: stf-prometheus-reader
+        namespace: '{{ ansible_operator_meta.namespace }}'
+
+- name: Create prometheus-reader Role
+  k8s:
+    state: '{{ "present" if servicetelemetry_vars.backends.metrics.prometheus.enabled else "absent" }}'
+    definition:
+      apiVersion: rbac.authorization.k8s.io/v1
+      kind: Role
+      metadata:
+        name: prometheus-reader
+        namespace: '{{ ansible_operator_meta.namespace }}'
+      rules:
+      - apiGroups:
+        - '{{ prometheus_operator_api_string | replace("/v1","") }}'
+        resources:
+        - prometheus
+        verbs:
+        - get
+        namespaces:
+        - '{{ ansible_operator_meta.namespace }}'
+
+- name: Create prometheus-reader RoleBinding for stf-prometheus-reader
+  k8s:
+    state: '{{ "present" if servicetelemetry_vars.backends.metrics.prometheus.enabled else "absent" }}'
+    definition:
+      apiVersion: rbac.authorization.k8s.io/v1
+      kind: RoleBinding
+      metadata:
+        name: stf-prometheus-reader
+        namespace: '{{ ansible_operator_meta.namespace }}'
+      roleRef:
+        apiGroup: rbac.authorization.k8s.io
+        kind: Role
+        name: prometheus-reader
+      subjects:
+      - kind: ServiceAccount
+        name: stf-prometheus-reader
+
+- name: Create an access token for stf-prometheus-reader
+  k8s:
+    state: '{{ "present" if servicetelemetry_vars.backends.metrics.prometheus.enabled else "absent" }}'
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: stf-prometheus-reader-token
+        namespace: '{{ ansible_operator_meta.namespace }}'
+        annotations:
+          kubernetes.io/service-account.name: stf-prometheus-reader
+      type: kubernetes.io/service-account-token

roles/servicetelemetry/tasks/main.yml

@@ -47,6 +47,8 @@
 - block:
   - name: Create Prometheus instance
     include_tasks: component_prometheus.yml
+  - name: Create Prometheus read-only user
+    include_tasks: component_prometheus_reader.yml
 
   # --> alerting
   - name: Create Alertmanager instance

roles/servicetelemetry/templates/manifest_grafana_ds.j2

@@ -12,14 +12,13 @@ spec:
       jsonData:
         timeInterval: 5s
         tlsAuthWithCACert: true
+        httpHeaderName1: 'Authorization'
       name: STFPrometheus
       type: prometheus
       url: 'https://{{ ansible_operator_meta.name }}-prometheus-proxy.{{ ansible_operator_meta.namespace }}.svc:9092'
       version: 1
-      basicAuth: true
-      basicAuthUser: internal
       secureJsonData:
-        basicAuthPassword: '{{ prom_basicauth_passwd }}'
+        httpHeaderValue1: 'Bearer {{prometheus_reader_token}}'
         tlsCACert: |
           {{ serving_certs_ca.resources[0].data['service-ca.crt'] | indent(10) }}
 {% endif %}

roles/servicetelemetry/templates/manifest_prometheus.j2

@@ -20,7 +20,6 @@ spec:
     labels:
       prometheus: '{{ ansible_operator_meta.name }}'
     annotations:
-      hash-of-entire-htpasswd-secret-to-force-restart-if-changed: {{ prometheus_htpasswd | sha1 }}
 {% if servicetelemetry_vars.alerting.enabled %}
   alerting:
     alertmanagers:
@@ -42,10 +41,11 @@ spec:
     - -tls-cert=/etc/tls/private/tls.crt
     - -tls-key=/etc/tls/private/tls.key
     - -upstream=http://localhost:9090/
-    - -htpasswd-file=/etc/proxy/htpasswd/auth
     - -cookie-secret-file=/etc/proxy/secrets/session_secret
     - -openshift-service-account=prometheus-stf
-    - '-openshift-sar={"resource": "namespaces", "verb": "get"}'
+    - '-openshift-sar={"namespace":"{{ ansible_operator_meta.namespace }}","resource": "prometheus", "group":"{{ prometheus_operator_api_string | replace("/v1","") }}", "verb":"get"}'
+    - '-openshift-delegate-urls={"/":{"namespace":"{{ ansible_operator_meta.namespace }}","resource": "prometheus", "group":"{{ prometheus_operator_api_string | replace("/v1","") }}", "verb":"get"}}'
+
     ports:
       - containerPort: 9092
         name: https
@@ -55,14 +55,11 @@ spec:
         name: secret-{{ ansible_operator_meta.name }}-prometheus-proxy-tls
       - mountPath: /etc/proxy/secrets
         name: secret-{{ ansible_operator_meta.name }}-session-secret
-      - mountPath: /etc/proxy/htpasswd
-        name: secret-{{ ansible_operator_meta.name }}-prometheus-htpasswd
   configMaps:
     - serving-certs-ca-bundle
   secrets:
     - '{{ ansible_operator_meta.name }}-prometheus-proxy-tls'
     - '{{ ansible_operator_meta.name }}-session-secret'
-    - '{{ ansible_operator_meta.name }}-prometheus-htpasswd'
 {% if servicetelemetry_vars.backends.metrics.prometheus.storage.strategy == "persistent" %}
   storage:
     volumeClaimTemplate:

tests/smoketest/smoketest.sh

@@ -48,7 +48,7 @@ echo "*** [INFO] Getting ElasticSearch authentication password"
 ELASTICSEARCH_AUTH_PASS=$(oc get secret elasticsearch-es-elastic-user -ogo-template='{{ .data.elastic | base64decode }}')
 
 echo "*** [INFO] Getting Prometheus authentication password"
-PROMETHEUS_AUTH_PASS=$(oc get secret default-prometheus-htpasswd -ogo-template='{{ .data.password | base64decode }}')
+PROMETHEUS_AUTH_TOKEN=$(oc create token stf-prometheus-reader)
 
 echo "*** [INFO] Creating configmaps..."
 oc delete configmap/stf-smoketest-healthcheck-log configmap/stf-smoketest-collectd-config configmap/stf-smoketest-sensubility-config configmap/stf-smoketest-collectd-entrypoint-script configmap/stf-smoketest-ceilometer-publisher configmap/stf-smoketest-ceilometer-entrypoint-script job/stf-smoketest || true
@@ -76,7 +76,7 @@ oc wait --for=jsonpath='{.status.phase}'=Running pod/qdr-test
 echo "*** [INFO] Creating smoketest jobs..."
 oc delete job -l app=stf-smoketest
 for NAME in "${CLOUDNAMES[@]}"; do
-    oc create -f <(sed -e "s/<<CLOUDNAME>>/${NAME}/;s/<<ELASTICSEARCH_AUTH_PASS>>/${ELASTICSEARCH_AUTH_PASS}/;s/<<PROMETHEUS_AUTH_PASS>>/${PROMETHEUS_AUTH_PASS}/" ${REL}/smoketest_job.yaml.template)
+    oc create -f <(sed -e "s/<<CLOUDNAME>>/${NAME}/;s/<<ELASTICSEARCH_AUTH_PASS>>/${ELASTICSEARCH_AUTH_PASS}/;s/<<PROMETHEUS_AUTH_TOKEN>>/${PROMETHEUS_AUTH_TOKEN}/" ${REL}/smoketest_job.yaml.template)
 done
 
 echo "*** [INFO] Triggering an alertmanager notification..."

tests/smoketest/smoketest_ceilometer_entrypoint.sh

@@ -5,7 +5,7 @@ set +e
 PROMETHEUS=${PROMETHEUS:-"https://default-prometheus-proxy:9092"}
 ELASTICSEARCH=${ELASTICSEARCH:-"https://elasticsearch-es-http:9200"}
 ELASTICSEARCH_AUTH_PASS=${ELASTICSEARCH_AUTH_PASS:-""}
-PROMETHEUS_AUTH_PASS=${PROMETHEUS_AUTH_PASS:-""}
+PROMETHEUS_AUTH_TOKEN=${PROMETHEUS_AUTH_TOKEN:-""}
 CLOUDNAME=${CLOUDNAME:-"smoke1"}
 POD=$(hostname)
 
@@ -20,14 +20,14 @@ echo "*** [INFO] Sleeping for 30 seconds to produce all metrics and events"
 sleep 30
 
 echo "*** [INFO] List of metric names for debugging..."
-curl -sk -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/label/__name__/values" 2>&2 | tee /tmp/label_names
+curl -sk -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/label/__name__/values" 2>&2 | tee /tmp/label_names
 echo; echo
 
 # Checks that the metrics actually appear in prometheus
 echo "*** [INFO] Checking for recent image metrics..."
 
 echo "[DEBUG] Running the curl command to return a query"
-curl -k -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=ceilometer_image_size' 2>&1 | grep '"result":\[{"metric":{"__name__":"ceilometer_image_size"'
+curl -k -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=ceilometer_image_size' 2>&1 | grep '"result":\[{"metric":{"__name__":"ceilometer_image_size"'
 metrics_result=$?
 echo "[DEBUG] Set metrics_result to $metrics_result"
 

tests/smoketest/smoketest_collectd_entrypoint.sh

@@ -5,7 +5,7 @@ set +e
 PROMETHEUS=${PROMETHEUS:-"https://default-prometheus-proxy:9092"}
 ELASTICSEARCH=${ELASTICSEARCH:-"https://elasticsearch-es-http:9200"}
 ELASTICSEARCH_AUTH_PASS=${ELASTICSEARCH_AUTH_PASS:-""}
-PROMETHEUS_AUTH_PASS=${PROMETHEUS_AUTH_PASS:-""}
+PROMETHEUS_AUTH_TOKEN=${PROMETHEUS_AUTH_TOKEN:-""}
 CLOUDNAME=${CLOUDNAME:-"smoke1"}
 POD=$(hostname)
 
@@ -37,12 +37,12 @@ sleep 30
 
 
 echo "*** [INFO] List of metric names for debugging..."
-curl -k -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/label/__name__/values" 2>&2 | tee /tmp/label_names
+curl -k -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/label/__name__/values" 2>&2 | tee /tmp/label_names
 echo; echo
 
 # Checks that the metrics actually appear in prometheus
 echo "*** [INFO] Checking for recent CPU metrics..."
-curl -k -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=collectd_cpu_total{container="sg-core",plugin_instance="0",type_instance="user",service="default-cloud1-coll-meter",host="'"${POD}"'"}[1m]' 2>&2 | tee /tmp/query_output
+curl -k -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=collectd_cpu_total{container="sg-core",plugin_instance="0",type_instance="user",service="default-cloud1-coll-meter",host="'"${POD}"'"}[1m]' 2>&2 | tee /tmp/query_output
 echo; echo
 
 # The egrep exit code is the result of the test and becomes the container/pod/job exit code
@@ -53,7 +53,7 @@ echo; echo
 
 # Checks that the metrics actually appear in prometheus
 echo "*** [INFO] Checking for recent healthcheck metrics..."
-curl -k -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=sensubility_container_health_status{container="sg-core",service="default-cloud1-sens-meter",host="'"${POD}"'"}[1m]' 2>&2 | tee /tmp/query_output
+curl -k -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=sensubility_container_health_status{container="sg-core",service="default-cloud1-sens-meter",host="'"${POD}"'"}[1m]' 2>&2 | tee /tmp/query_output
 echo; echo
 
 # The egrep exit code is the result of the test and becomes the container/pod/job exit code

tests/smoketest/smoketest_job.yaml.template

@@ -22,8 +22,8 @@ spec:
           value: <<CLOUDNAME>>
         - name: ELASTICSEARCH_AUTH_PASS
           value: "<<ELASTICSEARCH_AUTH_PASS>>"
-        - name: PROMETHEUS_AUTH_PASS
-          value: "<<PROMETHEUS_AUTH_PASS>>"
+        - name: PROMETHEUS_AUTH_TOKEN
+          value: "<<PROMETHEUS_AUTH_TOKEN>>"
         - name: OBSERVABILITY_STRATEGY
           value: "<<OBSERVABILITY_STRATEGY>>"
         volumeMounts:
@@ -51,8 +51,8 @@ spec:
           value: <<CLOUDNAME>>
         - name: ELASTICSEARCH_AUTH_PASS
           value: "<<ELASTICSEARCH_AUTH_PASS>>"
-        - name: PROMETHEUS_AUTH_PASS
-          value: "<<PROMETHEUS_AUTH_PASS>>"
+        - name: PROMETHEUS_AUTH_TOKEN
+          value: "<<PROMETHEUS_AUTH_TOKEN>>"
         - name: OBSERVABILITY_STRATEGY
           value: "<<OBSERVABILITY_STRATEGY>>"
         volumeMounts: