Use OAuth token instead of http basic auth for prometheus access

This change is required in order to match changes in STO[1] where we will stop
allowing http basic auth for access from grafana to prometheus, and use an
oauth token from a new restricted service account instead.

[1] infrawatch/service-telemetry-operator#549

roles/client_side_tests/tasks/get_prom_info.yml

@@ -1,14 +1,14 @@
 ---
-- name: "Get the default-prometheus-htpasswd secret"
+- name: "Generate an access token for prometheus"
   ansible.builtin.shell:
     cmd: |
-      oc  get secret default-prometheus-htpasswd -ojson | jq '.data.auth, .data.password' |  sed 's/"//g'
-  register: prom_secret
+      oc create token stf-prometheus-reader
+  register: prom_token_out
   changed_when: false
 
 - name: "Show the prom_secret value"
   ansible.builtin.debug:
-    var: prom_secret | string
+    var: prom_token_out | string
 
 - name: "Get the prom URL"
   ansible.builtin.shell:
@@ -23,16 +23,5 @@
 
 - name: "Get the prom creds from the secret"
   ansible.builtin.set_fact:
-    prom_user_decoded: "{{ (prom_secret.stdout_lines[0] | b64decode) }}"
-    prom_pass: "{{ prom_secret.stdout_lines[1] | b64decode }}"
+    prom_token: "{{ prom_token_out.stdout }}"
     prom_url: "{{ prom_route.stdout }}"
-
-- name: "Show the prom_user value"
-  ansible.builtin.debug:
-    var: prom_user
-
-- name: "Fetch user"
-  ansible.builtin.set_fact:
-    prom_user: "{{ prom_user_decoded.split(':')[0] }}"
-  when: prom_user_decoded is defined
-

roles/client_side_tests/tasks/test_e2e.yml

@@ -7,7 +7,7 @@
 #     Description: Query Prometheus for collectd_cpu_percent metrics and save the output into the file
   ansible.builtin.shell:
     cmd: >-
-      /usr/bin/curl -k -u "{{ prom_user }}:{{ prom_pass }}" \
+      /usr/bin/curl -k -H "Authorization: Bearer {{prom_token}}" \
         -g https://{{ prom_url }}/api/v1/query? \
         --data-urlencode 'query=collectd_cpu_percent {plugin_instance="0"}[1m]' \
         --output /tmp/query_collectd_cpu_percent
@@ -20,7 +20,7 @@
 #     Description: Query Prometheus for ceph_ceph_bytes metrics and save the output into the file
   ansible.builtin.shell:
     cmd: >-
-      /usr/bin/curl -k -u "{{ prom_user }}:{{ prom_pass }}" \
+      /usr/bin/curl -k -H "Authorization: Bearer {{prom_token}}" \
         -g https://{{ prom_url }}/api/v1/query? \
         --data-urlencode 'query=collectd_ceph_ceph_bytes {plugin_instance="ceph-osd.1"}[1m]' \
         --output /tmp/query_ceph_ceph_bytes
@@ -33,7 +33,7 @@
 #     Description: Query Prometheus for collectd_interface_if_packets_tx_total metrics and save the output into the file
   ansible.builtin.shell:
     cmd: >-
-      /usr/bin/curl -k -u "{{ prom_user }}:{{ prom_pass }}" \
+      /usr/bin/curl -k -H "Authorization: Bearer {{prom_token}}" \
         -g https://{{ prom_url }}/api/v1/query? \
         --data-urlencode 'query=collectd_interface_if_packets_tx_total {type_instance="base"}[1m]' \
         --output /tmp/query_collectd_interface_tx_total
@@ -46,7 +46,7 @@
 #     Description: Query Prometheus for collectd_memory metrics and save the output into the file
   ansible.builtin.shell:
     cmd: >-
-      /usr/bin/curl -k -u "{{ prom_user }}:{{ prom_pass }}" \
+      /usr/bin/curl -k -H "Authorization: Bearer {{prom_token}}" \
         -g https://{{ prom_url }}/api/v1/query? \
         --data-urlencode 'query=collectd_memory {plugin_instance="base"}[1m]' \
         --output /tmp/query_collectd_memory
@@ -59,7 +59,7 @@
 #     Description: Query Prometheus for collectd_load_longterm metrics and save the output into the file
   ansible.builtin.shell:
     cmd: >-
-      /usr/bin/curl -k -u "{{ prom_user }}:{{ prom_pass }}" \
+      /usr/bin/curl -k -H "Authorization: Bearer {{prom_token}}" \
         -g https://{{ prom_url }}/api/v1/query? \
         --data-urlencode 'query=collectd_load_longterm {plugin_instance="base"}[1m]' \
         --output /tmp/query_load_longterm

roles/test_alerts/tasks/test_create_an_alert.yml

@@ -31,7 +31,7 @@
 - name: "Check that the alert was created"
   ansible.builtin.command:
     cmd: |
-      curl -k --user "{{ prom_user }}:{{ prom_pass }}" https://{{ prom_url }}/api/v1/rules
+      curl -k -H "Authorization: Bearer {{prom_token}}" https://{{ prom_url }}/api/v1/rules
   register: cmd_output
   changed_when: false
   failed_when: cmd_output.rc != 0

roles/test_alerts/tasks/test_creating_a_standard_alert_route_in_alert_manager.yml

@@ -54,7 +54,7 @@
 - name: "Verify that the alert is firing in Prometheus"
   ansible.builtin.shell:
     cmd: >-
-      /usr/bin/curl -k -u "{{ prom_user }}:{{ prom_pass }}" \
+      /usr/bin/curl -k -H "Authorization: Bearer {{prom_token}}" \
         -g https://{{ prom_url }}/api/v1/alerts \
         | grep 'firing' | grep 'Collectd metrics receive rate is zero' | wc -l
   register: cmd_output

roles/test_sensubility/tasks/test_health_status.yml

@@ -47,7 +47,7 @@
     Description: Check that health status of container changed to 0
   ansible.builtin.shell:
     cmd: >-
-      /usr/bin/curl -k -u "{{ prom_user }}:{{ prom_pass }}" \
+      /usr/bin/curl -k -H "Authorization: Bearer {{prom_token}}" \
         -g https://{{ prom_url }}/api/v1/query? \
         --data-urlencode 'query=last_over_time(sensubility_container_health_status{process="logrotate_crond",host="{{ groups['overcloud_nodes'][0] }}"}[10m])' \
         | grep -oP '(?<="value":).*' | awk -F, '{ print $2 }' | grep -o '[0-9]\+' | grep 0 | wc -l
@@ -75,7 +75,7 @@
     Description: Check that health status of container changed to 1
   ansible.builtin.shell:
     cmd: >-
-      /usr/bin/curl -k -u "{{ prom_user }}:{{ prom_pass }}" \
+      /usr/bin/curl -k -H "Authorization: Bearer {{prom_token}}" \
         -g https://{{ prom_url }}/api/v1/query? \
         --data-urlencode 'query=last_over_time(sensubility_container_health_status{process="logrotate_crond",host="{{ groups['overcloud_nodes'][0] }}"}[10m])' \
         | grep -oP '(?<="value":).*' | awk -F, '{ print $2 }' |  grep -o '[0-9]\+' | grep 1 | wc -l