Merge pull request #310 from alphagov/more-quality-reporting

Improve search quality monitoring
alphagov · Aug 5, 2024 · b0cf6fa · b0cf6fa
2 parents be62f83 + a706fe1
commit b0cf6fa
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 11 deletions.
diff --git a/app/services/metrics/quality_monitoring.rb b/app/services/metrics/quality_monitoring.rb
@@ -11,6 +11,11 @@ def initialize(registry)
         docstring: "Quality monitoring failure count for Search API v2",
         labels: %i[dataset_type dataset_name],
       )
+      @quality_monitoring_total = registry.gauge(
+        :search_api_v2_quality_monitoring_total,
+        docstring: "Quality monitoring total count for Search API v2",
+        labels: %i[dataset_type dataset_name],
+      )
     end
 
     def record_score(dataset_type, dataset_name, score)
@@ -27,8 +32,15 @@ def record_failure_count(dataset_type, dataset_name, count)
       })
     end
 
+    def record_total_count(dataset_type, dataset_name, count)
+      quality_monitoring_total.set(count, labels: {
+        dataset_type: dataset_type.to_s,
+        dataset_name: dataset_name.to_s,
+      })
+    end
+
   private
 
-    attr_reader :quality_monitoring_score, :quality_monitoring_failures
+    attr_reader :quality_monitoring_score, :quality_monitoring_failures, :quality_monitoring_total
   end
 end
diff --git a/app/services/quality_monitoring/runner.rb b/app/services/quality_monitoring/runner.rb
@@ -57,7 +57,6 @@ def run
           mean_score,
         ),
       )
-      metric_collector.record_score(type, dataset_name, mean_score) if metric_collector
 
       if failure_details.any?
         Rails.logger.warn(
@@ -70,19 +69,12 @@ def run
             failure_details.join("\n"),
           ),
         )
-
-        err = FailuresEncountered.new(
-          "Quality monitoring: #{failure_details.size} failures encountered " \
-          "for #{type} dataset #{dataset_name}",
-        )
-        GovukError.notify(
-          err,
-          extra: { dataset_name:, type:, failure_details: failure_details.join("\n") },
-        )
       end
 
       if metric_collector
+        metric_collector.record_score(type, dataset_name, mean_score)
         metric_collector.record_failure_count(type, dataset_name, failure_details.size)
+        metric_collector.record_total_count(type, dataset_name, scores.size)
       end
     end
 

diff --git a/spec/services/metrics/quality_monitoring_spec.rb b/spec/services/metrics/quality_monitoring_spec.rb
@@ -4,12 +4,15 @@
   let(:registry) { double("registry") }
   let(:score_gauge) { double("score gauge") }
   let(:failure_gauge) { double("failure gauge") }
+  let(:total_gauge) { double("total gauge") }
 
   before do
     allow(registry).to receive(:gauge)
       .with(:search_api_v2_quality_monitoring_score, anything).and_return(score_gauge)
     allow(registry).to receive(:gauge)
       .with(:search_api_v2_quality_monitoring_failures, anything).and_return(failure_gauge)
+    allow(registry).to receive(:gauge)
+      .with(:search_api_v2_quality_monitoring_total, anything).and_return(total_gauge)
 
     allow(score_gauge).to receive(:set)
   end
@@ -31,4 +34,13 @@
       quality_monitoring.record_failure_count(:foo, "bar", 50)
     end
   end
+
+  describe "#record_total_count" do
+    it "records the total count" do
+      expect(total_gauge).to receive(:set)
+        .with(100, labels: { dataset_type: "foo", dataset_name: "bar" })
+
+      quality_monitoring.record_total_count(:foo, "bar", 100)
+    end
+  end
 end