Merge pull request #4 from oracle-quickstart/v2.0.3

Release v2.0.3

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Change Log
 
+## 2.0.3 - 2022-04-20
+### Added
+- Added Prometheus-client Api based internal metrics functionality.
+### Bug fix
+- Fixing minimum required version for OCI Ruby SDK (oci) runtime dependency
+
 ## 2.0.2 - 2022-02-17
 ### Added
 - Added required ruby version in gemspec.

README.md

@@ -20,13 +20,15 @@ Refer [Prerequisites](https://docs.oracle.com/en/learn/oci_logging_analytics_flu
     <th>ruby</th>
     <th>rubyzip</th>
     <th>oci</th>
+    <th>prometheus-client</th>
   </tr>
   <tr>
     <td>>= 2.0.0</td>
     <td>>= 0.14.10, < 2 </td>
     <td>>= 2.6</td>
     <td>~> 2.3.2 </td>
-    <td>~> 2.13</td>
+    <td>~> 2.16</td>
+    <td>~> 2.1.0</td>
   </tr>
 </table>
 
@@ -81,12 +83,49 @@ Use filter plugin (record_transformer) to transform the input log events to add
 
    - Example configuration that can be used for monitoring [kafka log](examples/kafka.conf)
 
-
-
 ## Start Viewing the Logs in Logging Analytics
 
 Refer [Viewing the Logs in Logging Analytics](https://docs.oracle.com/en/learn/oci_logging_analytics_fluentd/#start-viewing-the-logs-in-logging-analytics)
 
+## Metrics
+
+The plugin emits following metrics in Prometheus format, which provides stats/insights about the data being collected and processed by the plugin. Refer [monitoring-prometheus](https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus) for details on how to expose these and other various Fluentd metrics to Prometheus (*If the requirement is to collect and monitor core Fluentd and this plugin metrics alone using Prometheus then Step1 and Step2 from the referred document can be skipped*).
+
+    Metric Name: oci_la_fluentd_output_plugin_records_received 
+    labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set]
+    Description: Number of records received by the OCI Logging Analytics Fluentd output plugin.
+    Type : Gauge
+
+    Metric Name: oci_la_fluentd_output_plugin_records_valid 
+    labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set]
+    Description: Number of valid records received by the OCI Logging Analytics Fluentd output plugin.
+    Type : Gauge 
+
+    Metric Name: oci_la_fluentd_output_plugin_records_invalid 
+    labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:reason]
+    Description: Number of invalid records received by the OCI Logging Analytics Fluentd output plugin. 
+    Type : Gauge
+
+    Metric Name: oci_la_fluentd_output_plugin_records_post_error 
+    labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:error_code, :reason]
+    Description: Number of records failed posting to OCI Logging Analytics by the Fluentd output plugin.
+    Type : Gauge
+        
+    Metric Name: oci_la_fluentd_output_plugin_records_post_success 
+    labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set]
+    Description: Number of records posted by the OCI Logging Analytics Fluentd output plugin. 
+    Type : Gauge  
+
+    Metric Name: oci_la_fluentd_output_plugin_chunk_time_to_receive
+    labels: [:tag]
+    Description: Average time taken by Fluentd to deliver the collected records from Input plugin to OCI Logging Analytics output plugin.
+    Type : Histogram  
+
+    Metric Name: oci_la_fluentd_output_plugin_chunk_time_to_post 
+    labels: [:oci_la_log_group_id]
+    Description: Average time taken for posting the received records to OCI Logging Analytics by the Fluentd output plugin.
+    Type : Histogram
+
 
 ## Changes
 

fluent-plugin-oci-logging-analytics.gemspec

@@ -6,7 +6,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-oci-logging-analytics"
-  spec.version = "2.0.2"
+  spec.version = "2.0.3"
   spec.authors = ["Oracle","OCI Observability: Logging Analytics"]
   spec.email   = ["[email protected]"]
 
@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
   spec.add_runtime_dependency 'rubyzip', '~> 2.3.2'
-  spec.add_runtime_dependency "oci", "~>2.13"
+  spec.add_runtime_dependency "oci", "~>2.16"
+  spec.add_runtime_dependency "prometheus-client", "~>2.1.0"
 
 end

lib/fluent/metrics/metricsLabels.rb

@@ -0,0 +1,13 @@
+class MetricsLabels
+  attr_accessor :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency
+  def initialize
+      @tag = nil
+      @logGroupId = nil
+      @logSourceName = nil
+      @logSet = nil
+      @invalid_reason = nil
+      @records_valid = 0
+      @records_per_tag = 0
+      @latency = 0
+    end
+end

lib/fluent/metrics/prometheusMetrics.rb

@@ -0,0 +1,41 @@
+require 'prometheus/client'
+require 'prometheus/client/registry'
+require 'prometheus/client/gauge'
+require 'prometheus/client/histogram'
+require 'singleton'
+
+class PrometheusMetrics
+  include Singleton
+  attr_accessor :records_received, :records_valid, :records_invalid, :records_error, :records_posted,
+                :bytes_received, :bytes_posted, :chunk_time_to_receive, :chunk_time_to_upload
+  def initialize
+    createMetrics
+    registerMetrics
+  end
+  def createMetrics
+    gauge = Prometheus::Client::Gauge
+    @records_received = gauge.new(:oci_la_fluentd_output_plugin_records_received, docstring: 'Number of records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
+    @records_valid = gauge.new(:oci_la_fluentd_output_plugin_records_valid, docstring: 'Number of valid records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
+    @records_invalid = gauge.new(:oci_la_fluentd_output_plugin_records_invalid, docstring: 'Number of invalid records received by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:reason])
+    @records_error = gauge.new(:oci_la_fluentd_output_plugin_records_post_error, docstring: 'Number of records failed posting to OCI Logging Analytics by the Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set,:error_code, :reason])
+    @records_posted = gauge.new(:oci_la_fluentd_output_plugin_records_post_success, docstring: 'Number of records posted by the OCI Logging Analytics Fluentd output plugin.', labels: [:tag,:oci_la_log_group_id,:oci_la_log_source_name,:oci_la_log_set])
+    #@bytes_received = gauge.new(:oci_la_bytes_received, docstring: '...', labels: [:tag])
+    #@bytes_posted = gauge.new(:oci_la_bytes_posted, docstring: '...', labels: [:oci_la_log_group_id])
+    histogram = Prometheus::Client::Histogram
+    @chunk_time_to_receive = histogram.new(:oci_la_fluentd_output_plugin_chunk_time_to_receive, docstring: 'Average time taken by Fluentd to deliver the collected records from Input plugin to OCI Logging Analytics output plugin.', labels: [:tag])
+    @chunk_time_to_upload = histogram.new(:oci_la_fluentd_output_plugin_chunk_time_to_post, docstring: 'Average time taken for posting the received records to OCI Logging Analytics by the Fluentd output plugin.', labels: [:oci_la_log_group_id])
+  end
+
+  def registerMetrics
+    registry = Prometheus::Client.registry
+    registry.register(@records_received) unless registry.exist?('oci_la_fluentd_output_plugin_records_received')
+    registry.register(@records_valid) unless registry.exist?('oci_la_fluentd_output_plugin_records_valid')
+    registry.register(@records_invalid) unless registry.exist?('oci_la_fluentd_output_plugin_records_invalid')
+    registry.register(@records_error) unless registry.exist?('oci_la_fluentd_output_plugin_records_post_error')
+    registry.register(@records_posted) unless registry.exist?('oci_la_fluentd_output_plugin_records_post_success')
+    #registry.register(@bytes_received) unless registry.exist?('oci_la_bytes_received')
+    #registry.register(@bytes_posted) unless registry.exist?('oci_la_bytes_valid')
+    registry.register(@chunk_time_to_receive) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_receive')
+    registry.register(@chunk_time_to_upload) unless registry.exist?('oci_la_fluentd_output_plugin_chunk_time_to_post')
+  end
+end