From 1cb3bf1426be33fb66d0ab66d37cddb30c187e05 Mon Sep 17 00:00:00 2001 From: Arthur Silva Sens Date: Mon, 17 Jul 2023 17:59:23 -0300 Subject: [PATCH 1/4] expfmt/openmetrics: Write created timestamps for counters, summaries and histograms Signed-off-by: Arthur Silva Sens --- expfmt/openmetrics_create.go | 81 +++++++++++++++++++++++++++++-- expfmt/openmetrics_create_test.go | 42 ++++++++++++++-- 2 files changed, 115 insertions(+), 8 deletions(-) diff --git a/expfmt/openmetrics_create.go b/expfmt/openmetrics_create.go index 5622578e..f67a4311 100644 --- a/expfmt/openmetrics_create.go +++ b/expfmt/openmetrics_create.go @@ -23,10 +23,23 @@ import ( "strings" "github.com/prometheus/common/model" + "google.golang.org/protobuf/types/known/timestamppb" dto "github.com/prometheus/client_model/go" ) +type toOpenMetrics struct { + withCreatedLines bool +} + +type ToOpenMetricsOption func(*toOpenMetrics) + +func WithCreatedLines() ToOpenMetricsOption { + return func(t *toOpenMetrics) { + t.withCreatedLines = true + } +} + // MetricFamilyToOpenMetrics converts a MetricFamily proto message into the // OpenMetrics text format and writes the resulting lines to 'out'. It returns // the number of bytes written and any error encountered. The output will have @@ -64,15 +77,20 @@ import ( // its type will be set to `unknown` in that case to avoid invalid OpenMetrics // output. // -// - No support for the following (optional) features: `# UNIT` line, `_created` -// line, info type, stateset type, gaugehistogram type. +// - No support for the following (optional) features: `# UNIT` line, info type, +// stateset type, gaugehistogram type. // // - The size of exemplar labels is not checked (i.e. it's possible to create // exemplars that are larger than allowed by the OpenMetrics specification). // // - The value of Counters is not checked. (OpenMetrics doesn't allow counters // with a `NaN` value.) -func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int, err error) { +func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily, options ...ToOpenMetricsOption) (written int, err error) { + toOM := toOpenMetrics{} + for _, option := range options { + option(&toOM) + } + name := in.GetName() if name == "" { return 0, fmt.Errorf("MetricFamily has no name: %s", in) @@ -164,6 +182,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int return } + var createdTsBytesWritten int // Finally the samples, one line for each. for _, metric := range in.Metric { switch metricType { @@ -181,6 +200,10 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int metric.Counter.GetValue(), 0, false, metric.Counter.Exemplar, ) + if toOM.withCreatedLines && metric.Counter.CreatedTimestamp != nil { + createdTsBytesWritten, err = writeOpenMetricsCreated(w, name, "_total", metric, "", 0, metric.Counter.GetCreatedTimestamp()) + n += createdTsBytesWritten + } case dto.MetricType_GAUGE: if metric.Gauge == nil { return written, fmt.Errorf( @@ -235,6 +258,10 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int 0, metric.Summary.GetSampleCount(), true, nil, ) + if toOM.withCreatedLines && metric.Summary.CreatedTimestamp != nil { + createdTsBytesWritten, err = writeOpenMetricsCreated(w, name, "", metric, "", 0, metric.Summary.GetCreatedTimestamp()) + n += createdTsBytesWritten + } case dto.MetricType_HISTOGRAM: if metric.Histogram == nil { return written, fmt.Errorf( @@ -283,6 +310,10 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int 0, metric.Histogram.GetSampleCount(), true, nil, ) + if toOM.withCreatedLines && metric.Histogram.CreatedTimestamp != nil { + createdTsBytesWritten, err = writeOpenMetricsCreated(w, name, "", metric, "", 0, metric.Histogram.GetCreatedTimestamp()) + n += createdTsBytesWritten + } default: return written, fmt.Errorf( "unexpected type in metric %s %s", name, metric, @@ -473,6 +504,50 @@ func writeOpenMetricsNameAndLabelPairs( return written, nil } +// writeOpenMetricsCreated writes the created timestamp for a single time series +// following OpenMetrics text format to w, given the metric name, the metric proto +// message itself, optionally a suffix to be removed, e.g. '_total' for counters, +// an additional label name with a float64 value (use empty string as label name if +// not required) and the timestamp that represents the created timestamp. +// The function returns the number of bytes written and any error encountered. +func writeOpenMetricsCreated(w enhancedWriter, + name, suffixToTrim string, metric *dto.Metric, + additionalLabelName string, additionalLabelValue float64, + createdTimestamp *timestamppb.Timestamp, +) (int, error) { + written := 0 + n, err := writeOpenMetricsNameAndLabelPairs( + w, strings.TrimSuffix(name, suffixToTrim)+"_created", metric.Label, additionalLabelName, additionalLabelValue, + ) + written += n + if err != nil { + return written, err + } + + err = w.WriteByte(' ') + written++ + if err != nil { + return written, err + } + + ts := createdTimestamp.AsTime() + // TODO(beorn7): Format this directly from components of ts to + // avoid overflow/underflow and precision issues of the float + // conversion. + n, err = writeOpenMetricsFloat(w, float64(ts.UnixNano())/1e9) + written += n + if err != nil { + return written, err + } + + err = w.WriteByte('\n') + written++ + if err != nil { + return written, err + } + return written, nil +} + // writeExemplar writes the provided exemplar in OpenMetrics format to w. The // function returns the number of bytes written and any error encountered. func writeExemplar(w enhancedWriter, e *dto.Exemplar) (int, error) { diff --git a/expfmt/openmetrics_create_test.go b/expfmt/openmetrics_create_test.go index 8601834a..bf93b4e1 100644 --- a/expfmt/openmetrics_create_test.go +++ b/expfmt/openmetrics_create_test.go @@ -41,8 +41,9 @@ func TestCreateOpenMetrics(t *testing.T) { }() scenarios := []struct { - in *dto.MetricFamily - out string + in *dto.MetricFamily + options []ToOpenMetricsOption + out string }{ // 0: Counter, timestamp given, no _total suffix. { @@ -306,6 +307,7 @@ unknown_name{name_1="value 1"} -1.23e-45 Value: proto.Float64(0), }, }, + CreatedTimestamp: openMetricsTimestamp, }, }, { @@ -336,10 +338,12 @@ unknown_name{name_1="value 1"} -1.23e-45 Value: proto.Float64(3), }, }, + CreatedTimestamp: openMetricsTimestamp, }, }, }, }, + options: []ToOpenMetricsOption{WithCreatedLines()}, out: `# HELP summary_name summary docstring # TYPE summary_name summary summary_name{quantile="0.5"} -1.23 @@ -347,11 +351,13 @@ summary_name{quantile="0.9"} 0.2342354 summary_name{quantile="0.99"} 0.0 summary_name_sum -3.4567 summary_name_count 42 +summary_name_created 12345.6 summary_name{name_1="value 1",name_2="value 2",quantile="0.5"} 1.0 summary_name{name_1="value 1",name_2="value 2",quantile="0.9"} 2.0 summary_name{name_1="value 1",name_2="value 2",quantile="0.99"} 3.0 summary_name_sum{name_1="value 1",name_2="value 2"} 2010.1971 summary_name_count{name_1="value 1",name_2="value 2"} 4711 +summary_name_created{name_1="value 1",name_2="value 2"} 12345.6 `, }, // 7: Histogram @@ -387,10 +393,12 @@ summary_name_count{name_1="value 1",name_2="value 2"} 4711 CumulativeCount: proto.Uint64(2693), }, }, + CreatedTimestamp: openMetricsTimestamp, }, }, }, }, + options: []ToOpenMetricsOption{WithCreatedLines()}, out: `# HELP request_duration_microseconds The response latency. # TYPE request_duration_microseconds histogram request_duration_microseconds_bucket{le="100.0"} 123 @@ -400,6 +408,7 @@ request_duration_microseconds_bucket{le="172.8"} 1524 request_duration_microseconds_bucket{le="+Inf"} 2693 request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 +request_duration_microseconds_created 12345.6 `, }, // 8: Histogram with missing +Inf bucket. @@ -522,7 +531,30 @@ request_duration_microseconds_count 2693 Metric: []*dto.Metric{ { Counter: &dto.Counter{ - Value: proto.Float64(42), + Value: proto.Float64(42), + CreatedTimestamp: openMetricsTimestamp, + }, + }, + }, + }, + options: []ToOpenMetricsOption{WithCreatedLines()}, + out: `# HELP foos Number of foos. +# TYPE foos counter +foos_total 42.0 +foos_created 12345.6 +`, + }, + // 11: Simple Counter without created line. + { + in: &dto.MetricFamily{ + Name: proto.String("foos_total"), + Help: proto.String("Number of foos."), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + &dto.Metric{ + Counter: &dto.Counter{ + Value: proto.Float64(42), + CreatedTimestamp: openMetricsTimestamp, }, }, }, @@ -532,7 +564,7 @@ request_duration_microseconds_count 2693 foos_total 42.0 `, }, - // 11: No metric. + // 12: No metric. { in: &dto.MetricFamily{ Name: proto.String("name_total"), @@ -548,7 +580,7 @@ foos_total 42.0 for i, scenario := range scenarios { out := bytes.NewBuffer(make([]byte, 0, len(scenario.out))) - n, err := MetricFamilyToOpenMetrics(out, scenario.in) + n, err := MetricFamilyToOpenMetrics(out, scenario.in, scenario.options...) if err != nil { t.Errorf("%d. error: %s", i, err) continue From 38e075889e7902a89ca0c8a2053188db1a8fdca8 Mon Sep 17 00:00:00 2001 From: Arthur Silva Sens Date: Tue, 12 Dec 2023 19:24:08 -0300 Subject: [PATCH 2/4] expfmt/encoder: Allow opt-in for OM created lines Signed-off-by: Arthur Silva Sens --- expfmt/encode.go | 10 ++++++++-- expfmt/openmetrics_create.go | 22 ++++++++++++++++------ expfmt/openmetrics_create_test.go | 8 ++++---- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/expfmt/encode.go b/expfmt/encode.go index 97ee673d..97ba99ff 100644 --- a/expfmt/encode.go +++ b/expfmt/encode.go @@ -139,7 +139,13 @@ func NegotiateIncludingOpenMetrics(h http.Header) Format { // interface is kept for backwards compatibility. // In cases where the Format does not allow for UTF-8 names, the global // NameEscapingScheme will be applied. -func NewEncoder(w io.Writer, format Format) Encoder { +// +// NewEncoder can be called with additional options to customize the OpenMetrics text output. +// For example: +// NewEncoder(w, FmtOpenMetrics_1_0_0, WithCreatedLines()) +// +// Extra options are ignored for all other formats. +func NewEncoder(w io.Writer, format Format, options ...EncoderOption) Encoder { escapingScheme := format.ToEscapingScheme() switch format.FormatType() { @@ -178,7 +184,7 @@ func NewEncoder(w io.Writer, format Format) Encoder { case TypeOpenMetrics: return encoderCloser{ encode: func(v *dto.MetricFamily) error { - _, err := MetricFamilyToOpenMetrics(w, model.EscapeMetricFamily(v, escapingScheme)) + _, err := MetricFamilyToOpenMetrics(w, model.EscapeMetricFamily(v, escapingScheme), options...) return err }, close: func() error { diff --git a/expfmt/openmetrics_create.go b/expfmt/openmetrics_create.go index f67a4311..3c5a22de 100644 --- a/expfmt/openmetrics_create.go +++ b/expfmt/openmetrics_create.go @@ -28,14 +28,24 @@ import ( dto "github.com/prometheus/client_model/go" ) -type toOpenMetrics struct { +type encoderOption struct { withCreatedLines bool } -type ToOpenMetricsOption func(*toOpenMetrics) +type EncoderOption func(*encoderOption) -func WithCreatedLines() ToOpenMetricsOption { - return func(t *toOpenMetrics) { +// WithCreatedLines is an EncoderOption that configures the OpenMetrics encoder +// to include _created lines (See +// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#counter-1). +// Created timestamps can improve the accuracy of series reset detection, but +// come with a bandwidth cost. +// +// At the time of writing, created timestamp ingestion is still experimental in +// Prometheus and need to be enabled with the feature-flag +// `--feature-flag=created-timestamp-zero-ingestion`, and breaking changes are +// still possible. Therefore, it is recommended to use this feature with caution. +func WithCreatedLines() EncoderOption { + return func(t *encoderOption) { t.withCreatedLines = true } } @@ -85,8 +95,8 @@ func WithCreatedLines() ToOpenMetricsOption { // // - The value of Counters is not checked. (OpenMetrics doesn't allow counters // with a `NaN` value.) -func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily, options ...ToOpenMetricsOption) (written int, err error) { - toOM := toOpenMetrics{} +func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily, options ...EncoderOption) (written int, err error) { + toOM := encoderOption{} for _, option := range options { option(&toOM) } diff --git a/expfmt/openmetrics_create_test.go b/expfmt/openmetrics_create_test.go index bf93b4e1..e7a81a59 100644 --- a/expfmt/openmetrics_create_test.go +++ b/expfmt/openmetrics_create_test.go @@ -42,7 +42,7 @@ func TestCreateOpenMetrics(t *testing.T) { scenarios := []struct { in *dto.MetricFamily - options []ToOpenMetricsOption + options []EncoderOption out string }{ // 0: Counter, timestamp given, no _total suffix. @@ -343,7 +343,7 @@ unknown_name{name_1="value 1"} -1.23e-45 }, }, }, - options: []ToOpenMetricsOption{WithCreatedLines()}, + options: []EncoderOption{WithCreatedLines()}, out: `# HELP summary_name summary docstring # TYPE summary_name summary summary_name{quantile="0.5"} -1.23 @@ -398,7 +398,7 @@ summary_name_created{name_1="value 1",name_2="value 2"} 12345.6 }, }, }, - options: []ToOpenMetricsOption{WithCreatedLines()}, + options: []EncoderOption{WithCreatedLines()}, out: `# HELP request_duration_microseconds The response latency. # TYPE request_duration_microseconds histogram request_duration_microseconds_bucket{le="100.0"} 123 @@ -537,7 +537,7 @@ request_duration_microseconds_count 2693 }, }, }, - options: []ToOpenMetricsOption{WithCreatedLines()}, + options: []EncoderOption{WithCreatedLines()}, out: `# HELP foos Number of foos. # TYPE foos counter foos_total 42.0 From fb888460081ca3d4469014365728ce6c6d6b761a Mon Sep 17 00:00:00 2001 From: Arthur Silva Sens Date: Fri, 2 Feb 2024 11:27:09 -0300 Subject: [PATCH 3/4] Fix linting issues Signed-off-by: Arthur Silva Sens --- expfmt/openmetrics_create.go | 3 ++- expfmt/openmetrics_create_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/expfmt/openmetrics_create.go b/expfmt/openmetrics_create.go index 3c5a22de..9017bcca 100644 --- a/expfmt/openmetrics_create.go +++ b/expfmt/openmetrics_create.go @@ -22,9 +22,10 @@ import ( "strconv" "strings" - "github.com/prometheus/common/model" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/prometheus/common/model" + dto "github.com/prometheus/client_model/go" ) diff --git a/expfmt/openmetrics_create_test.go b/expfmt/openmetrics_create_test.go index e7a81a59..e82589b8 100644 --- a/expfmt/openmetrics_create_test.go +++ b/expfmt/openmetrics_create_test.go @@ -551,7 +551,7 @@ foos_created 12345.6 Help: proto.String("Number of foos."), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Counter: &dto.Counter{ Value: proto.Float64(42), CreatedTimestamp: openMetricsTimestamp, From 1727e21659783ceeb1251e660854b50b1f0e85c1 Mon Sep 17 00:00:00 2001 From: Arthur Silva Sens Date: Tue, 27 Feb 2024 09:20:25 -0300 Subject: [PATCH 4/4] remove unnecessary allocation Signed-off-by: Arthur Silva Sens --- expfmt/openmetrics_create.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/expfmt/openmetrics_create.go b/expfmt/openmetrics_create.go index 9017bcca..03c9e27c 100644 --- a/expfmt/openmetrics_create.go +++ b/expfmt/openmetrics_create.go @@ -541,11 +541,10 @@ func writeOpenMetricsCreated(w enhancedWriter, return written, err } - ts := createdTimestamp.AsTime() // TODO(beorn7): Format this directly from components of ts to // avoid overflow/underflow and precision issues of the float // conversion. - n, err = writeOpenMetricsFloat(w, float64(ts.UnixNano())/1e9) + n, err = writeOpenMetricsFloat(w, float64(createdTimestamp.AsTime().UnixNano())/1e9) written += n if err != nil { return written, err