Skip to content

Commit

Permalink
UTF-8 support in metric and label names
Browse files Browse the repository at this point in the history 
Signed-off-by: Owen Williams <[email protected]>
  • Loading branch information
@ywwg
ywwg committed Dec 4, 2023
1 parent 1d8c672 commit ed26db3
Show file tree
Hide file tree
Showing 18 changed files with 658 additions and 323 deletions.
13 changes: 10 additions & 3 deletions expfmt/decode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,16 @@ func NewDecoder(r io.Reader, format Format) Decoder {
return &textDecoder{r: r}
}

// WithUTF8Names enables support for UTF-8 metric and label names.
func (d *protoDecoder) WithUTF8Names(enable bool) Decoder {
d.utf8Names = enable
return d
}

// protoDecoder implements the Decoder interface for protocol buffers.
type protoDecoder struct {
r io.Reader
r io.Reader
utf8Names bool // if false, metric names will be constrained to MetricNameRE. Otherwise, strings are checked to be valid UTF8.
}

// Decode implements the Decoder interface.
Expand All @@ -90,7 +97,7 @@ func (d *protoDecoder) Decode(v *dto.MetricFamily) error {
if err != nil {
return err
}
if !model.IsValidMetricName(model.LabelValue(v.GetName())) {
if !model.IsValidMetricName(model.LabelValue(v.GetName()), d.utf8Names) {
return fmt.Errorf("invalid metric name %q", v.GetName())
}
for _, m := range v.GetMetric() {
Expand All @@ -104,7 +111,7 @@ func (d *protoDecoder) Decode(v *dto.MetricFamily) error {
if !model.LabelValue(l.GetValue()).IsValid() {
return fmt.Errorf("invalid label value %q", l.GetValue())
}
if !model.LabelName(l.GetName()).IsValid() {
if !model.LabelName(l.GetName()).IsValid(d.utf8Names) {
return fmt.Errorf("invalid label name %q", l.GetName())
}
}
Expand Down
56 changes: 50 additions & 6 deletions expfmt/decode_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package expfmt
import (
"bufio"
"io"
"math"
"net/http"
"reflect"
"sort"
Expand Down Expand Up @@ -104,9 +105,10 @@ func TestProtoDecoder(t *testing.T) {
var testTime = model.Now()

scenarios := []struct {
in string
expected model.Vector
fail bool
in string
expected model.Vector
legacyNameFail bool
fail bool
}{
{
in: "",
Expand Down Expand Up @@ -332,6 +334,30 @@ func TestProtoDecoder(t *testing.T) {
},
},
},
{
in: "\xa8\x01\n\ngauge.name\x12\x11gauge\ndoc\nstr\"ing\x18\x01\"T\n\x1b\n\x06name.1\x12\x11val with\nnew line\n*\n\x06name*2\x12 val with \\backslash and \"quotes\"\x12\t\t\x00\x00\x00\x00\x00\x00\xf0\x7f\"/\n\x10\n\x06name.1\x12\x06Björn\n\x10\n\x06name*2\x12\x06佖佥\x12\t\t\xd1\xcfD\xb9\xd0\x05\xc2H",
legacyNameFail: true,
expected: model.Vector{
&model.Sample{
Metric: model.Metric{
model.MetricNameLabel: "gauge.name",
"name.1": "val with\nnew line",
"name*2": "val with \\backslash and \"quotes\"",
},
Value: model.SampleValue(math.Inf(+1)),
Timestamp: testTime,
},
&model.Sample{
Metric: model.Metric{
model.MetricNameLabel: "gauge.name",
"name.1": "Björn",
"name*2": "佖佥",
},
Value: 3.14e42,
Timestamp: testTime,
},
},
},
}

for i, scenario := range scenarios {
Expand All @@ -349,6 +375,24 @@ func TestProtoDecoder(t *testing.T) {
if err == io.EOF {
break
}
if scenario.legacyNameFail {
if err == nil {
t.Fatal("Expected error when decoding without UTF-8 support enabled but got none")
}
dec := &SampleDecoder{
Dec: &protoDecoder{
r: strings.NewReader(scenario.in),
utf8Names: true,
},
Opts: &DecodeOptions{
Timestamp: testTime,
},
}
err = dec.Decode(&smpls)
if err == io.EOF {
break
}
}
if scenario.fail {
if err == nil {
t.Fatal("Expected error but got none")
Expand Down Expand Up @@ -435,7 +479,7 @@ func TestExtractSamples(t *testing.T) {
Help: proto.String("Help for foo."),
Type: dto.MetricType_COUNTER.Enum(),
Metric: []*dto.Metric{
&dto.Metric{
{
Counter: &dto.Counter{
Value: proto.Float64(4711),
},
Expand All @@ -447,7 +491,7 @@ func TestExtractSamples(t *testing.T) {
Help: proto.String("Help for bar."),
Type: dto.MetricType_GAUGE.Enum(),
Metric: []*dto.Metric{
&dto.Metric{
{
Gauge: &dto.Gauge{
Value: proto.Float64(3.14),
},
Expand All @@ -459,7 +503,7 @@ func TestExtractSamples(t *testing.T) {
Help: proto.String("Help for bad."),
Type: dto.MetricType(42).Enum(),
Metric: []*dto.Metric{
&dto.Metric{
{
Gauge: &dto.Gauge{
Value: proto.Float64(2.7),
},
Expand Down
80 changes: 54 additions & 26 deletions expfmt/openmetrics_create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int
if metricType == dto.MetricType_COUNTER && strings.HasSuffix(shortName, "_total") {
shortName = name[:len(name)-6]
}
// If the name does not satisfy the legacy validity check, we must quote it.
quotedName := shortName
if !model.IsValidMetricName(model.LabelValue(quotedName), false) {
quotedName = fmt.Sprintf(`"%s"`, quotedName)
}

// Comments, first HELP, then TYPE.
if in.Help != nil {
Expand All @@ -98,7 +103,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int
if err != nil {
return
}
n, err = w.WriteString(shortName)
n, err = w.WriteString(quotedName)
written += n
if err != nil {
return
Expand All @@ -124,7 +129,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int
if err != nil {
return
}
n, err = w.WriteString(shortName)
n, err = w.WriteString(quotedName)
written += n
if err != nil {
return
Expand Down Expand Up @@ -303,21 +308,9 @@ func writeOpenMetricsSample(
floatValue float64, intValue uint64, useIntValue bool,
exemplar *dto.Exemplar,
) (int, error) {
var written int
n, err := w.WriteString(name)
written += n
if err != nil {
return written, err
}
if suffix != "" {
n, err = w.WriteString(suffix)
written += n
if err != nil {
return written, err
}
}
n, err = writeOpenMetricsLabelPairs(
w, metric.Label, additionalLabelName, additionalLabelValue,
written := 0
n, err := writeOpenMetricsNameAndLabelPairs(
w, name+suffix, metric.Label, additionalLabelName, additionalLabelValue,
)
written += n
if err != nil {
Expand Down Expand Up @@ -365,27 +358,62 @@ func writeOpenMetricsSample(
return written, nil
}

// writeOpenMetricsLabelPairs works like writeOpenMetrics but formats the float
// writeOpenMetricsNameAndLabelPairs works like writeOpenMetrics but formats the float
// in OpenMetrics style.
func writeOpenMetricsLabelPairs(
func writeOpenMetricsNameAndLabelPairs(
w enhancedWriter,
name string,
in []*dto.LabelPair,
additionalLabelName string, additionalLabelValue float64,
) (int, error) {
if len(in) == 0 && additionalLabelName == "" {
return 0, nil
}
var (
written int
separator byte = '{'
written int
separator byte = '{'
metricInsideBraces = false
)

if name != "" {
// If the name does not pass the legacy validity check, we must put the
// metric name inside the braces, quoted.
if !model.IsValidMetricName(model.LabelValue(name), false) {
metricInsideBraces = true
err := w.WriteByte(separator)
written++
if err != nil {
return written, err
}
name = fmt.Sprintf(`"%s"`, name)
separator = ','
}
n, err := w.WriteString(name)
written += n
if err != nil {
return written, err
}
}

if len(in) == 0 && additionalLabelName == "" {
if metricInsideBraces {
err := w.WriteByte('}')
written++
if err != nil {
return written, err
}
}
return written, nil
}

for _, lp := range in {
err := w.WriteByte(separator)
written++
if err != nil {
return written, err
}
n, err := w.WriteString(lp.GetName())
labelName := lp.GetName()
if !model.IsValidMetricName(model.LabelValue(labelName), false) {
labelName = fmt.Sprintf(`"%s"`, labelName)
}
n, err := w.WriteString(labelName)
written += n
if err != nil {
return written, err
Expand Down Expand Up @@ -451,7 +479,7 @@ func writeExemplar(w enhancedWriter, e *dto.Exemplar) (int, error) {
if err != nil {
return written, err
}
n, err = writeOpenMetricsLabelPairs(w, e.Label, "", 0)
n, err = writeOpenMetricsNameAndLabelPairs(w, "", e.Label, "", 0)
written += n
if err != nil {
return written, err
Expand Down
Loading

0 comments on commit ed26db3

Please sign in to comment.