From 86c32bc703169dc840427a4a6ae5aeb3343c6f21 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Thu, 3 Oct 2024 15:43:43 +0530 Subject: [PATCH 01/45] Test cases for Server Metrics --- .../configs/server.yml | 2 +- .../metrics/PinotJMXToPromMetricsTest.java | 233 +++++++ .../metrics/ServerJMXToPromMetricsTest.java | 579 ++++++++++++++++++ 3 files changed, 813 insertions(+), 1 deletion(-) create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml index 780c9fffc584..f657a7956e17 100644 --- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml +++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml @@ -94,7 +94,7 @@ rules: - pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<>(\\w+)" name: "pinot_server_realtime_consumptionExceptions_$1" cache: true -- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<>(\\w+)" +- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<>(\\w+)" name: "pinot_server_$7_$8" cache: true labels: diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java new file mode 100644 index 000000000000..b5cd6a96e5fc --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pinot.common.metrics; + +import com.google.common.base.Objects; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.pinot.common.utils.SimpleHttpResponse; +import org.apache.pinot.common.utils.http.HttpClient; +import org.apache.pinot.spi.config.table.TableType; +import org.apache.pinot.spi.utils.builder.TableNameBuilder; +import org.testng.Assert; + + +public class PinotJMXToPromMetricsTest { + + protected HttpClient _httpClient; + + protected static final List METER_TYPES = + List.of("Count", "FiveMinuteRate", "MeanRate", "OneMinuteRate", "FifteenMinuteRate"); + + protected static final List TIMER_TYPES = + List.of("Count", "FiveMinuteRate", "Max", "999thPercentile", "95thPercentile", "75thPercentile", "98thPercentile", + "OneMinuteRate", "50thPercentile", "99thPercentile", "FifteenMinuteRate", "Mean", "StdDev", "MeanRate", + "Min"); + + protected static final String RAW_TABLE_NAME = "myTable"; + protected static final String TABLE_NAME_WITH_TYPE = + TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(RAW_TABLE_NAME); + + protected static final String KAFKA_TOPIC = "myTopic"; + protected static final String PARTITION_GROUP_ID = "partitionGroupId"; + protected static final String CLIENT_ID = + String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC, PARTITION_GROUP_ID); + protected static final String TABLE_STREAM_NAME = String.format("%s_%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC); + + protected static final List EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE = + List.of("table", "myTable", "tableType", "REALTIME"); + + protected static final List EXPORTED_LABELS_FOR_CLIENT_ID = + List.of("partition", PARTITION_GROUP_ID, "table", RAW_TABLE_NAME, "tableType", TableType.REALTIME.toString(), + "topic", KAFKA_TOPIC); + + protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) + throws IOException, URISyntaxException { + List promMetrics = + parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + Assert.assertTrue(promMetrics.contains( + ServerJMXToPromMetricsTest.PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value"))); + } + + protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, List labels, + String exportedMetricPrefix) + throws IOException, URISyntaxException { + List promMetrics = + parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withNameAndLabels( + exportedMetricPrefix + exportedGaugePrefix + "_" + "Value", labels))); + } + + protected void assertTimerExportedCorrectly(String exportedTimerPrefix, String exportedMetricPrefix) + throws IOException, URISyntaxException { + List promMetrics = + parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + for (String meterType : TIMER_TYPES) { + Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withName( + exportedMetricPrefix + exportedTimerPrefix + "_" + meterType))); + } + } + + protected void assertTimerExportedCorrectly(String exportedTimerPrefix, List labels, + String exportedMetricPrefix) + throws IOException, URISyntaxException { + List promMetrics = + parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + for (String meterType : METER_TYPES) { + Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withNameAndLabels( + exportedMetricPrefix + exportedTimerPrefix + "_" + meterType, labels))); + } + } + + protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String exportedMetricPrefix) + throws IOException, URISyntaxException { + List promMetrics = + parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + for (String meterType : METER_TYPES) { + Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withName( + exportedMetricPrefix + exportedMeterPrefix + "_" + meterType))); + } + } + + protected void assertMeterExportedCorrectly(String exportedMeterPrefix, List labels, + String exportedMetricPrefix) + throws IOException, URISyntaxException { + List promMetrics = + parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + for (String meterType : METER_TYPES) { + Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withNameAndLabels( + exportedMetricPrefix + exportedMeterPrefix + "_" + meterType, labels))); + } + } + + protected List parseExportedPromMetrics(String response) + throws IOException { + + List exportedPromMetrics = new ArrayList<>(); + + BufferedReader reader = new BufferedReader(new StringReader(response)); + + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("pinot_")) { + exportedPromMetrics.add(ServerJMXToPromMetricsTest.PromMetric.fromExportedMetric(line)); + } + } + reader.close(); + return exportedPromMetrics; + } + + protected SimpleHttpResponse getExportedPromMetrics() + throws IOException, URISyntaxException { + return _httpClient.sendGetRequest(new URI("http://localhost:9021/metrics")); + } + + public static class PromMetric { + private final String _metricName; + private final Map _labels; + + public String getMetricName() { + return _metricName; + } + + public Map getLabels() { + return _labels; + } + + private PromMetric(String metricName, Map labels) { + _metricName = metricName; + _labels = labels; + } + + public static PromMetric fromExportedMetric(String exportedMetric) { + int spaceIndex = exportedMetric.indexOf(' '); + String metricWithoutVal = exportedMetric.substring(0, spaceIndex); + int braceIndex = metricWithoutVal.indexOf('{'); + + if (braceIndex != -1) { + String metricName = metricWithoutVal.substring(0, braceIndex); + String labelsString = metricWithoutVal.substring(braceIndex + 1, metricWithoutVal.lastIndexOf('}')); + Map labels = parseLabels(labelsString); + return new PromMetric(metricName, labels); + } else { + return new PromMetric(metricWithoutVal, new LinkedHashMap<>()); + } + } + + private static Map parseLabels(String labelsString) { + return labelsString.isEmpty() ? new LinkedHashMap<>() + : java.util.Arrays.stream(labelsString.split(",")).map(kvPair -> kvPair.split("=")) + .collect(Collectors.toMap(kv -> kv[0], kv -> removeQuotes(kv[1]), (v1, v2) -> v2, LinkedHashMap::new)); + } + + private static String removeQuotes(String value) { + return value.startsWith("\"") ? value.substring(1, value.length() - 1) : value; + } + + public static PromMetric withName(String metricName) { + return new PromMetric(metricName, new LinkedHashMap<>()); + } + + public static PromMetric withNameAndLabels(String metricName, List labels) { + Map labelMap = new LinkedHashMap<>(); + for (int i = 0; i < labels.size(); i += 2) { + labelMap.put(labels.get(i), labels.get(i + 1)); + } + return new PromMetric(metricName, labelMap); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PromMetric that = (PromMetric) o; + return Objects.equal(_metricName, that._metricName) && Objects.equal(_labels, that._labels); + } + + @Override + public int hashCode() { + return Objects.hashCode(_metricName, _labels); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(_metricName); + if (!_labels.isEmpty()) { + sb.append('{'); + sb.append(_labels.entrySet().stream().map(e -> e.getKey() + "=\"" + e.getValue() + "\"") + .collect(Collectors.joining(","))); + sb.append('}'); + } + return sb.toString(); + } + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java new file mode 100644 index 000000000000..0d6a4beeb549 --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -0,0 +1,579 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pinot.common.metrics; + +import com.yammer.metrics.core.MetricsRegistry; +import com.yammer.metrics.reporting.JmxReporter; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import org.apache.pinot.common.utils.http.HttpClient; +import org.apache.pinot.common.version.PinotVersion; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; +import org.apache.pinot.spi.config.table.TableType; +import org.apache.pinot.spi.env.PinotConfiguration; +import org.apache.pinot.spi.metrics.PinotMetricUtils; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; + + +public class ServerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { + + private static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; + private ServerMetrics _serverMetrics; + + @BeforeClass + public void setup() { + PinotConfiguration pinotConfiguration = new PinotConfiguration(); + pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, + "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); + PinotMetricUtils.init(pinotConfiguration); + + // Initialize ServerMetrics with the registry + YammerMetricsRegistry yammerMetricsRegistry = new YammerMetricsRegistry(); + _serverMetrics = new ServerMetrics(yammerMetricsRegistry); + + // Enable JMX reporting + MetricsRegistry metricsRegistry = (MetricsRegistry) yammerMetricsRegistry.getMetricsRegistry(); + JmxReporter jmxReporter = new JmxReporter(metricsRegistry); + jmxReporter.start(); + + _httpClient = new HttpClient(); + } + + /** + * This test validates each timer defined in {@link ServerTimer} + */ + @Test + public void serverTimerTest() + throws IOException, URISyntaxException { + + //get all exposed metrics before we expose any timers + List promMetricsBefore = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + + for (ServerTimer serverTimer : ServerTimer.values()) { + if (serverTimer.isGlobal()) { + _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS); + //this gauge uses rawTableName + } else if (serverTimer == ServerTimer.SEGMENT_UPLOAD_TIME_MS) { + _serverMetrics.addTimedTableValue(RAW_TABLE_NAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); + } else { + _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); + } + } + //assert on timers with labels + assertTimerExportedCorrectly("freshnessLagMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("nettyConnectionSendResponseLatency", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("executionThreadCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("systemActivitiesCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("responseSerCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("segmentUploadTimeMs", List.of("table", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); + + assertTimerExportedCorrectly("totalCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("upsertPreloadTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("totalCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("upsertRemoveExpiredPrimaryKeysTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("grpcQueryExecutionMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("upsertSnapshotTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("dedupRemoveExpiredPrimaryKeysTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("secondaryQWaitTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + //assert on all global timers + assertTimerExportedCorrectly("hashJoinBuildTableCpuTimeMs", EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("multiStageSerializationCpuTimeMs", EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("multiStageDeserializationCpuTimeMs", EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("receiveDownstreamWaitCpuTimeMs", EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly("receiveUpstreamWaitCpuTimeMs", EXPORTED_METRIC_PREFIX); + + //now assert that we've added exported all timers present in ServerTimer.java + List promMetricsAfter = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + Assert.assertEquals(promMetricsAfter.size() - promMetricsBefore.size(), + ServerTimer.values().length * TIMER_TYPES.size()); + } + + /** + * This test validates each meter defined in {@link ServerMeter} + */ + @Test + public void serverMeterTest() + throws Exception { + + //get all exposed metrics before we expose any meters + List promMetricsBefore = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + + addGlobalMeter(ServerMeter.QUERIES); + assertMeterExportedCorrectly("queries", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.UNCAUGHT_EXCEPTIONS); + assertMeterExportedCorrectly("realtime_exceptions_uncaught", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS); + assertMeterExportedCorrectly("realtime_exceptions_requestDeserialization", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS); + assertMeterExportedCorrectly("realtime_exceptions_responseSerialization", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.QUERY_EXECUTION_EXCEPTIONS); + assertMeterExportedCorrectly("realtime_exceptions_queryExecution", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.HELIX_ZOOKEEPER_RECONNECTS); + assertMeterExportedCorrectly("helix_zookeeperReconnects", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.REALTIME_ROWS_CONSUMED); + assertMeterExportedCorrectly("realtime_rowsConsumed", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.REALTIME_CONSUMPTION_EXCEPTIONS); + assertMeterExportedCorrectly("realtime_consumptionExceptions", EXPORTED_METRIC_PREFIX); + + //todo: REALTIME_OFFSET_COMMITS and REALTIME_OFFSET_COMMIT_EXCEPTIONS are not used anywhere right now. This test + // case might need to be changed depending on how this metric is used in future + addGlobalMeter(ServerMeter.REALTIME_OFFSET_COMMITS); + assertMeterExportedCorrectly("realtime_offsetCommits", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.REALTIME_OFFSET_COMMIT_EXCEPTIONS); + assertMeterExportedCorrectly("realtime_exceptions_realtimeOffsetCommit", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_NOT_SENT); + assertMeterExportedCorrectly("llcControllerResponse_NotSent", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_COMMIT); + assertMeterExportedCorrectly("llcControllerResponse_Commit", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_HOLD); + assertMeterExportedCorrectly("llcControllerResponse_Hold", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_CATCH_UP); + assertMeterExportedCorrectly("llcControllerResponse_CatchUp", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_DISCARD); + assertMeterExportedCorrectly("llcControllerResponse_Discard", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_KEEP); + assertMeterExportedCorrectly("llcControllerResponse_Keep", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_NOT_LEADER); + assertMeterExportedCorrectly("llcControllerResponse_NotLeader", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_FAILED); + assertMeterExportedCorrectly("llcControllerResponse_Failed", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_COMMIT_SUCCESS); + assertMeterExportedCorrectly("llcControllerResponse_CommitSuccess", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_COMMIT_CONTINUE); + assertMeterExportedCorrectly("llcControllerResponse_CommitContinue", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_PROCESSED); + assertMeterExportedCorrectly("llcControllerResponse_Processed", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_UPLOAD_SUCCESS); + assertMeterExportedCorrectly("llcControllerResponse_UploadSuccess", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.NO_TABLE_ACCESS); + assertMeterExportedCorrectly("noTableAccess", EXPORTED_METRIC_PREFIX); + + addGlobalMeter(ServerMeter.INDEXING_FAILURES); + addGlobalMeter(ServerMeter.READINESS_CHECK_OK_CALLS); + addGlobalMeter(ServerMeter.READINESS_CHECK_BAD_CALLS); + addGlobalMeter(ServerMeter.QUERIES_KILLED); + addGlobalMeter(ServerMeter.HEAP_CRITICAL_LEVEL_EXCEEDED); + addGlobalMeter(ServerMeter.HEAP_PANIC_LEVEL_EXCEEDED); + addGlobalMeter(ServerMeter.NETTY_CONNECTION_BYTES_RECEIVED); + addGlobalMeter(ServerMeter.NETTY_CONNECTION_RESPONSES_SENT); + addGlobalMeter(ServerMeter.NETTY_CONNECTION_BYTES_SENT); + addGlobalMeter(ServerMeter.GRPC_QUERIES); + addGlobalMeter(ServerMeter.GRPC_BYTES_RECEIVED); + addGlobalMeter(ServerMeter.GRPC_BYTES_SENT); + addGlobalMeter(ServerMeter.GRPC_TRANSPORT_READY); + addGlobalMeter(ServerMeter.GRPC_TRANSPORT_TERMINATED); + + addGlobalMeter(ServerMeter.HASH_JOIN_TIMES_MAX_ROWS_REACHED); + addGlobalMeter(ServerMeter.AGGREGATE_TIMES_NUM_GROUPS_LIMIT_REACHED); + addGlobalMeter(ServerMeter.MULTI_STAGE_IN_MEMORY_MESSAGES); + addGlobalMeter(ServerMeter.MULTI_STAGE_RAW_MESSAGES); + addGlobalMeter(ServerMeter.MULTI_STAGE_RAW_BYTES); + addGlobalMeter(ServerMeter.WINDOW_TIMES_MAX_ROWS_REACHED); + + addMeterWithLables(ServerMeter.REALTIME_ROWS_CONSUMED, CLIENT_ID); + assertMeterExportedCorrectly("realtimeRowsConsumed", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.REALTIME_ROWS_SANITIZED, CLIENT_ID); + assertMeterExportedCorrectly("realtimeRowsSanitized", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.REALTIME_ROWS_FETCHED, CLIENT_ID); + assertMeterExportedCorrectly("realtimeRowsFetched", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.REALTIME_ROWS_FILTERED, CLIENT_ID); + assertMeterExportedCorrectly("realtimeRowsFiltered", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.INVALID_REALTIME_ROWS_DROPPED, CLIENT_ID); + assertMeterExportedCorrectly("invalidRealtimeRowsDropped", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, CLIENT_ID); + assertMeterExportedCorrectly("incompleteRealtimeRowsConsumed", EXPORTED_LABELS_FOR_CLIENT_ID, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, CLIENT_ID); + + addMeterWithLables(ServerMeter.REALTIME_CONSUMPTION_EXCEPTIONS, TABLE_STREAM_NAME); + assertMeterExportedCorrectly("realtimeConsumptionExceptions", List.of("table", "myTable_REALTIME_myTopic"), + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.REALTIME_MERGED_TEXT_IDX_TRUNCATED_DOCUMENT_SIZE, RAW_TABLE_NAME); + assertMeterExportedCorrectly("realtimeMergedTextIdxTruncatedDocumentSize", List.of("table", "myTable"), + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.QUERIES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("queries", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_SECONDARY_QUERIES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numSecondaryQueries", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_SECONDARY_QUERIES_SCHEDULED, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numSecondaryQueriesScheduled", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + //todo: SERVER_OUT_OF_CAPACITY_EXCEPTIONS is not used anywhere right now. This test case might need to be changed + // depending on how this metric is used in future + addMeterWithLables(ServerMeter.SERVER_OUT_OF_CAPACITY_EXCEPTIONS, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("serverOutOfCapacityExceptions", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("schedulingTimeoutExceptions", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.QUERY_EXECUTION_EXCEPTIONS, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("queryExecutionExceptions", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.DELETED_SEGMENT_COUNT, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("deletedSegmentCount", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.DELETE_TABLE_FAILURES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("deleteTableFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.REALTIME_PARTITION_MISMATCH, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("realtimePartitionMismatch", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.REALTIME_DEDUP_DROPPED, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("realtimeDedupDropped", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.UPSERT_KEYS_IN_WRONG_SEGMENT, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("upsertKeysInWrongSegment", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.UPSERT_OUT_OF_ORDER, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("upsertOutOfOrder", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.PARTIAL_UPSERT_KEYS_NOT_REPLACED, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("partialUpsertKeysNotReplaced", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.PARTIAL_UPSERT_OUT_OF_ORDER, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("partialUpsertOutOfOrder", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.DELETED_KEYS_TTL_PRIMARY_KEYS_REMOVED, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("deletedKeysTtlPrimaryKeysRemoved", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.TOTAL_KEYS_MARKED_FOR_DELETION, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("totalKeysMarkedForDeletion", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.DELETED_KEYS_WITHIN_TTL_WINDOW, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("deletedKeysWithinTtlWindow", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.DELETED_TTL_KEYS_IN_MULTIPLE_SEGMENTS, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("deletedTtlKeysInMultipleSegments", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.METADATA_TTL_PRIMARY_KEYS_REMOVED, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("metadataTtlPrimaryKeysRemoved", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.UPSERT_MISSED_VALID_DOC_ID_SNAPSHOT_COUNT, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("upsertMissedValidDocIdSnapshotCount", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.UPSERT_PRELOAD_FAILURE, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("upsertPreloadFailure", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_DOCS_SCANNED, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numDocsScanned", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_ENTRIES_SCANNED_IN_FILTER, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numEntriesScannedInFilter", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_ENTRIES_SCANNED_POST_FILTER, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numEntriesScannedPostFilter", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_SEGMENTS_QUERIED, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numSegmentsQueried", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_SEGMENTS_PROCESSED, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numSegmentsProcessed", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_SEGMENTS_MATCHED, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numSegmentsMatched", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_MISSING_SEGMENTS, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numMissingSegments", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.RELOAD_FAILURES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("reloadFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.REFRESH_FAILURES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("refreshFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.UNTAR_FAILURES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("untarFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.SEGMENT_STREAMED_DOWNLOAD_UNTAR_FAILURES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("segmentStreamedDownloadUntarFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.SEGMENT_DIR_MOVEMENT_FAILURES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("segmentDirMovementFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.SEGMENT_DOWNLOAD_FAILURES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("segmentDownloadFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.SEGMENT_DOWNLOAD_FROM_REMOTE_FAILURES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("segmentDownloadFromRemoteFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.SEGMENT_DOWNLOAD_FROM_PEERS_FAILURES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("segmentDownloadFromPeersFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_RESIZES, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numResizes", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.RESIZE_TIME_MS, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("resizeTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_SEGMENTS_PRUNED_INVALID, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numSegmentsPrunedInvalid", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_SEGMENTS_PRUNED_BY_LIMIT, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numSegmentsPrunedByLimit", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.NUM_SEGMENTS_PRUNED_BY_VALUE, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("numSegmentsPrunedByValue", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.LARGE_QUERY_RESPONSES_SENT, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("largeQueryResponsesSent", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.TOTAL_THREAD_CPU_TIME_MILLIS, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("totalThreadCpuTimeMillis", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.LARGE_QUERY_RESPONSE_SIZE_EXCEPTIONS, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly("largeQueryResponseSizeExceptions", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.ROWS_WITH_ERRORS, CLIENT_ID); + assertMeterExportedCorrectly("rowsWithErrors", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.ROWS_WITH_ERRORS, TABLE_STREAM_NAME); + assertMeterExportedCorrectly("rowsWithErrors", List.of("table", "myTable_REALTIME_myTopic"), + EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.SEGMENT_UPLOAD_FAILURE, RAW_TABLE_NAME); + assertMeterExportedCorrectly("segmentUploadFailure", List.of("table", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.SEGMENT_UPLOAD_SUCCESS, RAW_TABLE_NAME); + assertMeterExportedCorrectly("segmentUploadSuccess", List.of("table", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); + + addMeterWithLables(ServerMeter.SEGMENT_UPLOAD_TIMEOUT, RAW_TABLE_NAME); + assertMeterExportedCorrectly("segmentUploadTimeout", List.of("table", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); + + //get all exposed metrics now + List promMetricsAfter = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + + //We add 25 because 5 metrics (ROWS_WITH_ERRORS, QUERY_EXECUTION_EXCEPTIONS, QUERIES, REALTIME_ROWS_CONSUMED and + // REALTIME_CONSUMPTION_EXCEPTIONS) are used in 2 different ways in code + Assert.assertEquals(promMetricsAfter.size() - promMetricsBefore.size(), + ServerMeter.values().length * METER_TYPES.size() + 25); + } + + /** + * This test validates each meter defined in {@link ServerGauge} + */ + @Test + public void serverGaugeTest() + throws Exception { + + //get all exposed metrics before we expose any timers + List promMetricsBefore = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + + int partition = 3; + long someVal = 100L; + Supplier someValSupplier = () -> someVal; + + //global gauges + _serverMetrics.setValueOfGlobalGauge(ServerGauge.VERSION, PinotVersion.VERSION_METRIC_NAME, someVal); + _serverMetrics.addValueToGlobalGauge(ServerGauge.LLC_SIMULTANEOUS_SEGMENT_BUILDS, someVal); + _serverMetrics.setValueOfGlobalGauge(ServerGauge.JVM_HEAP_USED_BYTES, someVal); + _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_USED_DIRECT_MEMORY, someValSupplier); + _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_USED_HEAP_MEMORY, someValSupplier); + _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_ARENAS_DIRECT, someValSupplier); + _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_ARENAS_HEAP, someValSupplier); + _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_CACHE_SIZE_SMALL, someValSupplier); + _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_CACHE_SIZE_NORMAL, someValSupplier); + _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_CHUNK_SIZE, someValSupplier); + _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_THREADLOCALCACHE, someValSupplier); + + //local gauges + _serverMetrics.addValueToTableGauge(TABLE_NAME_WITH_TYPE, ServerGauge.DOCUMENT_COUNT, someVal); + _serverMetrics.addValueToTableGauge(TABLE_NAME_WITH_TYPE, ServerGauge.SEGMENT_COUNT, someVal); + _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 2, ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, someVal); + _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 2, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, someVal); + + //raw table name + _serverMetrics.addValueToTableGauge(RAW_TABLE_NAME, ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, someVal); + _serverMetrics.setValueOfTableGauge(RAW_TABLE_NAME, ServerGauge.REALTIME_MERGED_TEXT_IDX_DOCUMENT_AVG_LEN, someVal); + _serverMetrics.addValueToTableGauge(RAW_TABLE_NAME, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, someVal); + + _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LLC_PARTITION_CONSUMING, someVal); + _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, someVal); + _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, + TimeUnit.MILLISECONDS.toSeconds(someVal)); + _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, + TimeUnit.MILLISECONDS.toSeconds(someVal)); + _serverMetrics.setValueOfTableGauge(CLIENT_ID, + ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS, + TimeUnit.MILLISECONDS.toSeconds(someVal)); + _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, + TimeUnit.MILLISECONDS.toSeconds(someVal)); + _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS, + TimeUnit.MILLISECONDS.toSeconds(someVal)); + _serverMetrics.setValueOfTableGauge(RAW_TABLE_NAME, ServerGauge.CONSUMPTION_QUOTA_UTILIZATION, someVal); + + _serverMetrics.setValueOfTableGauge(TABLE_STREAM_NAME, ServerGauge.STREAM_DATA_LOSS, 1L); + + _serverMetrics.setOrUpdatePartitionGauge(TABLE_NAME_WITH_TYPE, partition, ServerGauge.REALTIME_INGESTION_DELAY_MS, + someValSupplier); + _serverMetrics.setOrUpdatePartitionGauge(TABLE_NAME_WITH_TYPE, partition, + ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS, someValSupplier); + _serverMetrics.setOrUpdatePartitionGauge(RAW_TABLE_NAME, partition, ServerGauge.LUCENE_INDEXING_DELAY_MS, + someValSupplier); + _serverMetrics.setOrUpdatePartitionGauge(RAW_TABLE_NAME, partition, ServerGauge.LUCENE_INDEXING_DELAY_DOCS, + someValSupplier); + _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, + ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, someVal); + _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, + ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, someVal); + _serverMetrics.setOrUpdatePartitionGauge(TABLE_NAME_WITH_TYPE, partition, ServerGauge.REALTIME_INGESTION_OFFSET_LAG, + someValSupplier); + + List labels = List.of("partition", String.valueOf(partition), "table", RAW_TABLE_NAME, "tableType", + TableType.REALTIME.toString()); + + assertGaugeExportedCorrectly("realtimeIngestionOffsetLag", labels, EXPORTED_METRIC_PREFIX); + + assertGaugeExportedCorrectly("upsertPrimaryKeysInSnapshotCount", labels, EXPORTED_METRIC_PREFIX); + + assertGaugeExportedCorrectly("upsertValidDocIdSnapshotCount", labels, EXPORTED_METRIC_PREFIX); + + assertGaugeExportedCorrectly("endToEndRealtimeIngestionDelayMs", labels, EXPORTED_METRIC_PREFIX); + + assertGaugeExportedCorrectly("realtimeIngestionDelayMs", labels, EXPORTED_METRIC_PREFIX); + +// we only match the metric name for version as the label value is dynamic (based on project version). + Optional pinotServerVersionMetricMaybe = + parseExportedPromMetrics(getExportedPromMetrics().getResponse()).stream() + .filter(exportedMetric -> exportedMetric.getMetricName().contains("version")).findAny(); + + Assert.assertTrue(pinotServerVersionMetricMaybe.isPresent()); + + assertGaugeExportedCorrectly("llcSimultaneousSegmentBuilds", EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly("nettyPooledUsedDirectMemory", EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly("nettyPooledUsedHeapMemory", EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly("nettyPooledUsedHeapMemory", EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly("nettyPooledArenasDirect", EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly("nettyPooledArenasHeap", EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly("nettyPooledCacheSizeSmall", EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly("nettyPooledCacheSizeNormal", EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly("nettyPooledChunkSize", EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly("jvmHeapUsedBytes", EXPORTED_METRIC_PREFIX); + + assertGaugeExportedCorrectly("llcPartitionConsuming", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + + assertGaugeExportedCorrectly("highestStreamOffsetConsumed", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + + assertGaugeExportedCorrectly("lastRealtimeSegmentCreationWaitTimeSeconds", EXPORTED_LABELS_FOR_CLIENT_ID, + EXPORTED_METRIC_PREFIX); + + assertGaugeExportedCorrectly("lastRealtimeSegmentInitialConsumptionDurationSeconds", EXPORTED_LABELS_FOR_CLIENT_ID, + EXPORTED_METRIC_PREFIX); + + assertGaugeExportedCorrectly("lastRealtimeSegmentCatchupDurationSeconds", EXPORTED_LABELS_FOR_CLIENT_ID, + EXPORTED_METRIC_PREFIX); + + List promMetricsAfter = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + Assert.assertEquals(promMetricsAfter.size() - promMetricsBefore.size(), ServerGauge.values().length); + } + + public void addGlobalMeter(ServerMeter serverMeter) { + _serverMetrics.addMeteredGlobalValue(serverMeter, 4L); + } + + public void addMeterWithLables(ServerMeter serverMeter, String label) { + _serverMetrics.addMeteredTableValue(label, serverMeter, 4L); + } +} From 51127e29c9c216a1c2279563f95b071d8f657130 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Fri, 4 Oct 2024 14:15:11 +0530 Subject: [PATCH 02/45] Removed hardcoded metrics --- .../metrics/PinotJMXToPromMetricsTest.java | 9 +- .../metrics/ServerJMXToPromMetricsTest.java | 147 ++++++------------ 2 files changed, 56 insertions(+), 100 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index b5cd6a96e5fc..4e18adbb4845 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -41,6 +41,8 @@ public class PinotJMXToPromMetricsTest { protected HttpClient _httpClient; + protected static final String LABEL_KEY_TABLE = "table"; + protected static final List METER_TYPES = List.of("Count", "FiveMinuteRate", "MeanRate", "OneMinuteRate", "FifteenMinuteRate"); @@ -109,8 +111,9 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String e List promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); for (String meterType : METER_TYPES) { - Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withName( - exportedMetricPrefix + exportedMeterPrefix + "_" + meterType))); + Assert.assertTrue(promMetrics.contains( + ServerJMXToPromMetricsTest.PromMetric.withName(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType)), + exportedMeterPrefix); } } @@ -121,7 +124,7 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, List promMetricsBefore = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - for (ServerTimer serverTimer : ServerTimer.values()) { if (serverTimer.isGlobal()) { _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS); @@ -84,44 +82,19 @@ public void serverTimerTest() _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); } } - //assert on timers with labels - assertTimerExportedCorrectly("freshnessLagMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("nettyConnectionSendResponseLatency", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("executionThreadCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("systemActivitiesCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("responseSerCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("segmentUploadTimeMs", List.of("table", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); - - assertTimerExportedCorrectly("totalCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("upsertPreloadTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("totalCpuTimeNs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("upsertRemoveExpiredPrimaryKeysTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("grpcQueryExecutionMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("upsertSnapshotTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("dedupRemoveExpiredPrimaryKeysTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("secondaryQWaitTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - //assert on all global timers - assertTimerExportedCorrectly("hashJoinBuildTableCpuTimeMs", EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("multiStageSerializationCpuTimeMs", EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("multiStageDeserializationCpuTimeMs", EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("receiveDownstreamWaitCpuTimeMs", EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly("receiveUpstreamWaitCpuTimeMs", EXPORTED_METRIC_PREFIX); - //now assert that we've added exported all timers present in ServerTimer.java - List promMetricsAfter = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - Assert.assertEquals(promMetricsAfter.size() - promMetricsBefore.size(), - ServerTimer.values().length * TIMER_TYPES.size()); + for (ServerTimer serverTimer : ServerTimer.values()) { + if (serverTimer.isGlobal()) { + assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_METRIC_PREFIX); + //this gauge uses rawTableName + } else if (serverTimer == ServerTimer.SEGMENT_UPLOAD_TIME_MS) { + assertMeterExportedCorrectly(serverTimer.getTimerName(), List.of(LABEL_KEY_TABLE, RAW_TABLE_NAME), + EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + } + } } /** @@ -131,80 +104,31 @@ public void serverTimerTest() public void serverMeterTest() throws Exception { - //get all exposed metrics before we expose any meters - List promMetricsBefore = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - addGlobalMeter(ServerMeter.QUERIES); - assertMeterExportedCorrectly("queries", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.UNCAUGHT_EXCEPTIONS); - assertMeterExportedCorrectly("realtime_exceptions_uncaught", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS); - assertMeterExportedCorrectly("realtime_exceptions_requestDeserialization", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS); - assertMeterExportedCorrectly("realtime_exceptions_responseSerialization", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.QUERY_EXECUTION_EXCEPTIONS); - assertMeterExportedCorrectly("realtime_exceptions_queryExecution", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.HELIX_ZOOKEEPER_RECONNECTS); - assertMeterExportedCorrectly("helix_zookeeperReconnects", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.REALTIME_ROWS_CONSUMED); - assertMeterExportedCorrectly("realtime_rowsConsumed", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.REALTIME_CONSUMPTION_EXCEPTIONS); - assertMeterExportedCorrectly("realtime_consumptionExceptions", EXPORTED_METRIC_PREFIX); - //todo: REALTIME_OFFSET_COMMITS and REALTIME_OFFSET_COMMIT_EXCEPTIONS are not used anywhere right now. This test // case might need to be changed depending on how this metric is used in future addGlobalMeter(ServerMeter.REALTIME_OFFSET_COMMITS); - assertMeterExportedCorrectly("realtime_offsetCommits", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.REALTIME_OFFSET_COMMIT_EXCEPTIONS); - assertMeterExportedCorrectly("realtime_exceptions_realtimeOffsetCommit", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_NOT_SENT); - assertMeterExportedCorrectly("llcControllerResponse_NotSent", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_COMMIT); - assertMeterExportedCorrectly("llcControllerResponse_Commit", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_HOLD); - assertMeterExportedCorrectly("llcControllerResponse_Hold", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_CATCH_UP); - assertMeterExportedCorrectly("llcControllerResponse_CatchUp", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_DISCARD); - assertMeterExportedCorrectly("llcControllerResponse_Discard", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_KEEP); - assertMeterExportedCorrectly("llcControllerResponse_Keep", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_NOT_LEADER); - assertMeterExportedCorrectly("llcControllerResponse_NotLeader", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_FAILED); - assertMeterExportedCorrectly("llcControllerResponse_Failed", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_COMMIT_SUCCESS); - assertMeterExportedCorrectly("llcControllerResponse_CommitSuccess", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_COMMIT_CONTINUE); - assertMeterExportedCorrectly("llcControllerResponse_CommitContinue", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_PROCESSED); - assertMeterExportedCorrectly("llcControllerResponse_Processed", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_UPLOAD_SUCCESS); - assertMeterExportedCorrectly("llcControllerResponse_UploadSuccess", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.NO_TABLE_ACCESS); - assertMeterExportedCorrectly("noTableAccess", EXPORTED_METRIC_PREFIX); - addGlobalMeter(ServerMeter.INDEXING_FAILURES); addGlobalMeter(ServerMeter.READINESS_CHECK_OK_CALLS); addGlobalMeter(ServerMeter.READINESS_CHECK_BAD_CALLS); @@ -227,6 +151,43 @@ public void serverMeterTest() addGlobalMeter(ServerMeter.MULTI_STAGE_RAW_BYTES); addGlobalMeter(ServerMeter.WINDOW_TIMES_MAX_ROWS_REACHED); + Arrays.stream(ServerMeter.values()).filter(ServerMeter::isGlobal).peek(this::addGlobalMeter) + .forEach(serverMeter -> { + try { + if (serverMeter == ServerMeter.HELIX_ZOOKEEPER_RECONNECTS) { + assertMeterExportedCorrectly("helix_zookeeperReconnects", EXPORTED_METRIC_PREFIX); + } else if (serverMeter == ServerMeter.REALTIME_CONSUMPTION_EXCEPTIONS) { + assertMeterExportedCorrectly("realtime_consumptionExceptions", EXPORTED_METRIC_PREFIX); + } else if (serverMeter == ServerMeter.REALTIME_ROWS_CONSUMED) { + assertMeterExportedCorrectly("realtime_rowsConsumed", EXPORTED_METRIC_PREFIX); + } else if (serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS) { + assertMeterExportedCorrectly("realtime_exceptions_uncaught", EXPORTED_METRIC_PREFIX); + } else if (serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS) { + assertMeterExportedCorrectly("realtime_exceptions_requestDeserialization", EXPORTED_METRIC_PREFIX); + } else if (serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS) { + assertMeterExportedCorrectly("realtime_exceptions_responseSerialization", EXPORTED_METRIC_PREFIX); + } else if (serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS) { + assertMeterExportedCorrectly("realtime_exceptions_schedulingTimeout", EXPORTED_METRIC_PREFIX); + } else if (serverMeter == ServerMeter.REALTIME_OFFSET_COMMITS) { + assertMeterExportedCorrectly("realtime_offsetCommits", EXPORTED_METRIC_PREFIX); + } else if (serverMeter.getMeterName().startsWith("llc")) { + String meterName = serverMeter.getMeterName(); + meterName.lastIndexOf("Response"); + assertMeterExportedCorrectly( + "llcControllerResponse_" + meterName.substring(meterName.lastIndexOf("Response") + 8), + EXPORTED_METRIC_PREFIX); + } else if (serverMeter.getMeterName().startsWith("nettyConnection")) { + String exportedMeterPrefix = "nettyConnection" + "_" + serverMeter.getMeterName() + .substring(serverMeter.getMeterName().indexOf("Connection") + 10); + assertMeterExportedCorrectly(exportedMeterPrefix, EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_METRIC_PREFIX); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + addMeterWithLables(ServerMeter.REALTIME_ROWS_CONSUMED, CLIENT_ID); assertMeterExportedCorrectly("realtimeRowsConsumed", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); @@ -441,14 +402,6 @@ public void serverMeterTest() addMeterWithLables(ServerMeter.SEGMENT_UPLOAD_TIMEOUT, RAW_TABLE_NAME); assertMeterExportedCorrectly("segmentUploadTimeout", List.of("table", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); - - //get all exposed metrics now - List promMetricsAfter = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - - //We add 25 because 5 metrics (ROWS_WITH_ERRORS, QUERY_EXECUTION_EXCEPTIONS, QUERIES, REALTIME_ROWS_CONSUMED and - // REALTIME_CONSUMPTION_EXCEPTIONS) are used in 2 different ways in code - Assert.assertEquals(promMetricsAfter.size() - promMetricsBefore.size(), - ServerMeter.values().length * METER_TYPES.size() + 25); } /** From 417a8a69cd642a7793238fe4cccd12b3c596915e Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Fri, 4 Oct 2024 15:02:35 +0530 Subject: [PATCH 03/45] WIP --- .../metrics/PinotJMXToPromMetricsTest.java | 10 +- .../metrics/ServerJMXToPromMetricsTest.java | 321 ++---------------- 2 files changed, 45 insertions(+), 286 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 4e18adbb4845..606b345bde1d 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -30,9 +30,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.spi.config.table.TableType; +import org.apache.pinot.spi.utils.StringUtil; import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; @@ -213,7 +215,13 @@ public boolean equals(Object o) { return false; } PromMetric that = (PromMetric) o; - return Objects.equal(_metricName, that._metricName) && Objects.equal(_labels, that._labels); + return metricNamesAreSimilar(that) && Objects.equal(_labels, that._labels); + } + + private boolean metricNamesAreSimilar(PromMetric that) { + String processedMetricNameThis = StringUtils.remove(_metricName, "_"); + String processedMetricNameThat = StringUtils.remove(that._metricName, "_"); + return StringUtils.equalsIgnoreCase(processedMetricNameThis, processedMetricNameThat); } @Override diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index 2ecdbddc57f0..30cce0b1c96f 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -103,82 +103,17 @@ public void serverTimerTest() @Test public void serverMeterTest() throws Exception { - - addGlobalMeter(ServerMeter.QUERIES); - addGlobalMeter(ServerMeter.UNCAUGHT_EXCEPTIONS); - addGlobalMeter(ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS); - addGlobalMeter(ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS); - addGlobalMeter(ServerMeter.QUERY_EXECUTION_EXCEPTIONS); - addGlobalMeter(ServerMeter.HELIX_ZOOKEEPER_RECONNECTS); - addGlobalMeter(ServerMeter.REALTIME_ROWS_CONSUMED); - addGlobalMeter(ServerMeter.REALTIME_CONSUMPTION_EXCEPTIONS); - //todo: REALTIME_OFFSET_COMMITS and REALTIME_OFFSET_COMMIT_EXCEPTIONS are not used anywhere right now. This test - // case might need to be changed depending on how this metric is used in future - addGlobalMeter(ServerMeter.REALTIME_OFFSET_COMMITS); - addGlobalMeter(ServerMeter.REALTIME_OFFSET_COMMIT_EXCEPTIONS); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_NOT_SENT); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_COMMIT); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_HOLD); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_CATCH_UP); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_DISCARD); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_KEEP); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_NOT_LEADER); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_FAILED); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_COMMIT_SUCCESS); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_COMMIT_CONTINUE); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_PROCESSED); - addGlobalMeter(ServerMeter.LLC_CONTROLLER_RESPONSE_UPLOAD_SUCCESS); - addGlobalMeter(ServerMeter.NO_TABLE_ACCESS); - addGlobalMeter(ServerMeter.INDEXING_FAILURES); - addGlobalMeter(ServerMeter.READINESS_CHECK_OK_CALLS); - addGlobalMeter(ServerMeter.READINESS_CHECK_BAD_CALLS); - addGlobalMeter(ServerMeter.QUERIES_KILLED); - addGlobalMeter(ServerMeter.HEAP_CRITICAL_LEVEL_EXCEEDED); - addGlobalMeter(ServerMeter.HEAP_PANIC_LEVEL_EXCEEDED); - addGlobalMeter(ServerMeter.NETTY_CONNECTION_BYTES_RECEIVED); - addGlobalMeter(ServerMeter.NETTY_CONNECTION_RESPONSES_SENT); - addGlobalMeter(ServerMeter.NETTY_CONNECTION_BYTES_SENT); - addGlobalMeter(ServerMeter.GRPC_QUERIES); - addGlobalMeter(ServerMeter.GRPC_BYTES_RECEIVED); - addGlobalMeter(ServerMeter.GRPC_BYTES_SENT); - addGlobalMeter(ServerMeter.GRPC_TRANSPORT_READY); - addGlobalMeter(ServerMeter.GRPC_TRANSPORT_TERMINATED); - - addGlobalMeter(ServerMeter.HASH_JOIN_TIMES_MAX_ROWS_REACHED); - addGlobalMeter(ServerMeter.AGGREGATE_TIMES_NUM_GROUPS_LIMIT_REACHED); - addGlobalMeter(ServerMeter.MULTI_STAGE_IN_MEMORY_MESSAGES); - addGlobalMeter(ServerMeter.MULTI_STAGE_RAW_MESSAGES); - addGlobalMeter(ServerMeter.MULTI_STAGE_RAW_BYTES); - addGlobalMeter(ServerMeter.WINDOW_TIMES_MAX_ROWS_REACHED); - + //first, assert on all global meters Arrays.stream(ServerMeter.values()).filter(ServerMeter::isGlobal).peek(this::addGlobalMeter) .forEach(serverMeter -> { try { - if (serverMeter == ServerMeter.HELIX_ZOOKEEPER_RECONNECTS) { - assertMeterExportedCorrectly("helix_zookeeperReconnects", EXPORTED_METRIC_PREFIX); - } else if (serverMeter == ServerMeter.REALTIME_CONSUMPTION_EXCEPTIONS) { - assertMeterExportedCorrectly("realtime_consumptionExceptions", EXPORTED_METRIC_PREFIX); - } else if (serverMeter == ServerMeter.REALTIME_ROWS_CONSUMED) { - assertMeterExportedCorrectly("realtime_rowsConsumed", EXPORTED_METRIC_PREFIX); - } else if (serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS) { - assertMeterExportedCorrectly("realtime_exceptions_uncaught", EXPORTED_METRIC_PREFIX); - } else if (serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS) { - assertMeterExportedCorrectly("realtime_exceptions_requestDeserialization", EXPORTED_METRIC_PREFIX); - } else if (serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS) { - assertMeterExportedCorrectly("realtime_exceptions_responseSerialization", EXPORTED_METRIC_PREFIX); - } else if (serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS) { - assertMeterExportedCorrectly("realtime_exceptions_schedulingTimeout", EXPORTED_METRIC_PREFIX); - } else if (serverMeter == ServerMeter.REALTIME_OFFSET_COMMITS) { - assertMeterExportedCorrectly("realtime_offsetCommits", EXPORTED_METRIC_PREFIX); - } else if (serverMeter.getMeterName().startsWith("llc")) { + if (serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS + || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS + || serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS + || serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS) { String meterName = serverMeter.getMeterName(); - meterName.lastIndexOf("Response"); - assertMeterExportedCorrectly( - "llcControllerResponse_" + meterName.substring(meterName.lastIndexOf("Response") + 8), - EXPORTED_METRIC_PREFIX); - } else if (serverMeter.getMeterName().startsWith("nettyConnection")) { - String exportedMeterPrefix = "nettyConnection" + "_" + serverMeter.getMeterName() - .substring(serverMeter.getMeterName().indexOf("Connection") + 10); + String exportedMeterPrefix = + "realtime_exceptions_" + meterName.substring(0, meterName.lastIndexOf("Exceptions")); assertMeterExportedCorrectly(exportedMeterPrefix, EXPORTED_METRIC_PREFIX); } else { assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_METRIC_PREFIX); @@ -188,220 +123,36 @@ public void serverMeterTest() } }); - addMeterWithLables(ServerMeter.REALTIME_ROWS_CONSUMED, CLIENT_ID); - assertMeterExportedCorrectly("realtimeRowsConsumed", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.REALTIME_ROWS_SANITIZED, CLIENT_ID); - assertMeterExportedCorrectly("realtimeRowsSanitized", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.REALTIME_ROWS_FETCHED, CLIENT_ID); - assertMeterExportedCorrectly("realtimeRowsFetched", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.REALTIME_ROWS_FILTERED, CLIENT_ID); - assertMeterExportedCorrectly("realtimeRowsFiltered", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.INVALID_REALTIME_ROWS_DROPPED, CLIENT_ID); - assertMeterExportedCorrectly("invalidRealtimeRowsDropped", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, CLIENT_ID); - assertMeterExportedCorrectly("incompleteRealtimeRowsConsumed", EXPORTED_LABELS_FOR_CLIENT_ID, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, CLIENT_ID); - - addMeterWithLables(ServerMeter.REALTIME_CONSUMPTION_EXCEPTIONS, TABLE_STREAM_NAME); - assertMeterExportedCorrectly("realtimeConsumptionExceptions", List.of("table", "myTable_REALTIME_myTopic"), - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.REALTIME_MERGED_TEXT_IDX_TRUNCATED_DOCUMENT_SIZE, RAW_TABLE_NAME); - assertMeterExportedCorrectly("realtimeMergedTextIdxTruncatedDocumentSize", List.of("table", "myTable"), - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.QUERIES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("queries", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_SECONDARY_QUERIES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numSecondaryQueries", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_SECONDARY_QUERIES_SCHEDULED, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numSecondaryQueriesScheduled", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - //todo: SERVER_OUT_OF_CAPACITY_EXCEPTIONS is not used anywhere right now. This test case might need to be changed - // depending on how this metric is used in future - addMeterWithLables(ServerMeter.SERVER_OUT_OF_CAPACITY_EXCEPTIONS, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("serverOutOfCapacityExceptions", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("schedulingTimeoutExceptions", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.QUERY_EXECUTION_EXCEPTIONS, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("queryExecutionExceptions", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.DELETED_SEGMENT_COUNT, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("deletedSegmentCount", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.DELETE_TABLE_FAILURES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("deleteTableFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.REALTIME_PARTITION_MISMATCH, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("realtimePartitionMismatch", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.REALTIME_DEDUP_DROPPED, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("realtimeDedupDropped", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); + List metersThatAcceptClientId = + List.of(ServerMeter.REALTIME_ROWS_CONSUMED, ServerMeter.REALTIME_ROWS_SANITIZED, + ServerMeter.REALTIME_ROWS_FETCHED, ServerMeter.REALTIME_ROWS_FILTERED, + ServerMeter.INVALID_REALTIME_ROWS_DROPPED, ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, + ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, ServerMeter.ROWS_WITH_ERRORS); + + List metersThatAcceptRawTableName = + List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, + ServerMeter.SEGMENT_UPLOAD_TIMEOUT); + + //remaining all meters accept table name with type + Arrays.stream(ServerMeter.values()).filter(serverMeter -> !serverMeter.isGlobal()).forEach(serverMeter -> { + try { + if (metersThatAcceptClientId.contains(serverMeter)) { + addMeterWithLables(serverMeter, CLIENT_ID); + assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_CLIENT_ID, + EXPORTED_METRIC_PREFIX); + } else if (metersThatAcceptRawTableName.contains(serverMeter)) { + addMeterWithLables(serverMeter, RAW_TABLE_NAME); + assertMeterExportedCorrectly(serverMeter.getMeterName(), List.of("table", RAW_TABLE_NAME), + EXPORTED_METRIC_PREFIX); + } else { + addMeterWithLables(serverMeter, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + } + } catch (Exception e) { - addMeterWithLables(ServerMeter.UPSERT_KEYS_IN_WRONG_SEGMENT, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("upsertKeysInWrongSegment", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.UPSERT_OUT_OF_ORDER, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("upsertOutOfOrder", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.PARTIAL_UPSERT_KEYS_NOT_REPLACED, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("partialUpsertKeysNotReplaced", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.PARTIAL_UPSERT_OUT_OF_ORDER, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("partialUpsertOutOfOrder", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.DELETED_KEYS_TTL_PRIMARY_KEYS_REMOVED, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("deletedKeysTtlPrimaryKeysRemoved", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.TOTAL_KEYS_MARKED_FOR_DELETION, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("totalKeysMarkedForDeletion", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.DELETED_KEYS_WITHIN_TTL_WINDOW, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("deletedKeysWithinTtlWindow", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.DELETED_TTL_KEYS_IN_MULTIPLE_SEGMENTS, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("deletedTtlKeysInMultipleSegments", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.METADATA_TTL_PRIMARY_KEYS_REMOVED, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("metadataTtlPrimaryKeysRemoved", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.UPSERT_MISSED_VALID_DOC_ID_SNAPSHOT_COUNT, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("upsertMissedValidDocIdSnapshotCount", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.UPSERT_PRELOAD_FAILURE, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("upsertPreloadFailure", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_DOCS_SCANNED, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numDocsScanned", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_ENTRIES_SCANNED_IN_FILTER, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numEntriesScannedInFilter", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_ENTRIES_SCANNED_POST_FILTER, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numEntriesScannedPostFilter", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_SEGMENTS_QUERIED, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numSegmentsQueried", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_SEGMENTS_PROCESSED, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numSegmentsProcessed", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_SEGMENTS_MATCHED, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numSegmentsMatched", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_MISSING_SEGMENTS, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numMissingSegments", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.RELOAD_FAILURES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("reloadFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.REFRESH_FAILURES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("refreshFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.UNTAR_FAILURES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("untarFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.SEGMENT_STREAMED_DOWNLOAD_UNTAR_FAILURES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("segmentStreamedDownloadUntarFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.SEGMENT_DIR_MOVEMENT_FAILURES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("segmentDirMovementFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.SEGMENT_DOWNLOAD_FAILURES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("segmentDownloadFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.SEGMENT_DOWNLOAD_FROM_REMOTE_FAILURES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("segmentDownloadFromRemoteFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.SEGMENT_DOWNLOAD_FROM_PEERS_FAILURES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("segmentDownloadFromPeersFailures", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_RESIZES, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numResizes", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.RESIZE_TIME_MS, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("resizeTimeMs", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_SEGMENTS_PRUNED_INVALID, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numSegmentsPrunedInvalid", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_SEGMENTS_PRUNED_BY_LIMIT, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numSegmentsPrunedByLimit", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.NUM_SEGMENTS_PRUNED_BY_VALUE, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("numSegmentsPrunedByValue", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.LARGE_QUERY_RESPONSES_SENT, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("largeQueryResponsesSent", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.TOTAL_THREAD_CPU_TIME_MILLIS, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("totalThreadCpuTimeMillis", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.LARGE_QUERY_RESPONSE_SIZE_EXCEPTIONS, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly("largeQueryResponseSizeExceptions", EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.ROWS_WITH_ERRORS, CLIENT_ID); - assertMeterExportedCorrectly("rowsWithErrors", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.ROWS_WITH_ERRORS, TABLE_STREAM_NAME); - assertMeterExportedCorrectly("rowsWithErrors", List.of("table", "myTable_REALTIME_myTopic"), - EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.SEGMENT_UPLOAD_FAILURE, RAW_TABLE_NAME); - assertMeterExportedCorrectly("segmentUploadFailure", List.of("table", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.SEGMENT_UPLOAD_SUCCESS, RAW_TABLE_NAME); - assertMeterExportedCorrectly("segmentUploadSuccess", List.of("table", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); - - addMeterWithLables(ServerMeter.SEGMENT_UPLOAD_TIMEOUT, RAW_TABLE_NAME); - assertMeterExportedCorrectly("segmentUploadTimeout", List.of("table", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); + } + }); } /** From f85d7c822365ce78d3d751cb815e61f7f8a99be1 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Fri, 4 Oct 2024 18:07:25 +0530 Subject: [PATCH 04/45] WIP --- .../ControllerJMXToPromMetricsTest.java | 87 +++++++++++++++++++ .../metrics/PinotJMXToPromMetricsTest.java | 32 +++---- .../metrics/ServerJMXToPromMetricsTest.java | 10 +-- 3 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java new file mode 100644 index 000000000000..eb43ad66a393 --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -0,0 +1,87 @@ +package org.apache.pinot.common.metrics; + +import com.yammer.metrics.core.MetricsRegistry; +import com.yammer.metrics.reporting.JmxReporter; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.concurrent.TimeUnit; +import org.apache.pinot.common.utils.http.HttpClient; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; +import org.apache.pinot.spi.env.PinotConfiguration; +import org.apache.pinot.spi.metrics.PinotMetricUtils; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; + + +public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { + private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; + private ControllerMetrics _controllerMetrics; + @BeforeClass + public void setup() { + PinotConfiguration pinotConfiguration = new PinotConfiguration(); + pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, + "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); + PinotMetricUtils.init(pinotConfiguration); + + // Initialize ServerMetrics with the registry + YammerMetricsRegistry yammerMetricsRegistry = new YammerMetricsRegistry(); + _controllerMetrics = new ControllerMetrics(yammerMetricsRegistry); + + // Enable JMX reporting + MetricsRegistry metricsRegistry = (MetricsRegistry) yammerMetricsRegistry.getMetricsRegistry(); + JmxReporter jmxReporter = new JmxReporter(metricsRegistry); + jmxReporter.start(); + + _httpClient = new HttpClient(); + } + + @Test + public void controllerTimerTest() + throws IOException, URISyntaxException { + + for (ControllerTimer controllerTimer : ControllerTimer.values()) { + if (controllerTimer.isGlobal()) { + _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS); + } else { + _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); + _controllerMetrics.addTimedTableValue(RAW_TABLE_NAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); + } + } + + for (ControllerTimer controllerTimer : ControllerTimer.values()) { + if (controllerTimer.isGlobal()) { + assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX); + } + } + } + + @Test + public void controllerMeterTest() + throws IOException, URISyntaxException { + for (ControllerMeter controllerMeter : ControllerMeter.values()) { + if (controllerMeter.isGlobal()) { + _controllerMetrics.addMeteredGlobalValue(controllerMeter, 5L); + } + } + for (ControllerMeter controllerMeter : ControllerMeter.values()) { + if (controllerMeter.isGlobal()) { + if (controllerMeter == ControllerMeter.CONTROLLER_INSTANCE_POST_ERROR) { + assertMeterExportedCorrectly("InstancePostError", EXPORTED_METRIC_PREFIX); + } else if (controllerMeter == ControllerMeter.CONTROLLER_INSTANCE_DELETE_ERROR) { + assertMeterExportedCorrectly("InstanceDeleteError", EXPORTED_METRIC_PREFIX); + } else if (controllerMeter == ControllerMeter.CONTROLLER_SEGMENT_UPLOAD_ERROR) { + assertMeterExportedCorrectly("SegmentUploadError", EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(controllerMeter.getMeterName(), EXPORTED_METRIC_PREFIX); + } + } + } + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 606b345bde1d..c2c68cfc8606 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -66,33 +66,35 @@ public class PinotJMXToPromMetricsTest { protected static final List EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE = List.of("table", "myTable", "tableType", "REALTIME"); + protected static final List EXPORTED_LABELS_FOR_RAW_TABLE_NAME = List.of("table", "myTable"); + protected static final List EXPORTED_LABELS_FOR_CLIENT_ID = List.of("partition", PARTITION_GROUP_ID, "table", RAW_TABLE_NAME, "tableType", TableType.REALTIME.toString(), "topic", KAFKA_TOPIC); protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) throws IOException, URISyntaxException { - List promMetrics = + List promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); Assert.assertTrue(promMetrics.contains( - ServerJMXToPromMetricsTest.PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value"))); + PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value"))); } protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, List labels, String exportedMetricPrefix) throws IOException, URISyntaxException { - List promMetrics = + List promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withNameAndLabels( + Assert.assertTrue(promMetrics.contains(PromMetric.withNameAndLabels( exportedMetricPrefix + exportedGaugePrefix + "_" + "Value", labels))); } protected void assertTimerExportedCorrectly(String exportedTimerPrefix, String exportedMetricPrefix) throws IOException, URISyntaxException { - List promMetrics = + List promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); for (String meterType : TIMER_TYPES) { - Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withName( + Assert.assertTrue(promMetrics.contains(PromMetric.withName( exportedMetricPrefix + exportedTimerPrefix + "_" + meterType))); } } @@ -100,21 +102,21 @@ protected void assertTimerExportedCorrectly(String exportedTimerPrefix, String e protected void assertTimerExportedCorrectly(String exportedTimerPrefix, List labels, String exportedMetricPrefix) throws IOException, URISyntaxException { - List promMetrics = + List promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); for (String meterType : METER_TYPES) { - Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withNameAndLabels( + Assert.assertTrue(promMetrics.contains(PromMetric.withNameAndLabels( exportedMetricPrefix + exportedTimerPrefix + "_" + meterType, labels))); } } protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String exportedMetricPrefix) throws IOException, URISyntaxException { - List promMetrics = + List promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); for (String meterType : METER_TYPES) { Assert.assertTrue(promMetrics.contains( - ServerJMXToPromMetricsTest.PromMetric.withName(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType)), + PromMetric.withName(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType)), exportedMeterPrefix); } } @@ -122,25 +124,25 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String e protected void assertMeterExportedCorrectly(String exportedMeterPrefix, List labels, String exportedMetricPrefix) throws IOException, URISyntaxException { - List promMetrics = + List promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); for (String meterType : METER_TYPES) { - Assert.assertTrue(promMetrics.contains(ServerJMXToPromMetricsTest.PromMetric.withNameAndLabels( + Assert.assertTrue(promMetrics.contains(PromMetric.withNameAndLabels( exportedMetricPrefix + exportedMeterPrefix + "_" + meterType, labels)), exportedMeterPrefix); } } - protected List parseExportedPromMetrics(String response) + protected List parseExportedPromMetrics(String response) throws IOException { - List exportedPromMetrics = new ArrayList<>(); + List exportedPromMetrics = new ArrayList<>(); BufferedReader reader = new BufferedReader(new StringReader(response)); String line; while ((line = reader.readLine()) != null) { if (line.startsWith("pinot_")) { - exportedPromMetrics.add(ServerJMXToPromMetricsTest.PromMetric.fromExportedMetric(line)); + exportedPromMetrics.add(PromMetric.fromExportedMetric(line)); } } reader.close(); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index 30cce0b1c96f..9252946cf222 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -75,24 +75,20 @@ public void serverTimerTest() for (ServerTimer serverTimer : ServerTimer.values()) { if (serverTimer.isGlobal()) { _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS); - //this gauge uses rawTableName - } else if (serverTimer == ServerTimer.SEGMENT_UPLOAD_TIME_MS) { - _serverMetrics.addTimedTableValue(RAW_TABLE_NAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); } else { _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); + _serverMetrics.addTimedTableValue(RAW_TABLE_NAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); } } for (ServerTimer serverTimer : ServerTimer.values()) { if (serverTimer.isGlobal()) { assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_METRIC_PREFIX); - //this gauge uses rawTableName - } else if (serverTimer == ServerTimer.SEGMENT_UPLOAD_TIME_MS) { - assertMeterExportedCorrectly(serverTimer.getTimerName(), List.of(LABEL_KEY_TABLE, RAW_TABLE_NAME), - EXPORTED_METRIC_PREFIX); } else { assertMeterExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX); } } } From af4a3d20edfa9ebd873b3e4b3950b54286c1e00a Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Sat, 5 Oct 2024 15:25:12 +0530 Subject: [PATCH 05/45] controllerMeterTest working --- .../ControllerJMXToPromMetricsTest.java | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index eb43ad66a393..41444344209b 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -4,11 +4,14 @@ import com.yammer.metrics.reporting.JmxReporter; import java.io.IOException; import java.net.URISyntaxException; +import java.util.List; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; +import org.apache.pinot.spi.utils.StringUtil; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -18,6 +21,7 @@ public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; private ControllerMetrics _controllerMetrics; + @BeforeClass public void setup() { PinotConfiguration pinotConfiguration = new PinotConfiguration(); @@ -25,7 +29,7 @@ public void setup() { "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); PinotMetricUtils.init(pinotConfiguration); - // Initialize ServerMetrics with the registry + // Initialize ControllerMetrics with the registry YammerMetricsRegistry yammerMetricsRegistry = new YammerMetricsRegistry(); _controllerMetrics = new ControllerMetrics(yammerMetricsRegistry); @@ -68,20 +72,35 @@ public void controllerMeterTest() for (ControllerMeter controllerMeter : ControllerMeter.values()) { if (controllerMeter.isGlobal()) { _controllerMetrics.addMeteredGlobalValue(controllerMeter, 5L); + } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { + _controllerMetrics.addMeteredTableValue("ClusterHealthCheck", controllerMeter, 1L); + } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { + _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + "ClusterHealthTask", controllerMeter, 1L); + } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { + _controllerMetrics.addMeteredTableValue("ClusterHealthCheck", controllerMeter, 1L); + } else { + _controllerMetrics.addMeteredTableValue(RAW_TABLE_NAME, controllerMeter, 5L); } } for (ControllerMeter controllerMeter : ControllerMeter.values()) { + String meterName = controllerMeter.getMeterName(); + String strippedMeterName = StringUtils.remove(meterName, "controller"); if (controllerMeter.isGlobal()) { - if (controllerMeter == ControllerMeter.CONTROLLER_INSTANCE_POST_ERROR) { - assertMeterExportedCorrectly("InstancePostError", EXPORTED_METRIC_PREFIX); - } else if (controllerMeter == ControllerMeter.CONTROLLER_INSTANCE_DELETE_ERROR) { - assertMeterExportedCorrectly("InstanceDeleteError", EXPORTED_METRIC_PREFIX); - } else if (controllerMeter == ControllerMeter.CONTROLLER_SEGMENT_UPLOAD_ERROR) { - assertMeterExportedCorrectly("SegmentUploadError", EXPORTED_METRIC_PREFIX); - } else { - assertMeterExportedCorrectly(controllerMeter.getMeterName(), EXPORTED_METRIC_PREFIX); - } + assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX); + } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { + assertMeterExportedCorrectly(meterName, List.of("table", "ClusterHealthCheck"), EXPORTED_METRIC_PREFIX); + } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { + assertMeterExportedCorrectly(meterName, + List.of("periodicTask", "ClusterHealthTask", "table", "myTable", "tableType", "REALTIME"), + EXPORTED_METRIC_PREFIX); + } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { + assertMeterExportedCorrectly(strippedMeterName + "_" + "ClusterHealthCheck", EXPORTED_METRIC_PREFIX); + } else if (controllerMeter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { + assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); } } } } + From 1f5884618fe801c8f017c052dff285f3c0e69917 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Sun, 6 Oct 2024 17:18:04 +0530 Subject: [PATCH 06/45] WIP --- .../ControllerJMXToPromMetricsTest.java | 198 +++++++++++++----- .../metrics/PinotJMXToPromMetricsTest.java | 60 ++++-- 2 files changed, 182 insertions(+), 76 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 41444344209b..8e366c6e748d 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -4,14 +4,17 @@ import com.yammer.metrics.reporting.JmxReporter; import java.io.IOException; import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.http.HttpClient; +import org.apache.pinot.common.version.PinotVersion; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; -import org.apache.pinot.spi.utils.StringUtil; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -20,6 +23,9 @@ public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; + private static final String TASK_TYPE_KEY = "taskType"; + private static final String TASK_TYPE_VAL = "ClusterHealthCheck"; + private ControllerMetrics _controllerMetrics; @BeforeClass @@ -42,65 +48,145 @@ public void setup() { } @Test - public void controllerTimerTest() - throws IOException, URISyntaxException { - - for (ControllerTimer controllerTimer : ControllerTimer.values()) { - if (controllerTimer.isGlobal()) { - _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS); - } else { - _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - _controllerMetrics.addTimedTableValue(RAW_TABLE_NAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - } - } - - for (ControllerTimer controllerTimer : ControllerTimer.values()) { - if (controllerTimer.isGlobal()) { - assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX); - } else { - assertMeterExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, - EXPORTED_METRIC_PREFIX); - } - } + public void controllerTimerTest() { + + Arrays.stream(ControllerTimer.values()).filter(ControllerTimer::isGlobal) + .peek(controllerTimer -> _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS)) + .forEach( + controllerTimer -> assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX)); + + Arrays.stream(ControllerTimer.values()).filter(controllerTimer -> !controllerTimer.isGlobal()) + .peek(controllerTimer -> { + _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); + _controllerMetrics.addTimedTableValue(RAW_TABLE_NAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); + }).forEach(controllerTimer -> { + assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX); + }); } @Test - public void controllerMeterTest() - throws IOException, URISyntaxException { - for (ControllerMeter controllerMeter : ControllerMeter.values()) { - if (controllerMeter.isGlobal()) { - _controllerMetrics.addMeteredGlobalValue(controllerMeter, 5L); - } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { - _controllerMetrics.addMeteredTableValue("ClusterHealthCheck", controllerMeter, 1L); - } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { - _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + "ClusterHealthTask", controllerMeter, 1L); - } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - _controllerMetrics.addMeteredTableValue("ClusterHealthCheck", controllerMeter, 1L); - } else { - _controllerMetrics.addMeteredTableValue(RAW_TABLE_NAME, controllerMeter, 5L); - } - } - for (ControllerMeter controllerMeter : ControllerMeter.values()) { - String meterName = controllerMeter.getMeterName(); - String strippedMeterName = StringUtils.remove(meterName, "controller"); - if (controllerMeter.isGlobal()) { - assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX); - } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, List.of("table", "ClusterHealthCheck"), EXPORTED_METRIC_PREFIX); - } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, - List.of("periodicTask", "ClusterHealthTask", "table", "myTable", "tableType", "REALTIME"), - EXPORTED_METRIC_PREFIX); - } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - assertMeterExportedCorrectly(strippedMeterName + "_" + "ClusterHealthCheck", EXPORTED_METRIC_PREFIX); - } else if (controllerMeter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { - assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); - } else { - assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + public void controllerMeterTest() { + + Arrays.stream(ControllerMeter.values()).filter(ControllerMeter::isGlobal) + .peek(controllerMeter -> _controllerMetrics.addMeteredGlobalValue(controllerMeter, 5L)) + .forEach(controllerMeter -> { + String meterName = controllerMeter.getMeterName(); + String strippedMeterName = StringUtils.remove(meterName, "controller"); + assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX); + }); + + Arrays.stream(ControllerMeter.values()).filter(controllerMeter -> !controllerMeter.isGlobal()) + .peek(controllerMeter -> { + if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { + _controllerMetrics.addMeteredTableValue(TASK_TYPE_VAL, controllerMeter, 1L); + } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { + _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + TASK_TYPE_VAL, controllerMeter, 1L); + } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { + _controllerMetrics.addMeteredTableValue(TASK_TYPE_VAL, controllerMeter, 1L); + } else { + _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, controllerMeter, 5L); + _controllerMetrics.addMeteredTableValue(RAW_TABLE_NAME, controllerMeter, 5L); + } + }).forEach(controllerMeter -> { + String meterName = controllerMeter.getMeterName(); + String strippedMeterName = StringUtils.remove(meterName, "controller"); + if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { + assertMeterExportedCorrectly(meterName, List.of("table", "ClusterHealthCheck"), EXPORTED_METRIC_PREFIX); + } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { + assertMeterExportedCorrectly(meterName, + List.of("periodicTask", "ClusterHealthTask", "table", "myTable", "tableType", "REALTIME"), + EXPORTED_METRIC_PREFIX); + } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { + assertMeterExportedCorrectly(strippedMeterName + "_" + "ClusterHealthCheck", EXPORTED_METRIC_PREFIX); + } else if (controllerMeter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { + assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + } + }); + } + + @Test + public void controllerGaugeTest() { + //that accept global gauge with suffix + List globalGaugesWithTaskTypeSuffix = + List.of(ControllerGauge.VERSION, ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, + ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, ControllerGauge.NUM_MINION_SUBTASKS_WAITING, + ControllerGauge.NUM_MINION_SUBTASKS_ERROR, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, + ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); + + Arrays.stream(ControllerGauge.values()).filter(ControllerGauge::isGlobal) + .filter(globalGaugesWithTaskTypeSuffix::contains).peek(controllerGauge -> { + if (controllerGauge == ControllerGauge.VERSION) { + _controllerMetrics.setValueOfGlobalGauge(ControllerGauge.VERSION, PinotVersion.VERSION_METRIC_NAME, 1L); + } else { + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE_VAL, 1L); + } + }).filter(controllerGauge -> controllerGauge != ControllerGauge.VERSION).forEach(controllerGauge -> { + try { + String strippedMetricName = getStrippedMetricName(controllerGauge); + assertGaugeExportedCorrectly(strippedMetricName, List.of(TASK_TYPE_KEY, TASK_TYPE_VAL), + EXPORTED_METRIC_PREFIX); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + //remaining guages are set without any suffix + Arrays.stream(ControllerGauge.values()).filter(ControllerGauge::isGlobal) + .filter(controllerGauge -> !globalGaugesWithTaskTypeSuffix.contains(controllerGauge)) + .peek(controllerGauge -> _controllerMetrics.setValueOfGlobalGauge(controllerGauge, 1L)) + .forEach(controllerGauge -> { + try { + String strippedMetricName = getStrippedMetricName(controllerGauge); + assertGaugeExportedCorrectly(strippedMetricName, EXPORTED_METRIC_PREFIX); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + //non-global gauges with accept partition + List nonGlobalGaugesThatAcceptPartition = + List.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS); + + //that accept partition gauge + Stream.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS).peek(controllerGauge -> { + _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, 10_000_000); + }).forEach(controllerGauge -> { + String strippedGaugeName = getStrippedMetricName(controllerGauge); + try { + ArrayList exportedLabels = new ArrayList<>(EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE); + exportedLabels.add("partition"); + exportedLabels.add("3"); + assertGaugeExportedCorrectly(strippedGaugeName, exportedLabels, EXPORTED_METRIC_PREFIX); + } catch (Exception e) { + throw new RuntimeException(e); } - } + }); + + //setOrUpdateTableGauge +// TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE("TimeMsSinceLastMinionTaskMetadataUpdate", false), +// TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION("TimeMsSinceLastSuccessfulMinionTaskGeneration", false), +// LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR("LastMinionTaskGenerationEncountersError", false), + + Stream.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, + ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, + ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR).peek(controllerGauge -> { +// _controllerMetrics.setOrUpdateTableGauge(); + }); + +// setOrUpdateGauge +// LLC_SEGMENTS_DEEP_STORE_UPLOAD_RETRY_QUEUE_SIZE + } + + private static String getStrippedMetricName(ControllerGauge controllerGauge) { + return StringUtils.remove(controllerGauge.getGaugeName(), "controller"); } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index c2c68cfc8606..b2d08aa2965a 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -74,10 +74,10 @@ public class PinotJMXToPromMetricsTest { protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) throws IOException, URISyntaxException { - List promMetrics = - parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - Assert.assertTrue(promMetrics.contains( - PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value"))); + List promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + Assert.assertTrue( + promMetrics.contains(PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value")), + exportedGaugePrefix); } protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, List labels, @@ -89,10 +89,15 @@ protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, List promMetrics = - parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + protected void assertTimerExportedCorrectly(String exportedTimerPrefix, String exportedMetricPrefix) { + List promMetrics = null; + try { + promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } for (String meterType : TIMER_TYPES) { Assert.assertTrue(promMetrics.contains(PromMetric.withName( exportedMetricPrefix + exportedTimerPrefix + "_" + meterType))); @@ -100,20 +105,30 @@ protected void assertTimerExportedCorrectly(String exportedTimerPrefix, String e } protected void assertTimerExportedCorrectly(String exportedTimerPrefix, List labels, - String exportedMetricPrefix) - throws IOException, URISyntaxException { - List promMetrics = - parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + String exportedMetricPrefix) { + List promMetrics; + try { + promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } for (String meterType : METER_TYPES) { Assert.assertTrue(promMetrics.contains(PromMetric.withNameAndLabels( exportedMetricPrefix + exportedTimerPrefix + "_" + meterType, labels))); } } - protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String exportedMetricPrefix) - throws IOException, URISyntaxException { - List promMetrics = - parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String exportedMetricPrefix) { + List promMetrics = null; + try { + promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } for (String meterType : METER_TYPES) { Assert.assertTrue(promMetrics.contains( PromMetric.withName(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType)), @@ -122,10 +137,15 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String e } protected void assertMeterExportedCorrectly(String exportedMeterPrefix, List labels, - String exportedMetricPrefix) - throws IOException, URISyntaxException { - List promMetrics = - parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + String exportedMetricPrefix) { + List promMetrics = null; + try { + promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } for (String meterType : METER_TYPES) { Assert.assertTrue(promMetrics.contains(PromMetric.withNameAndLabels( exportedMetricPrefix + exportedMeterPrefix + "_" + meterType, labels)), exportedMeterPrefix); From 836db8420141c4be65d7301fdc4c6b380bc84d08 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Sun, 13 Oct 2024 19:08:52 +0530 Subject: [PATCH 07/45] WIP --- .../metrics/PinotJMXToPromMetricsTest.java | 8 +- .../metrics/ServerJMXToPromMetricsTest.java | 92 +++++++++++-------- 2 files changed, 59 insertions(+), 41 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index b2d08aa2965a..d7bf069c6381 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -121,12 +121,10 @@ protected void assertTimerExportedCorrectly(String exportedTimerPrefix, List promMetrics = null; + List promMetrics; try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - } catch (IOException e) { - throw new RuntimeException(e); - } catch (URISyntaxException e) { + } catch (Exception e) { throw new RuntimeException(e); } for (String meterType : METER_TYPES) { @@ -138,7 +136,7 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String e protected void assertMeterExportedCorrectly(String exportedMeterPrefix, List labels, String exportedMetricPrefix) { - List promMetrics = null; + List promMetrics; try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); } catch (IOException e) { diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index 9252946cf222..c9e8dbc6c017 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -28,6 +28,7 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import java.util.stream.Stream; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.common.version.PinotVersion; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; @@ -69,8 +70,7 @@ public void setup() { * This test validates each timer defined in {@link ServerTimer} */ @Test - public void serverTimerTest() - throws IOException, URISyntaxException { + public void serverTimerTest() { for (ServerTimer serverTimer : ServerTimer.values()) { if (serverTimer.isGlobal()) { @@ -85,9 +85,9 @@ public void serverTimerTest() if (serverTimer.isGlobal()) { assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_METRIC_PREFIX); } else { - assertMeterExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); } } @@ -97,62 +97,53 @@ public void serverTimerTest() * This test validates each meter defined in {@link ServerMeter} */ @Test - public void serverMeterTest() - throws Exception { + public void serverMeterTest() { //first, assert on all global meters Arrays.stream(ServerMeter.values()).filter(ServerMeter::isGlobal).peek(this::addGlobalMeter) .forEach(serverMeter -> { try { - if (serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS - || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS - || serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS - || serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS) { - String meterName = serverMeter.getMeterName(); - String exportedMeterPrefix = - "realtime_exceptions_" + meterName.substring(0, meterName.lastIndexOf("Exceptions")); - assertMeterExportedCorrectly(exportedMeterPrefix, EXPORTED_METRIC_PREFIX); + //we cannot use raw meter names for all meters as exported metrics don't follow any convention currently. + // For example, meters that track realtime exceptions start with prefix "realtime_exceptions" + if (meterTracksRealtimeExceptions(serverMeter)) { + assertMeterExportedCorrectly(getRealtimeExceptionMeterName(serverMeter)); } else { - assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(serverMeter.getMeterName()); } } catch (Exception e) { throw new RuntimeException(e); } }); - List metersThatAcceptClientId = + //these meters accept the clientId + List metersAcceptingClientId = List.of(ServerMeter.REALTIME_ROWS_CONSUMED, ServerMeter.REALTIME_ROWS_SANITIZED, ServerMeter.REALTIME_ROWS_FETCHED, ServerMeter.REALTIME_ROWS_FILTERED, ServerMeter.INVALID_REALTIME_ROWS_DROPPED, ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, ServerMeter.ROWS_WITH_ERRORS); - List metersThatAcceptRawTableName = + metersAcceptingClientId.stream().peek(meter -> addMeterWithLables(meter, CLIENT_ID)) + .forEach(meter -> assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_CLIENT_ID)); + + //these meters accept raw table name + List metersAcceptingRawTableNames = List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, ServerMeter.SEGMENT_UPLOAD_TIMEOUT); - //remaining all meters accept table name with type - Arrays.stream(ServerMeter.values()).filter(serverMeter -> !serverMeter.isGlobal()).forEach(serverMeter -> { - try { - if (metersThatAcceptClientId.contains(serverMeter)) { - addMeterWithLables(serverMeter, CLIENT_ID); - assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_CLIENT_ID, - EXPORTED_METRIC_PREFIX); - } else if (metersThatAcceptRawTableName.contains(serverMeter)) { - addMeterWithLables(serverMeter, RAW_TABLE_NAME); - assertMeterExportedCorrectly(serverMeter.getMeterName(), List.of("table", RAW_TABLE_NAME), - EXPORTED_METRIC_PREFIX); - } else { - addMeterWithLables(serverMeter, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - } - } catch (Exception e) { + metersAcceptingRawTableNames.stream().peek(meter -> addMeterWithLables(meter, RAW_TABLE_NAME)).forEach(meter -> { + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME); + }); - } + //remaining all meters accept tableNameWithType + Arrays.stream(ServerMeter.values()).filter( + serverMeter -> !serverMeter.isGlobal() && !metersAcceptingRawTableNames.contains(serverMeter) + && !metersAcceptingClientId.contains(serverMeter)).forEach(serverMeter -> { + addMeterWithLables(serverMeter, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE); }); } /** - * This test validates each meter defined in {@link ServerGauge} + * This test validates each gauge defined in {@link ServerGauge} */ @Test public void serverGaugeTest() @@ -166,6 +157,7 @@ public void serverGaugeTest() Supplier someValSupplier = () -> someVal; //global gauges + _serverMetrics.setValueOfGlobalGauge(ServerGauge.VERSION, PinotVersion.VERSION_METRIC_NAME, someVal); _serverMetrics.addValueToGlobalGauge(ServerGauge.LLC_SIMULTANEOUS_SEGMENT_BUILDS, someVal); _serverMetrics.setValueOfGlobalGauge(ServerGauge.JVM_HEAP_USED_BYTES, someVal); @@ -178,6 +170,15 @@ public void serverGaugeTest() _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_CHUNK_SIZE, someValSupplier); _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_THREADLOCALCACHE, someValSupplier); + Stream.of(ServerGauge.values()).peek(gauge -> _serverMetrics.setOrUpdateGlobalGauge(gauge, someValSupplier)) + .forEach(gauge -> { + try { + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + //local gauges _serverMetrics.addValueToTableGauge(TABLE_NAME_WITH_TYPE, ServerGauge.DOCUMENT_COUNT, someVal); _serverMetrics.addValueToTableGauge(TABLE_NAME_WITH_TYPE, ServerGauge.SEGMENT_COUNT, someVal); @@ -276,4 +277,23 @@ public void addGlobalMeter(ServerMeter serverMeter) { public void addMeterWithLables(ServerMeter serverMeter, String label) { _serverMetrics.addMeteredTableValue(label, serverMeter, 4L); } + + private boolean meterTracksRealtimeExceptions(ServerMeter serverMeter) { + return serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS + || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS + || serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS || serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS; + } + + private String getRealtimeExceptionMeterName(ServerMeter serverMeter) { + String meterName = serverMeter.getMeterName(); + return "realtime_exceptions_" + meterName.substring(0, meterName.lastIndexOf("Exceptions")); + } + + private void assertMeterExportedCorrectly(String exportedMeterName) { + assertMeterExportedCorrectly(exportedMeterName, EXPORTED_METRIC_PREFIX); + } + + private void assertMeterExportedCorrectly(String exportedMeterName, List labels) { + assertMeterExportedCorrectly(exportedMeterName, labels, EXPORTED_METRIC_PREFIX); + } } From d938a472fcc94c762b0b13dd8523e16b8431deaf Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Mon, 14 Oct 2024 15:27:07 +0530 Subject: [PATCH 08/45] WIP --- .../metrics/BrokerJMXToPromMetricsTest.java | 102 ++++++++++++ .../ControllerJMXToPromMetricsTest.java | 2 - .../metrics/PinotJMXToPromMetricsTest.java | 64 ++++++-- .../metrics/ServerJMXToPromMetricsTest.java | 148 ++++++------------ 4 files changed, 206 insertions(+), 110 deletions(-) create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java new file mode 100644 index 000000000000..bcfb12c01935 --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java @@ -0,0 +1,102 @@ +package org.apache.pinot.common.metrics; + +import com.yammer.metrics.core.MetricsRegistry; +import com.yammer.metrics.reporting.JmxReporter; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.apache.pinot.common.utils.http.HttpClient; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; +import org.apache.pinot.spi.env.PinotConfiguration; +import org.apache.pinot.spi.metrics.PinotMetricUtils; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; + + +public class BrokerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { + + private static final String EXPORTED_METRIC_PREFIX = "pinot_broker_"; + private BrokerMetrics _brokerMetrics; + + @BeforeClass + public void setup() { + PinotConfiguration pinotConfiguration = new PinotConfiguration(); + pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, + "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); + PinotMetricUtils.init(pinotConfiguration); + + // Initialize ServerMetrics with the registry + YammerMetricsRegistry yammerMetricsRegistry = new YammerMetricsRegistry(); + _brokerMetrics = new BrokerMetrics(yammerMetricsRegistry); + + // Enable JMX reporting + MetricsRegistry metricsRegistry = (MetricsRegistry) yammerMetricsRegistry.getMetricsRegistry(); + JmxReporter jmxReporter = new JmxReporter(metricsRegistry); + jmxReporter.start(); + + _httpClient = new HttpClient(); + } + + @Test + public void brokerTimerTest() { + //first assert on global timers + Stream.of(BrokerTimer.values()).filter(BrokerTimer::isGlobal) + .peek(timer -> _brokerMetrics.addTimedValue(timer, 30_000, TimeUnit.MILLISECONDS)) + .forEach(timer -> assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_METRIC_PREFIX)); + //Assert on local timers + Stream.of(BrokerTimer.values()).filter(timer -> !timer.isGlobal()).peek(timer -> { + _brokerMetrics.addTimedTableValue(RAW_TABLE_NAME, timer, 30_000L, TimeUnit.MILLISECONDS); + }).forEach(timer -> assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX)); + } + + @Test + public void brokerGaugeTest() { + //global gauges + Stream.of(BrokerGauge.values()).filter(BrokerGauge::isGlobal) + .peek(gauge -> _brokerMetrics.setOrUpdateGlobalGauge(gauge, () -> 5L)) + .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX)); + //local gauges + Stream.of(BrokerGauge.values()).filter(gauge -> !gauge.isGlobal()).peek(gauge -> { + if (gauge == BrokerGauge.REQUEST_SIZE) { + _brokerMetrics.setOrUpdateTableGauge(RAW_TABLE_NAME, gauge, 5L); + } else { + _brokerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L); + } + }).forEach(gauge -> { + if (gauge == BrokerGauge.REQUEST_SIZE) { + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + } else { + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + } + }); + } + + @Test + public void brokerMeterTest() { + Stream.of(BrokerMeter.values()).filter(BrokerMeter::isGlobal) + .peek(meter -> _brokerMetrics.addMeteredGlobalValue(meter, 5L)).forEach(meter -> { + try { + if (meter == BrokerMeter.UNCAUGHT_GET_EXCEPTIONS || meter == BrokerMeter.UNCAUGHT_POST_EXCEPTIONS + || meter == BrokerMeter.QUERY_REJECTED_EXCEPTIONS || meter == BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS + || meter == BrokerMeter.RESOURCE_MISSING_EXCEPTIONS) { + assertMeterExportedCorrectly("exceptions" + "_" + StringUtils.remove(meter.getMeterName(), "Exceptions"), + EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Stream.of(BrokerMeter.values()).filter(meter -> !meter.isGlobal()) + .peek(meter -> _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L)).forEach(meter -> { + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + }); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 8e366c6e748d..975e6daa85cb 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -2,8 +2,6 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; -import java.io.IOException; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index d7bf069c6381..e800451cdd89 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -72,21 +72,43 @@ public class PinotJMXToPromMetricsTest { List.of("partition", PARTITION_GROUP_ID, "table", RAW_TABLE_NAME, "tableType", TableType.REALTIME.toString(), "topic", KAFKA_TOPIC); - protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) - throws IOException, URISyntaxException { - List promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - Assert.assertTrue( - promMetrics.contains(PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value")), - exportedGaugePrefix); + protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) { + List promMetrics; + try { + promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + Assert.assertTrue( + promMetrics.contains(PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value")), + exportedGaugePrefix); + } catch (Exception e) { + throw new RuntimeException(e); + } } protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, List labels, - String exportedMetricPrefix) - throws IOException, URISyntaxException { - List promMetrics = - parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - Assert.assertTrue(promMetrics.contains(PromMetric.withNameAndLabels( - exportedMetricPrefix + exportedGaugePrefix + "_" + "Value", labels))); + String exportedMetricPrefix) { + List promMetrics; + try { + promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + Assert.assertTrue(promMetrics.contains( + PromMetric.withNameAndLabels(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value", labels))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected void assertGaugeExportedCorrectly2(String exportedGaugePrefix, List> labels, + String exportedMetricPrefix) { + List promMetrics; + try { + promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + boolean matchesAnyLabelList = false; + for (List labelList : labels) { + matchesAnyLabelList = matchesAnyLabelList || promMetrics.contains( + PromMetric.withNameAndLabels(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value", labelList)); + } Assert.assertTrue(matchesAnyLabelList, exportedGaugePrefix); + } catch (Exception e) { + throw new RuntimeException(e); + } } protected void assertTimerExportedCorrectly(String exportedTimerPrefix, String exportedMetricPrefix) { @@ -150,6 +172,24 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, List> labels, + String exportedMetricPrefix) { + List promMetrics; + try { + promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + } catch (Exception e) { + throw new RuntimeException(e); + } + for (String meterType : METER_TYPES) { + boolean matchesAnyLabel = false; + for (List lableList : labels) { + matchesAnyLabel = matchesAnyLabel || promMetrics.contains( + PromMetric.withNameAndLabels(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType, lableList)); + } + Assert.assertTrue(matchesAnyLabel, exportedMeterPrefix); + } + } + protected List parseExportedPromMetrics(String response) throws IOException { diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index c9e8dbc6c017..2fe1a8cc4464 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -146,32 +146,14 @@ public void serverMeterTest() { * This test validates each gauge defined in {@link ServerGauge} */ @Test - public void serverGaugeTest() - throws Exception { - - //get all exposed metrics before we expose any timers - List promMetricsBefore = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); + public void serverGaugeTest() { int partition = 3; long someVal = 100L; - Supplier someValSupplier = () -> someVal; //global gauges - - _serverMetrics.setValueOfGlobalGauge(ServerGauge.VERSION, PinotVersion.VERSION_METRIC_NAME, someVal); - _serverMetrics.addValueToGlobalGauge(ServerGauge.LLC_SIMULTANEOUS_SEGMENT_BUILDS, someVal); - _serverMetrics.setValueOfGlobalGauge(ServerGauge.JVM_HEAP_USED_BYTES, someVal); - _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_USED_DIRECT_MEMORY, someValSupplier); - _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_USED_HEAP_MEMORY, someValSupplier); - _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_ARENAS_DIRECT, someValSupplier); - _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_ARENAS_HEAP, someValSupplier); - _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_CACHE_SIZE_SMALL, someValSupplier); - _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_CACHE_SIZE_NORMAL, someValSupplier); - _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_CHUNK_SIZE, someValSupplier); - _serverMetrics.setOrUpdateGlobalGauge(ServerGauge.NETTY_POOLED_THREADLOCALCACHE, someValSupplier); - - Stream.of(ServerGauge.values()).peek(gauge -> _serverMetrics.setOrUpdateGlobalGauge(gauge, someValSupplier)) - .forEach(gauge -> { + Stream.of(ServerGauge.values()).filter(ServerGauge::isGlobal) + .peek(gauge -> _serverMetrics.setValueOfGlobalGauge(gauge, 10L)).forEach(gauge -> { try { assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); } catch (Exception e) { @@ -185,89 +167,63 @@ public void serverGaugeTest() _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 2, ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, someVal); _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 2, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, someVal); - //raw table name - _serverMetrics.addValueToTableGauge(RAW_TABLE_NAME, ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, someVal); - _serverMetrics.setValueOfTableGauge(RAW_TABLE_NAME, ServerGauge.REALTIME_MERGED_TEXT_IDX_DOCUMENT_AVG_LEN, someVal); - _serverMetrics.addValueToTableGauge(RAW_TABLE_NAME, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, someVal); - - _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LLC_PARTITION_CONSUMING, someVal); - _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, someVal); - _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, - TimeUnit.MILLISECONDS.toSeconds(someVal)); - _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, - TimeUnit.MILLISECONDS.toSeconds(someVal)); - _serverMetrics.setValueOfTableGauge(CLIENT_ID, - ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS, - TimeUnit.MILLISECONDS.toSeconds(someVal)); - _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, - TimeUnit.MILLISECONDS.toSeconds(someVal)); - _serverMetrics.setValueOfTableGauge(CLIENT_ID, ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS, - TimeUnit.MILLISECONDS.toSeconds(someVal)); - _serverMetrics.setValueOfTableGauge(RAW_TABLE_NAME, ServerGauge.CONSUMPTION_QUOTA_UTILIZATION, someVal); - - _serverMetrics.setValueOfTableGauge(TABLE_STREAM_NAME, ServerGauge.STREAM_DATA_LOSS, 1L); - - _serverMetrics.setOrUpdatePartitionGauge(TABLE_NAME_WITH_TYPE, partition, ServerGauge.REALTIME_INGESTION_DELAY_MS, - someValSupplier); - _serverMetrics.setOrUpdatePartitionGauge(TABLE_NAME_WITH_TYPE, partition, - ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS, someValSupplier); - _serverMetrics.setOrUpdatePartitionGauge(RAW_TABLE_NAME, partition, ServerGauge.LUCENE_INDEXING_DELAY_MS, - someValSupplier); - _serverMetrics.setOrUpdatePartitionGauge(RAW_TABLE_NAME, partition, ServerGauge.LUCENE_INDEXING_DELAY_DOCS, - someValSupplier); - _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, - ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, someVal); - _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, - ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, someVal); - _serverMetrics.setOrUpdatePartitionGauge(TABLE_NAME_WITH_TYPE, partition, ServerGauge.REALTIME_INGESTION_OFFSET_LAG, - someValSupplier); + List gaugesAcceptingClientId = + List.of(ServerGauge.LLC_PARTITION_CONSUMING, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, + ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS); + + gaugesAcceptingClientId.stream().peek(gauge -> { + _serverMetrics.setValueOfTableGauge(CLIENT_ID, gauge, TimeUnit.MILLISECONDS.toSeconds(someVal)); + }).forEach(gauge -> { + try { + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); List labels = List.of("partition", String.valueOf(partition), "table", RAW_TABLE_NAME, "tableType", TableType.REALTIME.toString()); - assertGaugeExportedCorrectly("realtimeIngestionOffsetLag", labels, EXPORTED_METRIC_PREFIX); - - assertGaugeExportedCorrectly("upsertPrimaryKeysInSnapshotCount", labels, EXPORTED_METRIC_PREFIX); - - assertGaugeExportedCorrectly("upsertValidDocIdSnapshotCount", labels, EXPORTED_METRIC_PREFIX); - - assertGaugeExportedCorrectly("endToEndRealtimeIngestionDelayMs", labels, EXPORTED_METRIC_PREFIX); - - assertGaugeExportedCorrectly("realtimeIngestionDelayMs", labels, EXPORTED_METRIC_PREFIX); - -// we only match the metric name for version as the label value is dynamic (based on project version). - Optional pinotServerVersionMetricMaybe = - parseExportedPromMetrics(getExportedPromMetrics().getResponse()).stream() - .filter(exportedMetric -> exportedMetric.getMetricName().contains("version")).findAny(); - - Assert.assertTrue(pinotServerVersionMetricMaybe.isPresent()); - - assertGaugeExportedCorrectly("llcSimultaneousSegmentBuilds", EXPORTED_METRIC_PREFIX); - assertGaugeExportedCorrectly("nettyPooledUsedDirectMemory", EXPORTED_METRIC_PREFIX); - assertGaugeExportedCorrectly("nettyPooledUsedHeapMemory", EXPORTED_METRIC_PREFIX); - assertGaugeExportedCorrectly("nettyPooledUsedHeapMemory", EXPORTED_METRIC_PREFIX); - assertGaugeExportedCorrectly("nettyPooledArenasDirect", EXPORTED_METRIC_PREFIX); - assertGaugeExportedCorrectly("nettyPooledArenasHeap", EXPORTED_METRIC_PREFIX); - assertGaugeExportedCorrectly("nettyPooledCacheSizeSmall", EXPORTED_METRIC_PREFIX); - assertGaugeExportedCorrectly("nettyPooledCacheSizeNormal", EXPORTED_METRIC_PREFIX); - assertGaugeExportedCorrectly("nettyPooledChunkSize", EXPORTED_METRIC_PREFIX); - assertGaugeExportedCorrectly("jvmHeapUsedBytes", EXPORTED_METRIC_PREFIX); - - assertGaugeExportedCorrectly("llcPartitionConsuming", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + List gaugesAcceptingPartition = + List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, + ServerGauge.REALTIME_INGESTION_OFFSET_LAG, ServerGauge.REALTIME_INGESTION_DELAY_MS, + ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, + ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS); - assertGaugeExportedCorrectly("highestStreamOffsetConsumed", EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); + //todo: These metrics are not being exported, add regexps to export them +// ServerGauge.LUCENE_INDEXING_DELAY_MS, +// ServerGauge.LUCENE_INDEXING_DELAY_DOCS - assertGaugeExportedCorrectly("lastRealtimeSegmentCreationWaitTimeSeconds", EXPORTED_LABELS_FOR_CLIENT_ID, - EXPORTED_METRIC_PREFIX); - - assertGaugeExportedCorrectly("lastRealtimeSegmentInitialConsumptionDurationSeconds", EXPORTED_LABELS_FOR_CLIENT_ID, - EXPORTED_METRIC_PREFIX); + gaugesAcceptingPartition.stream() + .peek(gauge -> _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, gauge, someVal)) + .forEach(gauge -> { + try { + assertGaugeExportedCorrectly(gauge.getGaugeName(), labels, EXPORTED_METRIC_PREFIX); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); - assertGaugeExportedCorrectly("lastRealtimeSegmentCatchupDurationSeconds", EXPORTED_LABELS_FOR_CLIENT_ID, - EXPORTED_METRIC_PREFIX); + //todo: Add regexps for these + //REALTIME_OFFHEAP_MEMORY_USED + //REALTIME_SEGMENT_NUM_PARTITIONS - List promMetricsAfter = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - Assert.assertEquals(promMetricsAfter.size() - promMetricsBefore.size(), ServerGauge.values().length); + Stream.of(ServerGauge.values()).filter(gauge -> !gauge.isGlobal()) + .filter(gauge -> (!gaugesAcceptingClientId.contains(gauge) && !gaugesAcceptingPartition.contains(gauge))) + .peek(gauge -> _serverMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, gauge, someVal)) + .filter(gauge -> gauge != ServerGauge.REALTIME_OFFHEAP_MEMORY_USED) + .filter(gauge -> gauge != ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS).forEach(gauge -> { + try { + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } public void addGlobalMeter(ServerMeter serverMeter) { From 18f23e1a8b4112ed6048ea4ee136de3b75830582 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Mon, 14 Oct 2024 15:48:40 +0530 Subject: [PATCH 09/45] Finalise BrokerJMXToPromMetricsTest.java --- .../metrics/BrokerJMXToPromMetricsTest.java | 55 +++++++++++++------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java index bcfb12c01935..b0af87198dcc 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java @@ -19,6 +19,9 @@ public class BrokerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_broker_"; + + private static final String EXPORTED_METRIC_PREFIX_EXCEPTIONS = "exceptions"; + private BrokerMetrics _brokerMetrics; @BeforeClass @@ -78,25 +81,45 @@ public void brokerGaugeTest() { @Test public void brokerMeterTest() { + + List globalMetersWithExceptionsPrefix = + List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, + BrokerMeter.QUERY_REJECTED_EXCEPTIONS, BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, + BrokerMeter.RESOURCE_MISSING_EXCEPTIONS); + Stream.of(BrokerMeter.values()).filter(BrokerMeter::isGlobal) .peek(meter -> _brokerMetrics.addMeteredGlobalValue(meter, 5L)).forEach(meter -> { - try { - if (meter == BrokerMeter.UNCAUGHT_GET_EXCEPTIONS || meter == BrokerMeter.UNCAUGHT_POST_EXCEPTIONS - || meter == BrokerMeter.QUERY_REJECTED_EXCEPTIONS || meter == BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS - || meter == BrokerMeter.RESOURCE_MISSING_EXCEPTIONS) { - assertMeterExportedCorrectly("exceptions" + "_" + StringUtils.remove(meter.getMeterName(), "Exceptions"), - EXPORTED_METRIC_PREFIX); - } else { - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); - } - } catch (Exception e) { - throw new RuntimeException(e); + if (globalMetersWithExceptionsPrefix.contains(meter)) { + String exportedMeterPrefix = String.format("%s_%s", EXPORTED_METRIC_PREFIX_EXCEPTIONS, + StringUtils.remove(meter.getMeterName(), "Exceptions")); + assertMeterExportedCorrectly(exportedMeterPrefix, EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); } }); - Stream.of(BrokerMeter.values()).filter(meter -> !meter.isGlobal()) - .peek(meter -> _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L)).forEach(meter -> { - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - }); + + List localMetersThatAcceptRawTableName = + List.of(BrokerMeter.QUERIES, BrokerMeter.NO_SERVER_FOUND_EXCEPTIONS, BrokerMeter.DOCUMENTS_SCANNED, + BrokerMeter.ENTRIES_SCANNED_IN_FILTER, BrokerMeter.BROKER_RESPONSES_WITH_UNAVAILABLE_SEGMENTS, + BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, + BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, + BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED, BrokerMeter.BROKER_RESPONSES_WITH_TIMEOUTS, + BrokerMeter.ENTRIES_SCANNED_POST_FILTER, BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE, + BrokerMeter.QUERY_QUOTA_EXCEEDED); + + Stream.of(BrokerMeter.values()).filter(meter -> !meter.isGlobal()).peek(meter -> { + if (localMetersThatAcceptRawTableName.contains(meter)) { + _brokerMetrics.addMeteredTableValue(RAW_TABLE_NAME, meter, 5L); + } else { + _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); + } + }).forEach(meter -> { + if (localMetersThatAcceptRawTableName.contains(meter)) { + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + } + }); } } From 4436fe6ee1a83d3205365ba32527bbadad65d9ea Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Mon, 14 Oct 2024 16:55:12 +0530 Subject: [PATCH 10/45] WIP --- .../ControllerJMXToPromMetricsTest.java | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 975e6daa85cb..5ee7c0c2b76c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -22,8 +22,7 @@ public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; private static final String TASK_TYPE_KEY = "taskType"; - private static final String TASK_TYPE_VAL = "ClusterHealthCheck"; - + private static final String TASK_TYPE = "ClusterHealthCheck"; private ControllerMetrics _controllerMetrics; @BeforeClass @@ -47,27 +46,25 @@ public void setup() { @Test public void controllerTimerTest() { - - Arrays.stream(ControllerTimer.values()).filter(ControllerTimer::isGlobal) + Stream.of(ControllerTimer.values()).filter(ControllerTimer::isGlobal) .peek(controllerTimer -> _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS)) .forEach( controllerTimer -> assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX)); - Arrays.stream(ControllerTimer.values()).filter(controllerTimer -> !controllerTimer.isGlobal()) - .peek(controllerTimer -> { - _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - _controllerMetrics.addTimedTableValue(RAW_TABLE_NAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - }).forEach(controllerTimer -> { - assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, - EXPORTED_METRIC_PREFIX); - }); + Stream.of(ControllerTimer.values()).filter(controllerTimer -> !controllerTimer.isGlobal()).peek(controllerTimer -> { + _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); + _controllerMetrics.addTimedTableValue(RAW_TABLE_NAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); + }).forEach(controllerTimer -> { + assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX); + }); } @Test public void controllerMeterTest() { - + //global meters Arrays.stream(ControllerMeter.values()).filter(ControllerMeter::isGlobal) .peek(controllerMeter -> _controllerMetrics.addMeteredGlobalValue(controllerMeter, 5L)) .forEach(controllerMeter -> { @@ -78,12 +75,11 @@ public void controllerMeterTest() { Arrays.stream(ControllerMeter.values()).filter(controllerMeter -> !controllerMeter.isGlobal()) .peek(controllerMeter -> { - if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { - _controllerMetrics.addMeteredTableValue(TASK_TYPE_VAL, controllerMeter, 1L); + if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR + || controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { + _controllerMetrics.addMeteredTableValue(TASK_TYPE, controllerMeter, 1L); } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { - _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + TASK_TYPE_VAL, controllerMeter, 1L); - } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - _controllerMetrics.addMeteredTableValue(TASK_TYPE_VAL, controllerMeter, 1L); + _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + TASK_TYPE, controllerMeter, 1L); } else { _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, controllerMeter, 5L); _controllerMetrics.addMeteredTableValue(RAW_TABLE_NAME, controllerMeter, 5L); @@ -95,7 +91,7 @@ public void controllerMeterTest() { assertMeterExportedCorrectly(meterName, List.of("table", "ClusterHealthCheck"), EXPORTED_METRIC_PREFIX); } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { assertMeterExportedCorrectly(meterName, - List.of("periodicTask", "ClusterHealthTask", "table", "myTable", "tableType", "REALTIME"), + List.of("periodicTask", "ClusterHealthCheck", "table", "myTable", "tableType", "REALTIME"), EXPORTED_METRIC_PREFIX); } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { assertMeterExportedCorrectly(strippedMeterName + "_" + "ClusterHealthCheck", EXPORTED_METRIC_PREFIX); @@ -124,13 +120,12 @@ public void controllerGaugeTest() { if (controllerGauge == ControllerGauge.VERSION) { _controllerMetrics.setValueOfGlobalGauge(ControllerGauge.VERSION, PinotVersion.VERSION_METRIC_NAME, 1L); } else { - _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE_VAL, 1L); + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE, 1L); } }).filter(controllerGauge -> controllerGauge != ControllerGauge.VERSION).forEach(controllerGauge -> { try { String strippedMetricName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedMetricName, List.of(TASK_TYPE_KEY, TASK_TYPE_VAL), - EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(strippedMetricName, List.of(TASK_TYPE_KEY, TASK_TYPE), EXPORTED_METRIC_PREFIX); } catch (Exception e) { throw new RuntimeException(e); } From 4d2d7e15682dc376a48c1e709950d60e7c8f7d3d Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 15 Oct 2024 15:14:13 +0530 Subject: [PATCH 11/45] WIP --- .../configs/server.yml | 12 ++ .../metrics/PinotJMXToPromMetricsTest.java | 62 ++++------ .../metrics/ServerJMXToPromMetricsTest.java | 108 +++++++----------- 3 files changed, 79 insertions(+), 103 deletions(-) diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml index f657a7956e17..8751bfa5170b 100644 --- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml +++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml @@ -132,6 +132,18 @@ rules: labels: database: "$2" table: "$1$3" +- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<>(\\w+)" + name: "pinot_server_luceneIndexingDelayMs_$4" + cache: true + labels: + database: "$2" + table: "$1$3" +- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<>(\\w+)" + name: "pinot_server_luceneIndexingDelayDocs_$4" + cache: true + labels: + database: "$2" + table: "$1$3" - pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<>(\\w+)" name: "pinot_server_numResizes_$5" cache: true diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index e800451cdd89..a41e409bf2c6 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -72,6 +72,9 @@ public class PinotJMXToPromMetricsTest { List.of("partition", PARTITION_GROUP_ID, "table", RAW_TABLE_NAME, "tableType", TableType.REALTIME.toString(), "topic", KAFKA_TOPIC); + protected static final List EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE = + List.of("partition", "3", "table", RAW_TABLE_NAME, "tableType", TableType.REALTIME.toString()); + protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) { List promMetrics; try { @@ -90,56 +93,38 @@ protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, List> labels, - String exportedMetricPrefix) { + protected void assertTimerExportedCorrectly(String exportedTimerPrefix, String exportedMetricPrefix) { List promMetrics; try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - boolean matchesAnyLabelList = false; - for (List labelList : labels) { - matchesAnyLabelList = matchesAnyLabelList || promMetrics.contains( - PromMetric.withNameAndLabels(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value", labelList)); - } Assert.assertTrue(matchesAnyLabelList, exportedGaugePrefix); + for (String meterType : TIMER_TYPES) { + Assert.assertTrue( + promMetrics.contains(PromMetric.withName(exportedMetricPrefix + exportedTimerPrefix + "_" + meterType))); + } } catch (Exception e) { throw new RuntimeException(e); } } - protected void assertTimerExportedCorrectly(String exportedTimerPrefix, String exportedMetricPrefix) { - List promMetrics = null; - try { - promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - } catch (IOException e) { - throw new RuntimeException(e); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - for (String meterType : TIMER_TYPES) { - Assert.assertTrue(promMetrics.contains(PromMetric.withName( - exportedMetricPrefix + exportedTimerPrefix + "_" + meterType))); - } - } - protected void assertTimerExportedCorrectly(String exportedTimerPrefix, List labels, String exportedMetricPrefix) { List promMetrics; try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - } catch (IOException e) { - throw new RuntimeException(e); - } catch (URISyntaxException e) { + for (String meterType : METER_TYPES) { + Assert.assertTrue(promMetrics.contains( + PromMetric.withNameAndLabels(exportedMetricPrefix + exportedTimerPrefix + "_" + meterType, labels))); + } + } catch (Exception e) { throw new RuntimeException(e); } - for (String meterType : METER_TYPES) { - Assert.assertTrue(promMetrics.contains(PromMetric.withNameAndLabels( - exportedMetricPrefix + exportedTimerPrefix + "_" + meterType, labels))); - } } protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String exportedMetricPrefix) { @@ -150,8 +135,8 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String e throw new RuntimeException(e); } for (String meterType : METER_TYPES) { - Assert.assertTrue(promMetrics.contains( - PromMetric.withName(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType)), + Assert.assertTrue( + promMetrics.contains(PromMetric.withName(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType)), exportedMeterPrefix); } } @@ -161,15 +146,14 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, List promMetrics; try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - } catch (IOException e) { - throw new RuntimeException(e); - } catch (URISyntaxException e) { + for (String meterType : METER_TYPES) { + Assert.assertTrue(promMetrics.contains( + PromMetric.withNameAndLabels(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType, labels)), + exportedMeterPrefix); + } + } catch (Exception e) { throw new RuntimeException(e); } - for (String meterType : METER_TYPES) { - Assert.assertTrue(promMetrics.contains(PromMetric.withNameAndLabels( - exportedMetricPrefix + exportedMeterPrefix + "_" + meterType, labels)), exportedMeterPrefix); - } } protected void assertMeterExportedCorrectly2(String exportedMeterPrefix, List> labels, diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index 2fe1a8cc4464..fb9c16adfdce 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -101,16 +101,12 @@ public void serverMeterTest() { //first, assert on all global meters Arrays.stream(ServerMeter.values()).filter(ServerMeter::isGlobal).peek(this::addGlobalMeter) .forEach(serverMeter -> { - try { - //we cannot use raw meter names for all meters as exported metrics don't follow any convention currently. - // For example, meters that track realtime exceptions start with prefix "realtime_exceptions" - if (meterTracksRealtimeExceptions(serverMeter)) { - assertMeterExportedCorrectly(getRealtimeExceptionMeterName(serverMeter)); - } else { - assertMeterExportedCorrectly(serverMeter.getMeterName()); - } - } catch (Exception e) { - throw new RuntimeException(e); + //we cannot use raw meter names for all meters as exported metrics don't follow any convention currently. + // For example, meters that track realtime exceptions start with prefix "realtime_exceptions" + if (meterTracksRealtimeExceptions(serverMeter)) { + assertMeterExportedCorrectly(getRealtimeExceptionMeterName(serverMeter)); + } else { + assertMeterExportedCorrectly(serverMeter.getMeterName()); } }); @@ -153,20 +149,11 @@ public void serverGaugeTest() { //global gauges Stream.of(ServerGauge.values()).filter(ServerGauge::isGlobal) - .peek(gauge -> _serverMetrics.setValueOfGlobalGauge(gauge, 10L)).forEach(gauge -> { - try { - assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); + .peek(gauge -> _serverMetrics.setValueOfGlobalGauge(gauge, 10L)) + .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX)); //local gauges - _serverMetrics.addValueToTableGauge(TABLE_NAME_WITH_TYPE, ServerGauge.DOCUMENT_COUNT, someVal); - _serverMetrics.addValueToTableGauge(TABLE_NAME_WITH_TYPE, ServerGauge.SEGMENT_COUNT, someVal); - _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 2, ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, someVal); - _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 2, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, someVal); - + //gauges that accept clientId List gaugesAcceptingClientId = List.of(ServerGauge.LLC_PARTITION_CONSUMING, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, @@ -175,55 +162,48 @@ public void serverGaugeTest() { ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS); - gaugesAcceptingClientId.stream().peek(gauge -> { - _serverMetrics.setValueOfTableGauge(CLIENT_ID, gauge, TimeUnit.MILLISECONDS.toSeconds(someVal)); - }).forEach(gauge -> { - try { - assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_CLIENT_ID, EXPORTED_METRIC_PREFIX); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - List labels = List.of("partition", String.valueOf(partition), "table", RAW_TABLE_NAME, "tableType", - TableType.REALTIME.toString()); + gaugesAcceptingClientId.stream() + .peek(gauge -> _serverMetrics.setValueOfTableGauge(CLIENT_ID, gauge, TimeUnit.MILLISECONDS.toSeconds(someVal))) + .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_CLIENT_ID, + EXPORTED_METRIC_PREFIX)); + //gauges accepting partition List gaugesAcceptingPartition = List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, ServerGauge.REALTIME_INGESTION_OFFSET_LAG, ServerGauge.REALTIME_INGESTION_DELAY_MS, - ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, - ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS); - - //todo: These metrics are not being exported, add regexps to export them -// ServerGauge.LUCENE_INDEXING_DELAY_MS, -// ServerGauge.LUCENE_INDEXING_DELAY_DOCS + ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS); gaugesAcceptingPartition.stream() .peek(gauge -> _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, gauge, someVal)) - .forEach(gauge -> { - try { - assertGaugeExportedCorrectly(gauge.getGaugeName(), labels, EXPORTED_METRIC_PREFIX); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - //todo: Add regexps for these - //REALTIME_OFFHEAP_MEMORY_USED - //REALTIME_SEGMENT_NUM_PARTITIONS - - Stream.of(ServerGauge.values()).filter(gauge -> !gauge.isGlobal()) - .filter(gauge -> (!gaugesAcceptingClientId.contains(gauge) && !gaugesAcceptingPartition.contains(gauge))) - .peek(gauge -> _serverMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, gauge, someVal)) - .filter(gauge -> gauge != ServerGauge.REALTIME_OFFHEAP_MEMORY_USED) - .filter(gauge -> gauge != ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS).forEach(gauge -> { - try { - assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); + .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), + EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE, EXPORTED_METRIC_PREFIX)); + + //gauges accepting raw table name + List gaugesAcceptingRawTableName = + List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, + ServerGauge.LUCENE_INDEXING_DELAY_MS, ServerGauge.LUCENE_INDEXING_DELAY_DOCS); + + gaugesAcceptingRawTableName.stream().peek(gauge -> _serverMetrics.setValueOfTableGauge(RAW_TABLE_NAME, gauge, 5L)) + .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX)); + + //all remaining gauges + Stream.of(ServerGauge.values()).filter(gauge -> !gauge.isGlobal()).filter( + gauge -> (!gaugesAcceptingClientId.contains(gauge) && !gaugesAcceptingPartition.contains(gauge) + && !gaugesAcceptingRawTableName.contains(gauge))) + .peek(gauge -> _serverMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, gauge, someVal)).forEach( + gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX)); + + //this gauge is currently exported as: `pinot_server_3_Value{database="dedupPrimaryKeysCount", + // table="dedupPrimaryKeysCount.myTable",tableType="REALTIME",}`. We add an explicit test for it to maintain + // backward compatibility. todo: ServerGauge.DEDUP_PRIMARY_KEYS_COUNT should be moved to + // gaugesThatAcceptPartition. It should be exported as: `pinot_server_dedupPrimaryKeysCount_Value{partition="3", + // table="myTable",tableType="REALTIME",}` + _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 4, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, 5L); + assertGaugeExportedCorrectly("4", + List.of("database", "dedupPrimaryKeysCount", "table", "dedupPrimaryKeysCount.myTable", "tableType", "REALTIME"), + EXPORTED_METRIC_PREFIX); } public void addGlobalMeter(ServerMeter serverMeter) { From 37462acace795483403b63e5a1e3e5a4d0cc69e8 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 15 Oct 2024 16:15:32 +0530 Subject: [PATCH 12/45] Controller gauge test --- .../ControllerJMXToPromMetricsTest.java | 122 ++++++++++-------- .../metrics/MinionJMXToPromMetricsTest.java | 51 ++++++++ .../metrics/PinotJMXToPromMetricsTest.java | 28 ++-- .../metrics/ServerJMXToPromMetricsTest.java | 15 +-- 4 files changed, 132 insertions(+), 84 deletions(-) create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 5ee7c0c2b76c..aff774f0f7b0 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -8,6 +8,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import org.apache.helix.task.TaskState; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.common.version.PinotVersion; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; @@ -46,11 +47,13 @@ public void setup() { @Test public void controllerTimerTest() { + //global timers Stream.of(ControllerTimer.values()).filter(ControllerTimer::isGlobal) .peek(controllerTimer -> _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS)) .forEach( controllerTimer -> assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX)); + //local timers Stream.of(ControllerTimer.values()).filter(controllerTimer -> !controllerTimer.isGlobal()).peek(controllerTimer -> { _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); _controllerMetrics.addTimedTableValue(RAW_TABLE_NAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); @@ -69,10 +72,14 @@ public void controllerMeterTest() { .peek(controllerMeter -> _controllerMetrics.addMeteredGlobalValue(controllerMeter, 5L)) .forEach(controllerMeter -> { String meterName = controllerMeter.getMeterName(); + //some meters contain a "controller" prefix. For example, controllerInstancePostError. These meters are + // exported as 'pinot_controller_pinot_controller_InstancePostError'. So we strip the 'controller' from + // 'controllerInstancePostError' String strippedMeterName = StringUtils.remove(meterName, "controller"); assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX); }); + //local meters Arrays.stream(ControllerMeter.values()).filter(controllerMeter -> !controllerMeter.isGlobal()) .peek(controllerMeter -> { if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR @@ -109,73 +116,78 @@ public void controllerMeterTest() { @Test public void controllerGaugeTest() { //that accept global gauge with suffix - List globalGaugesWithTaskTypeSuffix = - List.of(ControllerGauge.VERSION, ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, - ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, ControllerGauge.NUM_MINION_SUBTASKS_WAITING, - ControllerGauge.NUM_MINION_SUBTASKS_ERROR, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, - ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); - - Arrays.stream(ControllerGauge.values()).filter(ControllerGauge::isGlobal) - .filter(globalGaugesWithTaskTypeSuffix::contains).peek(controllerGauge -> { - if (controllerGauge == ControllerGauge.VERSION) { - _controllerMetrics.setValueOfGlobalGauge(ControllerGauge.VERSION, PinotVersion.VERSION_METRIC_NAME, 1L); - } else { - _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE, 1L); - } - }).filter(controllerGauge -> controllerGauge != ControllerGauge.VERSION).forEach(controllerGauge -> { - try { - String strippedMetricName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedMetricName, List.of(TASK_TYPE_KEY, TASK_TYPE), EXPORTED_METRIC_PREFIX); - } catch (Exception e) { - throw new RuntimeException(e); - } + List globalGaugesWithTaskType = + List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, + ControllerGauge.NUM_MINION_SUBTASKS_WAITING, ControllerGauge.NUM_MINION_SUBTASKS_ERROR, + ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); + + Arrays.stream(ControllerGauge.values()).filter(ControllerGauge::isGlobal).filter(globalGaugesWithTaskType::contains) + .peek(controllerGauge -> _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE, 1L)) + .filter(controllerGauge -> controllerGauge != ControllerGauge.VERSION).forEach(controllerGauge -> { + String strippedMetricName = getStrippedMetricName(controllerGauge); + assertGaugeExportedCorrectly(strippedMetricName, List.of(TASK_TYPE_KEY, TASK_TYPE), EXPORTED_METRIC_PREFIX); }); //remaining guages are set without any suffix Arrays.stream(ControllerGauge.values()).filter(ControllerGauge::isGlobal) - .filter(controllerGauge -> !globalGaugesWithTaskTypeSuffix.contains(controllerGauge)) + .filter(controllerGauge -> !globalGaugesWithTaskType.contains(controllerGauge)) .peek(controllerGauge -> _controllerMetrics.setValueOfGlobalGauge(controllerGauge, 1L)) .forEach(controllerGauge -> { - try { - String strippedMetricName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedMetricName, EXPORTED_METRIC_PREFIX); - } catch (Exception e) { - throw new RuntimeException(e); - } + String strippedMetricName = getStrippedMetricName(controllerGauge); + assertGaugeExportedCorrectly(strippedMetricName, EXPORTED_METRIC_PREFIX); }); - //non-global gauges with accept partition - List nonGlobalGaugesThatAcceptPartition = + //local gauges that accept partition + List gaugesAcceptingPartition = List.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS); - //that accept partition gauge - Stream.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS).peek(controllerGauge -> { - _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, 10_000_000); - }).forEach(controllerGauge -> { - String strippedGaugeName = getStrippedMetricName(controllerGauge); - try { - ArrayList exportedLabels = new ArrayList<>(EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE); - exportedLabels.add("partition"); - exportedLabels.add("3"); - assertGaugeExportedCorrectly(strippedGaugeName, exportedLabels, EXPORTED_METRIC_PREFIX); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - //setOrUpdateTableGauge -// TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE("TimeMsSinceLastMinionTaskMetadataUpdate", false), -// TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION("TimeMsSinceLastSuccessfulMinionTaskGeneration", false), -// LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR("LastMinionTaskGenerationEncountersError", false), + gaugesAcceptingPartition.stream().peek( + controllerGauge -> _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, + 10L)) + .forEach(controllerGauge -> { + String strippedGaugeName = getStrippedMetricName(controllerGauge); + ArrayList exportedLabels = new ArrayList<>(EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE); + assertGaugeExportedCorrectly(strippedGaugeName, exportedLabels, EXPORTED_METRIC_PREFIX); + }); - Stream.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, - ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, - ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR).peek(controllerGauge -> { -// _controllerMetrics.setOrUpdateTableGauge(); - }); + //these accept task type + List gaugesAcceptingTaskType = + List.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, + ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, + ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR); -// setOrUpdateGauge -// LLC_SEGMENTS_DEEP_STORE_UPLOAD_RETRY_QUEUE_SIZE + gaugesAcceptingTaskType.stream() + .peek(gauge -> _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, TASK_TYPE, gauge, () -> 50L)) + .forEach( + gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_TABLETYPE_TASKTYPE, + EXPORTED_METRIC_PREFIX)); + + List gaugesAcceptingRawTableName = List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); + gaugesAcceptingRawTableName.stream() + .peek(gauge -> _controllerMetrics.setValueOfTableGauge(RAW_TABLE_NAME, gauge, 5L)).forEach( + gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX)); + + //ad-hoc + _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TABLE_NAME_WITH_TYPE, TASK_TYPE), + ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED, 5L); + assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(), + EXPORTED_LABELS_FOR_TABLENAMEANDTYPE_AND_TASKTYPE, EXPORTED_METRIC_PREFIX); + + _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TASK_TYPE, TaskState.IN_PROGRESS), + ControllerGauge.TASK_STATUS, 5); + assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), EXPORTED_LABELS_FOR_TASK_TYPE_AND_STATUS, + EXPORTED_METRIC_PREFIX); + + //all remaining gauges + Stream.of(ControllerGauge.values()).filter( + gauge -> !gauge.isGlobal() && !gaugesAcceptingPartition.contains(gauge) && !gaugesAcceptingTaskType.contains( + gauge) && !gaugesAcceptingRawTableName.contains(gauge)) + .filter(gauge -> gauge != ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED && gauge != ControllerGauge.TASK_STATUS) + .peek(gauge -> _controllerMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L)).forEach(gauge -> { + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + }); } private static String getStrippedMetricName(ControllerGauge controllerGauge) { diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java new file mode 100644 index 000000000000..ce3f74a30b07 --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java @@ -0,0 +1,51 @@ +package org.apache.pinot.common.metrics; + +import com.yammer.metrics.core.MetricsRegistry; +import com.yammer.metrics.reporting.JmxReporter; +import org.apache.pinot.common.utils.http.HttpClient; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; +import org.apache.pinot.spi.env.PinotConfiguration; +import org.apache.pinot.spi.metrics.PinotMetricUtils; +import org.testng.annotations.BeforeClass; + +import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; + + +public class MinionJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { + + private static final String EXPORTED_METRIC_PREFIX = "pinot_minion_"; + private static final String TASK_TYPE_KEY = "taskType"; + private static final String TASK_TYPE = "ClusterHealthCheck"; + private MinionMetrics _minionMetrics; + + @BeforeClass + public void setup() { + PinotConfiguration pinotConfiguration = new PinotConfiguration(); + pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, + "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); + PinotMetricUtils.init(pinotConfiguration); + + // Initialize ControllerMetrics with the registry + YammerMetricsRegistry yammerMetricsRegistry = new YammerMetricsRegistry(); + _minionMetrics = new MinionMetrics(yammerMetricsRegistry); + + // Enable JMX reporting + MetricsRegistry metricsRegistry = (MetricsRegistry) yammerMetricsRegistry.getMetricsRegistry(); + JmxReporter jmxReporter = new JmxReporter(metricsRegistry); + jmxReporter.start(); + + _httpClient = new HttpClient(); + } + + public void timerTest() { + + } + + public void meterTest() { + + } + + public void gaugeTest() { + + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index a41e409bf2c6..045836606ece 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -37,6 +37,7 @@ import org.apache.pinot.spi.utils.StringUtil; import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; +import org.testng.annotations.Test; public class PinotJMXToPromMetricsTest { @@ -75,6 +76,15 @@ public class PinotJMXToPromMetricsTest { protected static final List EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE = List.of("partition", "3", "table", RAW_TABLE_NAME, "tableType", TableType.REALTIME.toString()); + protected static final List EXPORTED_LABELS_FOR_TABLE_TABLETYPE_TASKTYPE = + List.of("table", "myTable", "tableType", "REALTIME", "taskType", "ClusterHealthCheck"); + + protected static final List EXPORTED_LABELS_FOR_TABLENAMEANDTYPE_AND_TASKTYPE = + List.of("table", "myTable_REALTIME", "taskType", "ClusterHealthCheck"); + + protected static final List EXPORTED_LABELS_FOR_TASK_TYPE_AND_STATUS = + List.of("status", "IN_PROGRESS", "taskType", "ClusterHealthCheck"); + protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) { List promMetrics; try { @@ -156,24 +166,6 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, List> labels, - String exportedMetricPrefix) { - List promMetrics; - try { - promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - } catch (Exception e) { - throw new RuntimeException(e); - } - for (String meterType : METER_TYPES) { - boolean matchesAnyLabel = false; - for (List lableList : labels) { - matchesAnyLabel = matchesAnyLabel || promMetrics.contains( - PromMetric.withNameAndLabels(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType, lableList)); - } - Assert.assertTrue(matchesAnyLabel, exportedMeterPrefix); - } - } - protected List parseExportedPromMetrics(String response) throws IOException { diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index fb9c16adfdce..cd047125703c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -21,21 +21,14 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; -import java.io.IOException; -import java.net.URISyntaxException; import java.util.Arrays; import java.util.List; -import java.util.Optional; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import java.util.stream.Stream; import org.apache.pinot.common.utils.http.HttpClient; -import org.apache.pinot.common.version.PinotVersion; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; -import org.apache.pinot.spi.config.table.TableType; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; -import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -103,7 +96,7 @@ public void serverMeterTest() { .forEach(serverMeter -> { //we cannot use raw meter names for all meters as exported metrics don't follow any convention currently. // For example, meters that track realtime exceptions start with prefix "realtime_exceptions" - if (meterTracksRealtimeExceptions(serverMeter)) { + if (meterTrackingRealtimeExceptions(serverMeter)) { assertMeterExportedCorrectly(getRealtimeExceptionMeterName(serverMeter)); } else { assertMeterExportedCorrectly(serverMeter.getMeterName()); @@ -200,8 +193,8 @@ public void serverGaugeTest() { // backward compatibility. todo: ServerGauge.DEDUP_PRIMARY_KEYS_COUNT should be moved to // gaugesThatAcceptPartition. It should be exported as: `pinot_server_dedupPrimaryKeysCount_Value{partition="3", // table="myTable",tableType="REALTIME",}` - _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 4, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, 5L); - assertGaugeExportedCorrectly("4", + _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, 5L); + assertGaugeExportedCorrectly(String.valueOf(partition), List.of("database", "dedupPrimaryKeysCount", "table", "dedupPrimaryKeysCount.myTable", "tableType", "REALTIME"), EXPORTED_METRIC_PREFIX); } @@ -214,7 +207,7 @@ public void addMeterWithLables(ServerMeter serverMeter, String label) { _serverMetrics.addMeteredTableValue(label, serverMeter, 4L); } - private boolean meterTracksRealtimeExceptions(ServerMeter serverMeter) { + private boolean meterTrackingRealtimeExceptions(ServerMeter serverMeter) { return serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS || serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS || serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS; From ea7b3289c356f5ed16248728559275d05a9dd7ba Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 15 Oct 2024 16:21:01 +0530 Subject: [PATCH 13/45] WIP --- .../pinot/common/metrics/ControllerJMXToPromMetricsTest.java | 5 ++--- .../pinot/common/metrics/PinotJMXToPromMetricsTest.java | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index aff774f0f7b0..20e989350852 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -97,11 +97,10 @@ public void controllerMeterTest() { if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { assertMeterExportedCorrectly(meterName, List.of("table", "ClusterHealthCheck"), EXPORTED_METRIC_PREFIX); } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, - List.of("periodicTask", "ClusterHealthCheck", "table", "myTable", "tableType", "REALTIME"), + assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_PERIODIC_TASK_TABLE_TABLETYPE, EXPORTED_METRIC_PREFIX); } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - assertMeterExportedCorrectly(strippedMeterName + "_" + "ClusterHealthCheck", EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(String.format("%s_%s", strippedMeterName, TASK_TYPE), EXPORTED_METRIC_PREFIX); } else if (controllerMeter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 045836606ece..c9a7ac48fb4c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -85,6 +85,9 @@ public class PinotJMXToPromMetricsTest { protected static final List EXPORTED_LABELS_FOR_TASK_TYPE_AND_STATUS = List.of("status", "IN_PROGRESS", "taskType", "ClusterHealthCheck"); + protected static final List EXPORTED_LABELS_PERIODIC_TASK_TABLE_TABLETYPE = + List.of("periodicTask", "ClusterHealthCheck", "table", "myTable", "tableType", "REALTIME"); + protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) { List promMetrics; try { From 01d2aee23cfaeea939f330d7ea74a98d56fd08a3 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 15 Oct 2024 17:17:51 +0530 Subject: [PATCH 14/45] WIP --- pinot-common/pom.xml | 2 + .../ControllerJMXToPromMetricsTest.java | 11 +++-- .../metrics/MinionJMXToPromMetricsTest.java | 41 +++++++++++++++++-- .../metrics/PinotJMXToPromMetricsTest.java | 2 - 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index f2dbc427a000..a3136a85783d 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -52,6 +52,8 @@ false + @{argLine} + -javaagent:/Users/prashantpandey/Downloads/jmx_prometheus_javaagent-0.19.0.jar=9021:/Users/prashantpandey/StarTree/projects/pinot-all-3/pinot-all/pinot/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 20e989350852..3cb2d87346a7 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -10,7 +10,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.helix.task.TaskState; import org.apache.pinot.common.utils.http.HttpClient; -import org.apache.pinot.common.version.PinotVersion; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; @@ -180,8 +179,7 @@ public void controllerGaugeTest() { //all remaining gauges Stream.of(ControllerGauge.values()).filter( - gauge -> !gauge.isGlobal() && !gaugesAcceptingPartition.contains(gauge) && !gaugesAcceptingTaskType.contains( - gauge) && !gaugesAcceptingRawTableName.contains(gauge)) + gauge -> isRemaining(gauge, gaugesAcceptingPartition, gaugesAcceptingTaskType, gaugesAcceptingRawTableName)) .filter(gauge -> gauge != ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED && gauge != ControllerGauge.TASK_STATUS) .peek(gauge -> _controllerMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L)).forEach(gauge -> { assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, @@ -189,8 +187,13 @@ public void controllerGaugeTest() { }); } + private boolean isRemaining(ControllerGauge gauge, List gaugesAcceptingPartition, + List gaugesAcceptingTaskType, List gaugesAcceptingRawTableName) { + return !gauge.isGlobal() && !gaugesAcceptingPartition.contains(gauge) && !gaugesAcceptingTaskType.contains(gauge) + && !gaugesAcceptingRawTableName.contains(gauge); + } + private static String getStrippedMetricName(ControllerGauge controllerGauge) { return StringUtils.remove(controllerGauge.getGaugeName(), "controller"); } } - diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java index ce3f74a30b07..9af0e388c4e9 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java @@ -2,11 +2,14 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; +import java.util.List; +import java.util.stream.Stream; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; @@ -15,7 +18,7 @@ public class MinionJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_minion_"; private static final String TASK_TYPE_KEY = "taskType"; - private static final String TASK_TYPE = "ClusterHealthCheck"; + private static final String TASK_TYPE = "SegmentImportTask"; private MinionMetrics _minionMetrics; @BeforeClass @@ -37,15 +40,47 @@ public void setup() { _httpClient = new HttpClient(); } + @Test public void timerTest() { - } + @Test public void meterTest() { + //global meters + Stream.of(MinionMeter.values()).filter(MinionMeter::isGlobal) + .peek(meter -> _minionMetrics.addMeteredGlobalValue(meter, 5L)) + .forEach(meter -> assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX)); + //local meters + Stream.of(MinionMeter.values()).filter(meter -> !meter.isGlobal()) + .filter(meter -> meter.getMeterName().startsWith("numberTasks")).peek(meter -> { + _minionMetrics.addMeteredTableValue(RAW_TABLE_NAME, meter, 1L); + _minionMetrics.addMeteredValue(TASK_TYPE, meter, 1L); + }).forEach(meter -> { + assertMeterExportedCorrectly(meter.getMeterName(), List.of("id", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), List.of("id", TASK_TYPE), EXPORTED_METRIC_PREFIX); + }); + + List metersAcceptingTableNameWithType = + List.of(MinionMeter.SEGMENT_UPLOAD_FAIL_COUNT, MinionMeter.SEGMENT_DOWNLOAD_FAIL_COUNT); + metersAcceptingTableNameWithType.stream() + .peek(meter -> _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 1L)).forEach( + meter -> assertMeterExportedCorrectly(meter.getMeterName(), List.of("id", TABLE_NAME_WITH_TYPE), + EXPORTED_METRIC_PREFIX)); + + Stream.of(MinionMeter.values()).filter(meter -> !meter.isGlobal()) + .filter(meter -> !meter.getMeterName().startsWith("numberTasks")) + .filter(meter -> !metersAcceptingTableNameWithType.contains(meter)).peek(meter -> { + _minionMetrics.addMeteredGlobalValue(meter, 1L); + _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, TASK_TYPE, meter, 1L); + }).forEach(meter -> { + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), + List.of("table", RAW_TABLE_NAME, "tableType", "REALTIME", "taskType", TASK_TYPE), EXPORTED_METRIC_PREFIX); + }); } + @Test public void gaugeTest() { - } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index c9a7ac48fb4c..0a0c483f4d27 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -34,10 +34,8 @@ import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.spi.config.table.TableType; -import org.apache.pinot.spi.utils.StringUtil; import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; -import org.testng.annotations.Test; public class PinotJMXToPromMetricsTest { From 04e7c8e8c2fe4bdefb146ec9534258c4eea5c928 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 15 Oct 2024 19:47:05 +0530 Subject: [PATCH 15/45] Agent working --- pinot-common/pom.xml | 38 ++++++++++++++++++- .../metrics/BrokerJMXToPromMetricsTest.java | 19 ++++++++++ .../ControllerJMXToPromMetricsTest.java | 19 ++++++++++ .../metrics/MinionJMXToPromMetricsTest.java | 19 ++++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index a3136a85783d..049a67962e4c 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -52,8 +52,7 @@ false - @{argLine} - -javaagent:/Users/prashantpandey/Downloads/jmx_prometheus_javaagent-0.19.0.jar=9021:/Users/prashantpandey/StarTree/projects/pinot-all-3/pinot-all/pinot/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml + -javaagent:${project.build.directory}/agent/jmx_prometheus_javaagent-0.19.0.jar=9021:${project.basedir}/../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml @@ -109,6 +108,12 @@ + + io.prometheus.jmx + jmx_prometheus_javaagent + 0.19.0 + test + org.apache.pinot pinot-segment-spi @@ -341,6 +346,35 @@ package + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy + package + + copy + + + + + + + io.prometheus.jmx + jmx_prometheus_javaagent + 0.19.0 + jar + false + ${project.build.directory}/agent + + + + + + + false -javaagent:${project.build.directory}/agent/jmx_prometheus_javaagent-0.19.0.jar=9021:${project.basedir}/../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml @@ -112,7 +112,6 @@ io.prometheus.jmx jmx_prometheus_javaagent 0.19.0 - test org.apache.pinot @@ -276,6 +275,12 @@ equalsverifier test + + ai.startree.pinot + startree-confluent-common + 1.3.0-SNAPSHOT + test + @@ -337,15 +342,11 @@ - - build-shaded-jar + download-test-dependencies true - - package - @@ -354,7 +355,7 @@ copy - package + test-compile copy @@ -376,6 +377,16 @@ + + + build-shaded-jar + + true + + + package + + diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java index c6100b73bf4b..1a8f855a016c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java @@ -21,10 +21,14 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; +import io.prometheus.jmx.JavaAgent; +import java.net.URI; import java.util.List; +import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; @@ -43,8 +47,18 @@ public class BrokerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private BrokerMetrics _brokerMetrics; + private int _exporterPort; + @BeforeClass - public void setup() { + public void setup() + throws Exception { + + _exporterPort = 9000 + new Random().nextInt(1000); + String agentArgs = String.format("%s:%s", _exporterPort, + "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml"); + + JavaAgent.agentmain(agentArgs, null); + PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); @@ -141,4 +155,13 @@ public void brokerMeterTest() { } }); } + + @Override + protected SimpleHttpResponse getExportedPromMetrics() { + try { + return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 130fe376a26f..9e64cf3f4d8d 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -21,14 +21,27 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; +import io.prometheus.jmx.BuildInfoCollector; import io.prometheus.jmx.JavaAgent; +import io.prometheus.jmx.JmxCollector; +import io.prometheus.jmx.common.http.ConfigurationException; +import io.prometheus.jmx.common.http.HTTPServerFactory; +import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry; +import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; +import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.helix.task.TaskState; +import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; @@ -45,10 +58,33 @@ public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String TASK_TYPE = "ClusterHealthCheck"; private ControllerMetrics _controllerMetrics; + private int _exporterPort; + + private HTTPServer _httpServer; + @BeforeClass public void setup() throws Exception { + _exporterPort = 9000 + new Random().nextInt(1000); + String agentArgs = String.format("%s:%s", _exporterPort, + "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml"); + + String host = "0.0.0.0"; + + try { + JMXExporterConfig config = parseConfig(agentArgs, host); + CollectorRegistry registry = new CollectorRegistry(); + (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry); + DefaultExports.register(registry); + _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file)); + } catch (ConfigurationException var4) { + System.err.println("Configuration Exception : " + var4.getMessage()); + System.exit(1); + } catch (IllegalArgumentException var5) { + System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]: " + var5.getMessage()); + System.exit(1); + } PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, @@ -219,4 +255,13 @@ private boolean isRemaining(ControllerGauge gauge, List gaugesA private static String getStrippedMetricName(ControllerGauge controllerGauge) { return StringUtils.remove(controllerGauge.getGaugeName(), "controller"); } + + @Override + protected SimpleHttpResponse getExportedPromMetrics() { + try { + return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java index 34abd0f39b80..f0a416bfad2a 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java @@ -23,6 +23,7 @@ import com.yammer.metrics.reporting.JmxReporter; import java.util.List; import java.util.stream.Stream; +import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; @@ -102,4 +103,9 @@ public void meterTest() { @Test public void gaugeTest() { } + + @Override + protected SimpleHttpResponse getExportedPromMetrics() { + return null; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 0a0c483f4d27..5566f6bd39ce 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -20,15 +20,19 @@ package org.apache.pinot.common.metrics; import com.google.common.base.Objects; +import io.prometheus.jmx.JavaAgent; import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; @@ -38,7 +42,7 @@ import org.testng.Assert; -public class PinotJMXToPromMetricsTest { +public abstract class PinotJMXToPromMetricsTest { protected HttpClient _httpClient; @@ -184,10 +188,7 @@ protected List parseExportedPromMetrics(String response) return exportedPromMetrics; } - protected SimpleHttpResponse getExportedPromMetrics() - throws IOException, URISyntaxException { - return _httpClient.sendGetRequest(new URI("http://localhost:9021/metrics")); - } + protected abstract SimpleHttpResponse getExportedPromMetrics(); public static class PromMetric { private final String _metricName; @@ -278,4 +279,40 @@ public String toString() { return sb.toString(); } } + + public static JMXExporterConfig parseConfig(String args, String ifc) { + Pattern pattern = Pattern.compile("^(?:((?:[\\w.-]+)|(?:\\[.+])):)?(\\d{1,5}):(.+)"); + Matcher matcher = pattern.matcher(args); + if (!matcher.matches()) { + throw new IllegalArgumentException("Malformed arguments - " + args); + } else { + String givenHost = matcher.group(1); + String givenPort = matcher.group(2); + String givenConfigFile = matcher.group(3); + int port = Integer.parseInt(givenPort); + InetSocketAddress socket; + if (givenHost != null && !givenHost.isEmpty()) { + socket = new InetSocketAddress(givenHost, port); + } else { + socket = new InetSocketAddress(ifc, port); + givenHost = ifc; + } + + return new JMXExporterConfig(givenHost, port, givenConfigFile, socket); + } + } + + public static class JMXExporterConfig { + String host; + int port; + String file; + InetSocketAddress socket; + + JMXExporterConfig(String host, int port, String file, InetSocketAddress socket) { + this.host = host; + this.port = port; + this.file = file; + this.socket = socket; + } + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index cd047125703c..40d366327afc 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -21,10 +21,22 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; +import io.prometheus.jmx.BuildInfoCollector; +import io.prometheus.jmx.JavaAgent; +import io.prometheus.jmx.JmxCollector; +import io.prometheus.jmx.common.http.ConfigurationException; +import io.prometheus.jmx.common.http.HTTPServerFactory; +import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry; +import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; +import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports; +import java.io.File; +import java.net.URI; import java.util.Arrays; import java.util.List; +import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; @@ -40,8 +52,35 @@ public class ServerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; private ServerMetrics _serverMetrics; + private int _exporterPort; + + private HTTPServer _httpServer; + @BeforeClass - public void setup() { + public void setup() + throws Exception { + + _exporterPort = 9000 + new Random().nextInt(1000); + String agentArgs = String.format("%s:%s", _exporterPort, + "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml"); + + String host = "0.0.0.0"; + + try { + JMXExporterConfig config = parseConfig(agentArgs, host); + CollectorRegistry registry = new CollectorRegistry(); + (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry); + DefaultExports.register(registry); + _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file)); + } catch (ConfigurationException var4) { + System.err.println("Configuration Exception : " + var4.getMessage()); + System.exit(1); + } catch (IllegalArgumentException var5) { + System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]: " + var5.getMessage()); + System.exit(1); + } + + PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); @@ -225,4 +264,13 @@ private void assertMeterExportedCorrectly(String exportedMeterName) { private void assertMeterExportedCorrectly(String exportedMeterName, List labels) { assertMeterExportedCorrectly(exportedMeterName, labels, EXPORTED_METRIC_PREFIX); } + + @Override + protected SimpleHttpResponse getExportedPromMetrics() { + try { + return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } From 1b69b170f30271cf39e13092058d5af845a25d0d Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Thu, 17 Oct 2024 00:18:57 +0530 Subject: [PATCH 18/45] WIP --- .../metrics/BrokerJMXToPromMetricsTest.java | 11 +++---- .../ControllerJMXToPromMetricsTest.java | 33 ++----------------- .../metrics/PinotJMXToPromMetricsTest.java | 33 +++++++++++++++++-- .../metrics/ServerJMXToPromMetricsTest.java | 23 ++----------- 4 files changed, 39 insertions(+), 61 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java index 1a8f855a016c..0cf4ce030773 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java @@ -22,6 +22,7 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; import io.prometheus.jmx.JavaAgent; +import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; import java.util.List; import java.util.Random; @@ -49,15 +50,13 @@ public class BrokerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private int _exporterPort; + private HTTPServer _httpServer; + @BeforeClass public void setup() throws Exception { - _exporterPort = 9000 + new Random().nextInt(1000); - String agentArgs = String.format("%s:%s", _exporterPort, - "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml"); - - JavaAgent.agentmain(agentArgs, null); + _httpServer = startExporter(PinotComponent.BROKER); PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, @@ -159,7 +158,7 @@ public void brokerMeterTest() { @Override protected SimpleHttpResponse getExportedPromMetrics() { try { - return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics")); + return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 9e64cf3f4d8d..5d71dd08d394 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -21,22 +21,11 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; -import io.prometheus.jmx.BuildInfoCollector; -import io.prometheus.jmx.JavaAgent; -import io.prometheus.jmx.JmxCollector; -import io.prometheus.jmx.common.http.ConfigurationException; -import io.prometheus.jmx.common.http.HTTPServerFactory; -import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; -import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports; -import java.io.File; -import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; @@ -66,25 +55,7 @@ public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { public void setup() throws Exception { - _exporterPort = 9000 + new Random().nextInt(1000); - String agentArgs = String.format("%s:%s", _exporterPort, - "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml"); - - String host = "0.0.0.0"; - - try { - JMXExporterConfig config = parseConfig(agentArgs, host); - CollectorRegistry registry = new CollectorRegistry(); - (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry); - DefaultExports.register(registry); - _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file)); - } catch (ConfigurationException var4) { - System.err.println("Configuration Exception : " + var4.getMessage()); - System.exit(1); - } catch (IllegalArgumentException var5) { - System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]: " + var5.getMessage()); - System.exit(1); - } + _httpServer = startExporter(PinotComponent.CONTROLLER); PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, @@ -259,7 +230,7 @@ private static String getStrippedMetricName(ControllerGauge controllerGauge) { @Override protected SimpleHttpResponse getExportedPromMetrics() { try { - return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics")); + return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 5566f6bd39ce..296f20511b70 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -20,13 +20,16 @@ package org.apache.pinot.common.metrics; import com.google.common.base.Objects; -import io.prometheus.jmx.JavaAgent; +import io.prometheus.jmx.JmxCollector; +import io.prometheus.jmx.common.http.HTTPServerFactory; +import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry; +import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; +import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.StringReader; import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -44,6 +47,12 @@ public abstract class PinotJMXToPromMetricsTest { + private static final Map PINOT_COMPONENT_CONFIG_FILE_MAP = + Map.of(PinotComponent.CONTROLLER, "controller.yml", PinotComponent.SERVER, "server.yml", PinotComponent.MINION, + "minion.yml", PinotComponent.BROKER, "broker.yml"); + + private static final String CONFIG_DIR = "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs"; + protected HttpClient _httpClient; protected static final String LABEL_KEY_TABLE = "table"; @@ -90,6 +99,20 @@ public abstract class PinotJMXToPromMetricsTest { protected static final List EXPORTED_LABELS_PERIODIC_TASK_TABLE_TABLETYPE = List.of("periodicTask", "ClusterHealthCheck", "table", "myTable", "tableType", "REALTIME"); + public HTTPServer startExporter(PinotComponent pinotComponent) { + String args = + String.format("%s:%s/%s", 0, CONFIG_DIR, PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); + try { + JMXExporterConfig config = parseConfig(args, "0.0.0.0"); + CollectorRegistry registry = new CollectorRegistry(); + (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry); + DefaultExports.register(registry); + return (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) { List promMetrics; try { @@ -315,4 +338,8 @@ public static class JMXExporterConfig { this.socket = socket; } } + + public enum PinotComponent { + SERVER, BROKER, CONTROLLER, MINION + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index 40d366327afc..8b9c0dc24d80 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -60,26 +60,7 @@ public class ServerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { public void setup() throws Exception { - _exporterPort = 9000 + new Random().nextInt(1000); - String agentArgs = String.format("%s:%s", _exporterPort, - "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml"); - - String host = "0.0.0.0"; - - try { - JMXExporterConfig config = parseConfig(agentArgs, host); - CollectorRegistry registry = new CollectorRegistry(); - (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry); - DefaultExports.register(registry); - _httpServer = (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file)); - } catch (ConfigurationException var4) { - System.err.println("Configuration Exception : " + var4.getMessage()); - System.exit(1); - } catch (IllegalArgumentException var5) { - System.err.println("Usage: -javaagent:/path/to/JavaAgent.jar=[host:]: " + var5.getMessage()); - System.exit(1); - } - + _httpServer = startExporter(PinotComponent.SERVER); PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, @@ -268,7 +249,7 @@ private void assertMeterExportedCorrectly(String exportedMeterName, List @Override protected SimpleHttpResponse getExportedPromMetrics() { try { - return _httpClient.sendGetRequest(new URI("http://localhost:" + _exporterPort + "/metrics")); + return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); } catch (Exception e) { throw new RuntimeException(e); } From 10c483171c488d4ad258d3b0ce34ab6aa3175cf9 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Thu, 17 Oct 2024 00:52:15 +0530 Subject: [PATCH 19/45] Added MinionJMXToPromMetricsTest.java --- .../metrics/MinionJMXToPromMetricsTest.java | 37 ++++++++++++++++++- .../metrics/PinotJMXToPromMetricsTest.java | 3 +- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java index f0a416bfad2a..0a73d5ccb570 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java @@ -21,8 +21,12 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; +import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; +import java.net.URI; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import javax.validation.constraints.Min; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; @@ -41,8 +45,13 @@ public class MinionJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String TASK_TYPE = "SegmentImportTask"; private MinionMetrics _minionMetrics; + private HTTPServer _httpServer; + @BeforeClass public void setup() { + + _httpServer = startExporter(PinotComponent.MINION); + PinotConfiguration pinotConfiguration = new PinotConfiguration(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); @@ -62,6 +71,20 @@ public void setup() { @Test public void timerTest() { + Stream.of(MinionTimer.values()).peek(timer -> { + _minionMetrics.addTimedValue(TASK_TYPE, timer, 30L, TimeUnit.MILLISECONDS); + _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, TASK_TYPE, timer, 30L, TimeUnit.MILLISECONDS); + }).forEach(timer -> { + assertTimerExportedCorrectly(timer.getTimerName(), List.of("id", TASK_TYPE), EXPORTED_METRIC_PREFIX); + if (timer == MinionTimer.TASK_THREAD_CPU_TIME_NS) { + assertTimerExportedCorrectly(timer.getTimerName(), + List.of("database", "myTable_REALTIME", "table", "myTable_REALTIME.SegmentImportTask"), + EXPORTED_METRIC_PREFIX); + } else { + assertTimerExportedCorrectly(timer.getTimerName(), + List.of("table", RAW_TABLE_NAME, "tableType", "REALTIME", "taskType", TASK_TYPE), EXPORTED_METRIC_PREFIX); + } + }); } @Test @@ -102,10 +125,22 @@ public void meterTest() { @Test public void gaugeTest() { + Stream.of(MinionGauge.values()).filter(MinionGauge::isGlobal).peek(gauge -> { + _minionMetrics.setValueOfGlobalGauge(gauge, 1L); + }).forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX)); + + Stream.of(MinionGauge.values()).filter(gauge -> !gauge.isGlobal()) + .peek(gauge -> _minionMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 1L)).forEach( + gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX)); } @Override protected SimpleHttpResponse getExportedPromMetrics() { - return null; + try { + return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); + } catch (Exception e) { + throw new RuntimeException(e); + } } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 296f20511b70..d6cc703e53f5 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -158,7 +158,8 @@ protected void assertTimerExportedCorrectly(String exportedTimerPrefix, List Date: Thu, 17 Oct 2024 11:16:08 +0530 Subject: [PATCH 20/45] Refactor --- .../metrics/BrokerJMXToPromMetricsTest.java | 16 +-- .../ControllerJMXToPromMetricsTest.java | 24 ++-- .../metrics/MinionJMXToPromMetricsTest.java | 48 ++++--- .../metrics/PinotJMXToPromMetricsTest.java | 129 +++++++++++------- .../metrics/ServerJMXToPromMetricsTest.java | 37 ++--- 5 files changed, 139 insertions(+), 115 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java index 0cf4ce030773..eeedd0835c02 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java @@ -21,11 +21,9 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; -import io.prometheus.jmx.JavaAgent; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; import java.util.List; -import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; @@ -48,8 +46,6 @@ public class BrokerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private BrokerMetrics _brokerMetrics; - private int _exporterPort; - private HTTPServer _httpServer; @BeforeClass @@ -76,20 +72,20 @@ public void setup() } @Test - public void brokerTimerTest() { + public void timerTest() { //first assert on global timers Stream.of(BrokerTimer.values()).filter(BrokerTimer::isGlobal) .peek(timer -> _brokerMetrics.addTimedValue(timer, 30_000, TimeUnit.MILLISECONDS)) .forEach(timer -> assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_METRIC_PREFIX)); //Assert on local timers Stream.of(BrokerTimer.values()).filter(timer -> !timer.isGlobal()).peek(timer -> { - _brokerMetrics.addTimedTableValue(RAW_TABLE_NAME, timer, 30_000L, TimeUnit.MILLISECONDS); + _brokerMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, timer, 30_000L, TimeUnit.MILLISECONDS); }).forEach(timer -> assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX)); } @Test - public void brokerGaugeTest() { + public void gaugeTest() { //global gauges Stream.of(BrokerGauge.values()).filter(BrokerGauge::isGlobal) .peek(gauge -> _brokerMetrics.setOrUpdateGlobalGauge(gauge, () -> 5L)) @@ -97,7 +93,7 @@ public void brokerGaugeTest() { //local gauges Stream.of(BrokerGauge.values()).filter(gauge -> !gauge.isGlobal()).peek(gauge -> { if (gauge == BrokerGauge.REQUEST_SIZE) { - _brokerMetrics.setOrUpdateTableGauge(RAW_TABLE_NAME, gauge, 5L); + _brokerMetrics.setOrUpdateTableGauge(LABEL_VAL_RAW_TABLENAME, gauge, 5L); } else { _brokerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L); } @@ -112,7 +108,7 @@ public void brokerGaugeTest() { } @Test - public void brokerMeterTest() { + public void meterTest() { List globalMetersWithExceptionsPrefix = List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, @@ -141,7 +137,7 @@ public void brokerMeterTest() { Stream.of(BrokerMeter.values()).filter(meter -> !meter.isGlobal()).peek(meter -> { if (localMetersThatAcceptRawTableName.contains(meter)) { - _brokerMetrics.addMeteredTableValue(RAW_TABLE_NAME, meter, 5L); + _brokerMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 5L); } else { _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 5d71dd08d394..44e36e6b5846 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -43,7 +43,7 @@ public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; - private static final String TASK_TYPE_KEY = "taskType"; + private static final String LABEL_KEY_TASK_TYPE = "taskType"; private static final String TASK_TYPE = "ClusterHealthCheck"; private ControllerMetrics _controllerMetrics; @@ -75,7 +75,7 @@ public void setup() } @Test - public void controllerTimerTest() { + public void timerTest() { //global timers Stream.of(ControllerTimer.values()).filter(ControllerTimer::isGlobal) .peek(controllerTimer -> _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS)) @@ -85,7 +85,7 @@ public void controllerTimerTest() { //local timers Stream.of(ControllerTimer.values()).filter(controllerTimer -> !controllerTimer.isGlobal()).peek(controllerTimer -> { _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - _controllerMetrics.addTimedTableValue(RAW_TABLE_NAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); + _controllerMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); }).forEach(controllerTimer -> { assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); @@ -95,7 +95,7 @@ public void controllerTimerTest() { } @Test - public void controllerMeterTest() { + public void meterTest() { //global meters Arrays.stream(ControllerMeter.values()).filter(ControllerMeter::isGlobal) .peek(controllerMeter -> _controllerMetrics.addMeteredGlobalValue(controllerMeter, 5L)) @@ -118,13 +118,13 @@ public void controllerMeterTest() { _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + TASK_TYPE, controllerMeter, 1L); } else { _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, controllerMeter, 5L); - _controllerMetrics.addMeteredTableValue(RAW_TABLE_NAME, controllerMeter, 5L); + _controllerMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, controllerMeter, 5L); } }).forEach(controllerMeter -> { String meterName = controllerMeter.getMeterName(); String strippedMeterName = StringUtils.remove(meterName, "controller"); if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, List.of("table", "ClusterHealthCheck"), EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, List.of(LABEL_KEY_TABLE, TASK_TYPE), EXPORTED_METRIC_PREFIX); } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_PERIODIC_TASK_TABLE_TABLETYPE, EXPORTED_METRIC_PREFIX); @@ -142,7 +142,7 @@ public void controllerMeterTest() { } @Test - public void controllerGaugeTest() { + public void gaugeTest() { //that accept global gauge with suffix List globalGaugesWithTaskType = List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, @@ -153,7 +153,8 @@ public void controllerGaugeTest() { .peek(controllerGauge -> _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE, 1L)) .filter(controllerGauge -> controllerGauge != ControllerGauge.VERSION).forEach(controllerGauge -> { String strippedMetricName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedMetricName, List.of(TASK_TYPE_KEY, TASK_TYPE), EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(strippedMetricName, List.of(LABEL_KEY_TASK_TYPE, TASK_TYPE), + EXPORTED_METRIC_PREFIX); }); //remaining guages are set without any suffix @@ -192,7 +193,7 @@ public void controllerGaugeTest() { List gaugesAcceptingRawTableName = List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); gaugesAcceptingRawTableName.stream() - .peek(gauge -> _controllerMetrics.setValueOfTableGauge(RAW_TABLE_NAME, gauge, 5L)).forEach( + .peek(gauge -> _controllerMetrics.setValueOfTableGauge(LABEL_VAL_RAW_TABLENAME, gauge, 5L)).forEach( gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX)); @@ -209,7 +210,8 @@ public void controllerGaugeTest() { //all remaining gauges Stream.of(ControllerGauge.values()).filter( - gauge -> isRemaining(gauge, gaugesAcceptingPartition, gaugesAcceptingTaskType, gaugesAcceptingRawTableName)) + gauge -> getRemaining(gauge, gaugesAcceptingPartition, gaugesAcceptingTaskType, + gaugesAcceptingRawTableName)) .filter(gauge -> gauge != ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED && gauge != ControllerGauge.TASK_STATUS) .peek(gauge -> _controllerMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L)).forEach(gauge -> { assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, @@ -217,7 +219,7 @@ public void controllerGaugeTest() { }); } - private boolean isRemaining(ControllerGauge gauge, List gaugesAcceptingPartition, + private boolean getRemaining(ControllerGauge gauge, List gaugesAcceptingPartition, List gaugesAcceptingTaskType, List gaugesAcceptingRawTableName) { return !gauge.isGlobal() && !gaugesAcceptingPartition.contains(gauge) && !gaugesAcceptingTaskType.contains(gauge) && !gaugesAcceptingRawTableName.contains(gauge); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java index 0a73d5ccb570..f20f887f7f2e 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; -import javax.validation.constraints.Min; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; @@ -41,12 +40,18 @@ public class MinionJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_minion_"; - private static final String TASK_TYPE_KEY = "taskType"; - private static final String TASK_TYPE = "SegmentImportTask"; - private MinionMetrics _minionMetrics; + private static final String LABEL_KEY_TASKTYPE = "taskType"; + private static final String LABEL_VAL_TASKTYPE = "SegmentImportTask"; + private static final String LABEL_KEY_ID = "id"; + private static final String METER_PREFIX_NO_TASKS = "numberTasks"; + private MinionMetrics _minionMetrics; private HTTPServer _httpServer; + private static final List EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE = + List.of(LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME, + LABEL_KEY_TASKTYPE, LABEL_VAL_TASKTYPE); + @BeforeClass public void setup() { @@ -72,17 +77,18 @@ public void setup() { @Test public void timerTest() { Stream.of(MinionTimer.values()).peek(timer -> { - _minionMetrics.addTimedValue(TASK_TYPE, timer, 30L, TimeUnit.MILLISECONDS); - _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, TASK_TYPE, timer, 30L, TimeUnit.MILLISECONDS); + _minionMetrics.addTimedValue(LABEL_VAL_TASKTYPE, timer, 30L, TimeUnit.MILLISECONDS); + _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, LABEL_VAL_TASKTYPE, timer, 30L, TimeUnit.MILLISECONDS); }).forEach(timer -> { - assertTimerExportedCorrectly(timer.getTimerName(), List.of("id", TASK_TYPE), EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(timer.getTimerName(), List.of(LABEL_KEY_ID, LABEL_VAL_TASKTYPE), + EXPORTED_METRIC_PREFIX); if (timer == MinionTimer.TASK_THREAD_CPU_TIME_NS) { assertTimerExportedCorrectly(timer.getTimerName(), - List.of("database", "myTable_REALTIME", "table", "myTable_REALTIME.SegmentImportTask"), - EXPORTED_METRIC_PREFIX); + List.of(LABEL_KEY_DATABASE, LABEL_VAL_TABLENAME_WITH_TYPE_REALTIME, LABEL_KEY_TABLE, + "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); } else { - assertTimerExportedCorrectly(timer.getTimerName(), - List.of("table", RAW_TABLE_NAME, "tableType", "REALTIME", "taskType", TASK_TYPE), EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, + EXPORTED_METRIC_PREFIX); } }); } @@ -95,12 +101,14 @@ public void meterTest() { .forEach(meter -> assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX)); //local meters Stream.of(MinionMeter.values()).filter(meter -> !meter.isGlobal()) - .filter(meter -> meter.getMeterName().startsWith("numberTasks")).peek(meter -> { - _minionMetrics.addMeteredTableValue(RAW_TABLE_NAME, meter, 1L); - _minionMetrics.addMeteredValue(TASK_TYPE, meter, 1L); + .filter(meter -> meter.getMeterName().startsWith(METER_PREFIX_NO_TASKS)).peek(meter -> { + _minionMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 1L); + _minionMetrics.addMeteredValue(LABEL_VAL_TASKTYPE, meter, 1L); }).forEach(meter -> { - assertMeterExportedCorrectly(meter.getMeterName(), List.of("id", RAW_TABLE_NAME), EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meter.getMeterName(), List.of("id", TASK_TYPE), EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, LABEL_VAL_RAW_TABLENAME), + EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, LABEL_VAL_TASKTYPE), + EXPORTED_METRIC_PREFIX); }); List metersAcceptingTableNameWithType = @@ -108,18 +116,18 @@ public void meterTest() { metersAcceptingTableNameWithType.stream() .peek(meter -> _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 1L)).forEach( - meter -> assertMeterExportedCorrectly(meter.getMeterName(), List.of("id", TABLE_NAME_WITH_TYPE), + meter -> assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, TABLE_NAME_WITH_TYPE), EXPORTED_METRIC_PREFIX)); Stream.of(MinionMeter.values()).filter(meter -> !meter.isGlobal()) .filter(meter -> !meter.getMeterName().startsWith("numberTasks")) .filter(meter -> !metersAcceptingTableNameWithType.contains(meter)).peek(meter -> { _minionMetrics.addMeteredGlobalValue(meter, 1L); - _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, TASK_TYPE, meter, 1L); + _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, LABEL_VAL_TASKTYPE, meter, 1L); }).forEach(meter -> { assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meter.getMeterName(), - List.of("table", RAW_TABLE_NAME, "tableType", "REALTIME", "taskType", TASK_TYPE), EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, + EXPORTED_METRIC_PREFIX); }); } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index d6cc703e53f5..c6dca74c9d46 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -56,6 +56,19 @@ public abstract class PinotJMXToPromMetricsTest { protected HttpClient _httpClient; protected static final String LABEL_KEY_TABLE = "table"; + protected static final String LABEL_VAL_RAW_TABLENAME = "myTable"; + protected static final String LABEL_KEY_TABLETYPE = "tableType"; + protected static final String LABEL_VAL_TABLETYPE_REALTIME = "REALTIME"; + protected static final String LABEL_VAL_TABLENAME_WITH_TYPE_REALTIME = + TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(LABEL_VAL_RAW_TABLENAME); + protected static final String LABEL_KEY_DATABASE = "database"; + protected static final String LABEL_KEY_TOPIC = "topic"; + protected static final String LABEL_KEY_PARTITION = "partition"; + protected static final String LABEL_KEY_CONTROLLER_TASKTYPE = "taskType"; + protected static final String LABEL_VAL_CONTROLLER_TASKTYPE_CHC = "ClusterHealthCheck"; + protected static final String LABEL_KEY_CONTROLLER_PERIODIC_TASK = "periodicTask"; + protected static final String LABEL_KEY_STATUS = "status"; + protected static final String LABEL_VAL_STATUS_IN_PROGRESS = "IN_PROGRESS"; protected static final List METER_TYPES = List.of("Count", "FiveMinuteRate", "MeanRate", "OneMinuteRate", "FifteenMinuteRate"); @@ -65,52 +78,55 @@ public abstract class PinotJMXToPromMetricsTest { "OneMinuteRate", "50thPercentile", "99thPercentile", "FifteenMinuteRate", "Mean", "StdDev", "MeanRate", "Min"); - protected static final String RAW_TABLE_NAME = "myTable"; protected static final String TABLE_NAME_WITH_TYPE = - TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(RAW_TABLE_NAME); + TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(LABEL_VAL_RAW_TABLENAME); protected static final String KAFKA_TOPIC = "myTopic"; protected static final String PARTITION_GROUP_ID = "partitionGroupId"; protected static final String CLIENT_ID = String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC, PARTITION_GROUP_ID); protected static final String TABLE_STREAM_NAME = String.format("%s_%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC); - protected static final List EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE = - List.of("table", "myTable", "tableType", "REALTIME"); + List.of(LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME); - protected static final List EXPORTED_LABELS_FOR_RAW_TABLE_NAME = List.of("table", "myTable"); + protected static final List EXPORTED_LABELS_FOR_RAW_TABLE_NAME = + List.of(LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME); protected static final List EXPORTED_LABELS_FOR_CLIENT_ID = - List.of("partition", PARTITION_GROUP_ID, "table", RAW_TABLE_NAME, "tableType", TableType.REALTIME.toString(), - "topic", KAFKA_TOPIC); + List.of(LABEL_KEY_PARTITION, PARTITION_GROUP_ID, LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, + TableType.REALTIME.toString(), LABEL_KEY_TOPIC, KAFKA_TOPIC); protected static final List EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE = - List.of("partition", "3", "table", RAW_TABLE_NAME, "tableType", TableType.REALTIME.toString()); + List.of(LABEL_KEY_PARTITION, "3", LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, + TableType.REALTIME.toString()); protected static final List EXPORTED_LABELS_FOR_TABLE_TABLETYPE_TASKTYPE = - List.of("table", "myTable", "tableType", "REALTIME", "taskType", "ClusterHealthCheck"); + List.of(LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME, + LABEL_KEY_CONTROLLER_TASKTYPE, LABEL_VAL_CONTROLLER_TASKTYPE_CHC); protected static final List EXPORTED_LABELS_FOR_TABLENAMEANDTYPE_AND_TASKTYPE = - List.of("table", "myTable_REALTIME", "taskType", "ClusterHealthCheck"); + List.of(LABEL_KEY_TABLE, LABEL_VAL_TABLENAME_WITH_TYPE_REALTIME, LABEL_KEY_CONTROLLER_TASKTYPE, + LABEL_VAL_CONTROLLER_TASKTYPE_CHC); protected static final List EXPORTED_LABELS_FOR_TASK_TYPE_AND_STATUS = - List.of("status", "IN_PROGRESS", "taskType", "ClusterHealthCheck"); + List.of(LABEL_KEY_STATUS, LABEL_VAL_STATUS_IN_PROGRESS, LABEL_KEY_CONTROLLER_TASKTYPE, + LABEL_VAL_CONTROLLER_TASKTYPE_CHC); protected static final List EXPORTED_LABELS_PERIODIC_TASK_TABLE_TABLETYPE = - List.of("periodicTask", "ClusterHealthCheck", "table", "myTable", "tableType", "REALTIME"); - - public HTTPServer startExporter(PinotComponent pinotComponent) { - String args = - String.format("%s:%s/%s", 0, CONFIG_DIR, PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); - try { - JMXExporterConfig config = parseConfig(args, "0.0.0.0"); - CollectorRegistry registry = new CollectorRegistry(); - (new JmxCollector(new File(config.file), JmxCollector.Mode.AGENT)).register(registry); - DefaultExports.register(registry); - return (new HTTPServerFactory()).createHTTPServer(config.socket, registry, true, new File(config.file)); - } catch (Exception e) { - throw new RuntimeException(e); - } + List.of(LABEL_KEY_CONTROLLER_PERIODIC_TASK, LABEL_VAL_CONTROLLER_TASKTYPE_CHC, LABEL_KEY_TABLE, + LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME); + + protected HTTPServer startExporter(PinotComponent pinotComponent) { + String args = String.format("%s:%s/%s", 0, CONFIG_DIR, PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); + try { + JMXExporterConfig config = parseExporterConfig(args, "0.0.0.0"); + CollectorRegistry registry = new CollectorRegistry(); + (new JmxCollector(new File(config._file), JmxCollector.Mode.AGENT)).register(registry); + DefaultExports.register(registry); + return (new HTTPServerFactory()).createHTTPServer(config._socket, registry, true, new File(config._file)); + } catch (Exception e) { + throw new RuntimeException(e); + } } protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String exportedMetricPrefix) { @@ -214,6 +230,12 @@ protected List parseExportedPromMetrics(String response) protected abstract SimpleHttpResponse getExportedPromMetrics(); + protected abstract void timerTest(); + + protected abstract void gaugeTest(); + + protected abstract void meterTest(); + public static class PromMetric { private final String _metricName; private final Map _labels; @@ -246,16 +268,6 @@ public static PromMetric fromExportedMetric(String exportedMetric) { } } - private static Map parseLabels(String labelsString) { - return labelsString.isEmpty() ? new LinkedHashMap<>() - : java.util.Arrays.stream(labelsString.split(",")).map(kvPair -> kvPair.split("=")) - .collect(Collectors.toMap(kv -> kv[0], kv -> removeQuotes(kv[1]), (v1, v2) -> v2, LinkedHashMap::new)); - } - - private static String removeQuotes(String value) { - return value.startsWith("\"") ? value.substring(1, value.length() - 1) : value; - } - public static PromMetric withName(String metricName) { return new PromMetric(metricName, new LinkedHashMap<>()); } @@ -280,12 +292,6 @@ public boolean equals(Object o) { return metricNamesAreSimilar(that) && Objects.equal(_labels, that._labels); } - private boolean metricNamesAreSimilar(PromMetric that) { - String processedMetricNameThis = StringUtils.remove(_metricName, "_"); - String processedMetricNameThat = StringUtils.remove(that._metricName, "_"); - return StringUtils.equalsIgnoreCase(processedMetricNameThis, processedMetricNameThat); - } - @Override public int hashCode() { return Objects.hashCode(_metricName, _labels); @@ -302,9 +308,30 @@ public String toString() { } return sb.toString(); } + + private boolean metricNamesAreSimilar(PromMetric that) { + String processedMetricNameThis = StringUtils.remove(_metricName, "_"); + String processedMetricNameThat = StringUtils.remove(that._metricName, "_"); + return StringUtils.equalsIgnoreCase(processedMetricNameThis, processedMetricNameThat); + } + + private static Map parseLabels(String labelsString) { + return labelsString.isEmpty() ? new LinkedHashMap<>() + : java.util.Arrays.stream(labelsString.split(",")).map(kvPair -> kvPair.split("=")) + .collect(Collectors.toMap(kv -> kv[0], kv -> removeQuotes(kv[1]), (v1, v2) -> v2, LinkedHashMap::new)); + } + + private static String removeQuotes(String value) { + return value.startsWith("\"") ? value.substring(1, value.length() - 1) : value; + } } - public static JMXExporterConfig parseConfig(String args, String ifc) { + /* + Implementation copied from: https://github + .com/prometheus/jmx_exporter/blob/a3b9443564ff5a78c25fd6566396fda2b7cbf216/jmx_prometheus_javaagent/src/main/java + /io/prometheus/jmx/JavaAgent.java#L88 + */ + private static JMXExporterConfig parseExporterConfig(String args, String ifc) { Pattern pattern = Pattern.compile("^(?:((?:[\\w.-]+)|(?:\\[.+])):)?(\\d{1,5}):(.+)"); Matcher matcher = pattern.matcher(args); if (!matcher.matches()) { @@ -326,17 +353,17 @@ public static JMXExporterConfig parseConfig(String args, String ifc) { } } - public static class JMXExporterConfig { - String host; - int port; - String file; - InetSocketAddress socket; + static class JMXExporterConfig { + String _host; + int _port; + String _file; + InetSocketAddress _socket; JMXExporterConfig(String host, int port, String file, InetSocketAddress socket) { - this.host = host; - this.port = port; - this.file = file; - this.socket = socket; + _host = host; + _port = port; + _file = file; + _socket = socket; } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index 8b9c0dc24d80..f0dc533903f4 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -21,19 +21,10 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; -import io.prometheus.jmx.BuildInfoCollector; -import io.prometheus.jmx.JavaAgent; -import io.prometheus.jmx.JmxCollector; -import io.prometheus.jmx.common.http.ConfigurationException; -import io.prometheus.jmx.common.http.HTTPServerFactory; -import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; -import io.prometheus.jmx.shaded.io.prometheus.client.hotspot.DefaultExports; -import java.io.File; import java.net.URI; import java.util.Arrays; import java.util.List; -import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.pinot.common.utils.SimpleHttpResponse; @@ -52,8 +43,6 @@ public class ServerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; private ServerMetrics _serverMetrics; - private int _exporterPort; - private HTTPServer _httpServer; @BeforeClass @@ -83,14 +72,14 @@ public void setup() * This test validates each timer defined in {@link ServerTimer} */ @Test - public void serverTimerTest() { + public void timerTest() { for (ServerTimer serverTimer : ServerTimer.values()) { if (serverTimer.isGlobal()) { _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS); } else { _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); - _serverMetrics.addTimedTableValue(RAW_TABLE_NAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); + _serverMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); } } @@ -110,7 +99,7 @@ public void serverTimerTest() { * This test validates each meter defined in {@link ServerMeter} */ @Test - public void serverMeterTest() { + public void meterTest() { //first, assert on all global meters Arrays.stream(ServerMeter.values()).filter(ServerMeter::isGlobal).peek(this::addGlobalMeter) .forEach(serverMeter -> { @@ -138,9 +127,10 @@ public void serverMeterTest() { List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, ServerMeter.SEGMENT_UPLOAD_TIMEOUT); - metersAcceptingRawTableNames.stream().peek(meter -> addMeterWithLables(meter, RAW_TABLE_NAME)).forEach(meter -> { - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME); - }); + metersAcceptingRawTableNames.stream().peek(meter -> addMeterWithLables(meter, LABEL_VAL_RAW_TABLENAME)) + .forEach(meter -> { + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME); + }); //remaining all meters accept tableNameWithType Arrays.stream(ServerMeter.values()).filter( @@ -155,7 +145,7 @@ public void serverMeterTest() { * This test validates each gauge defined in {@link ServerGauge} */ @Test - public void serverGaugeTest() { + public void gaugeTest() { int partition = 3; long someVal = 100L; @@ -196,9 +186,10 @@ public void serverGaugeTest() { List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, ServerGauge.LUCENE_INDEXING_DELAY_MS, ServerGauge.LUCENE_INDEXING_DELAY_DOCS); - gaugesAcceptingRawTableName.stream().peek(gauge -> _serverMetrics.setValueOfTableGauge(RAW_TABLE_NAME, gauge, 5L)) - .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, - EXPORTED_METRIC_PREFIX)); + gaugesAcceptingRawTableName.stream() + .peek(gauge -> _serverMetrics.setValueOfTableGauge(LABEL_VAL_RAW_TABLENAME, gauge, 5L)).forEach( + gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX)); //all remaining gauges Stream.of(ServerGauge.values()).filter(gauge -> !gauge.isGlobal()).filter( @@ -215,8 +206,8 @@ public void serverGaugeTest() { // table="myTable",tableType="REALTIME",}` _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, 5L); assertGaugeExportedCorrectly(String.valueOf(partition), - List.of("database", "dedupPrimaryKeysCount", "table", "dedupPrimaryKeysCount.myTable", "tableType", "REALTIME"), - EXPORTED_METRIC_PREFIX); + List.of(LABEL_KEY_DATABASE, "dedupPrimaryKeysCount", LABEL_KEY_TABLE, "dedupPrimaryKeysCount.myTable", + LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME), EXPORTED_METRIC_PREFIX); } public void addGlobalMeter(ServerMeter serverMeter) { From 1bbfd75dfd9e6eeb9b6d60a6f4c75fb39071ba4f Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Thu, 17 Oct 2024 11:24:37 +0530 Subject: [PATCH 21/45] Addressed comments --- .../metrics/PinotJMXToPromMetricsTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index c6dca74c9d46..4241fe6e1bb8 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -216,16 +216,16 @@ protected List parseExportedPromMetrics(String response) List exportedPromMetrics = new ArrayList<>(); - BufferedReader reader = new BufferedReader(new StringReader(response)); - - String line; - while ((line = reader.readLine()) != null) { - if (line.startsWith("pinot_")) { - exportedPromMetrics.add(PromMetric.fromExportedMetric(line)); + try (BufferedReader reader = new BufferedReader(new StringReader(response))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("pinot_")) { + exportedPromMetrics.add(PromMetric.fromExportedMetric(line)); + } } + reader.close(); + return exportedPromMetrics; } - reader.close(); - return exportedPromMetrics; } protected abstract SimpleHttpResponse getExportedPromMetrics(); From 5b4e091b2130eba162883b344cdb0f4718794ebd Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Thu, 17 Oct 2024 11:28:57 +0530 Subject: [PATCH 22/45] Rollback inadvertent changes --- pinot-common/pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index 799308ce3b62..a8156e4c77bd 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -49,10 +49,9 @@ usedefaultlisteners - false + false - From 4b4a94d5cf8b2535b69addb1c89bb95728066004 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Thu, 17 Oct 2024 12:12:15 +0530 Subject: [PATCH 23/45] Added a comment --- .../apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 4241fe6e1bb8..4c8e5abc346b 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -117,6 +117,7 @@ public abstract class PinotJMXToPromMetricsTest { LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME); protected HTTPServer startExporter(PinotComponent pinotComponent) { + //See: https://github.com/prometheus/jmx_exporter/blob/a3b9443564ff5a78c25fd6566396fda2b7cbf216/jmx_prometheus_javaagent/src/main/java/io/prometheus/jmx/JavaAgent.java#L48 String args = String.format("%s:%s/%s", 0, CONFIG_DIR, PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); try { JMXExporterConfig config = parseExporterConfig(args, "0.0.0.0"); From e057f79a2fd00a2b4e6cb303c3f59e890821ec89 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Thu, 17 Oct 2024 12:30:05 +0530 Subject: [PATCH 24/45] Add comment --- .../pinot/common/metrics/PinotJMXToPromMetricsTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 4c8e5abc346b..6deded3a575e 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -116,8 +116,13 @@ public abstract class PinotJMXToPromMetricsTest { List.of(LABEL_KEY_CONTROLLER_PERIODIC_TASK, LABEL_VAL_CONTROLLER_TASKTYPE_CHC, LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME); + /** + * For impl, see: https://github.com/prometheus/jmx_exporter/blob/a3b9443564ff5a78c25fd6566396fda2b7cbf216 + * /jmx_prometheus_javaagent/src/main/java/io/prometheus/jmx/JavaAgent.java#L48 + * @param pinotComponent the Pinot component to start the server for + * @return the corresponding HTTP server on a random unoccupied port + */ protected HTTPServer startExporter(PinotComponent pinotComponent) { - //See: https://github.com/prometheus/jmx_exporter/blob/a3b9443564ff5a78c25fd6566396fda2b7cbf216/jmx_prometheus_javaagent/src/main/java/io/prometheus/jmx/JavaAgent.java#L48 String args = String.format("%s:%s/%s", 0, CONFIG_DIR, PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); try { JMXExporterConfig config = parseExporterConfig(args, "0.0.0.0"); From 4b756e0046ae5fe843255c5e01bf998a3db6d5a3 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Thu, 17 Oct 2024 12:45:37 +0530 Subject: [PATCH 25/45] Remove dependency --- pinot-common/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index a8156e4c77bd..3e5c912857a2 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -274,12 +274,6 @@ equalsverifier test - - ai.startree.pinot - startree-confluent-common - 1.3.0-SNAPSHOT - test - From c0047a6b5a35d99924c83ac3850938f80154fea0 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Fri, 18 Oct 2024 12:09:29 +0530 Subject: [PATCH 26/45] Address comments --- pinot-common/pom.xml | 37 ------------------------------------- pom.xml | 8 ++++++++ 2 files changed, 8 insertions(+), 37 deletions(-) diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index 3e5c912857a2..ea5cbc30c4a9 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -110,7 +110,6 @@ io.prometheus.jmx jmx_prometheus_javaagent - 0.19.0 org.apache.pinot @@ -335,42 +334,6 @@ - - download-test-dependencies - - true - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy - test-compile - - copy - - - - - - - io.prometheus.jmx - jmx_prometheus_javaagent - 0.19.0 - jar - false - ${project.build.directory}/agent - - - - - - - - build-shaded-jar diff --git a/pom.xml b/pom.xml index 291a72ee3fef..06ccd78d6cc4 100644 --- a/pom.xml +++ b/pom.xml @@ -799,6 +799,14 @@ + + + io.prometheus.jmx + jmx_prometheus_javaagent + 0.19.0 + test + + io.netty From 7d1e854a38e997d052b14bba79c57e72a81cb8a3 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Fri, 18 Oct 2024 22:28:29 +0530 Subject: [PATCH 27/45] WIP --- .../metrics/BrokerJMXToPromMetricsTest.java | 89 +++---- .../ControllerJMXToPromMetricsTest.java | 229 ++++++++--------- .../metrics/MinionJMXToPromMetricsTest.java | 141 ++++++----- .../metrics/PinotJMXToPromMetricsTest.java | 32 ++- .../metrics/ServerJMXToPromMetricsTest.java | 236 +++++++++--------- 5 files changed, 371 insertions(+), 356 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java index eeedd0835c02..a6aeecb7a3a1 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java @@ -24,6 +24,7 @@ import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; import java.util.List; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; @@ -33,6 +34,7 @@ import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; @@ -71,61 +73,42 @@ public void setup() _httpClient = new HttpClient(); } - @Test - public void timerTest() { - //first assert on global timers - Stream.of(BrokerTimer.values()).filter(BrokerTimer::isGlobal) - .peek(timer -> _brokerMetrics.addTimedValue(timer, 30_000, TimeUnit.MILLISECONDS)) - .forEach(timer -> assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_METRIC_PREFIX)); - //Assert on local timers - Stream.of(BrokerTimer.values()).filter(timer -> !timer.isGlobal()).peek(timer -> { + @Test(dataProvider = "brokerTimers") + public void timerTest(BrokerTimer timer) { + if (timer.isGlobal()) { + _brokerMetrics.addTimedValue(timer, 30_000, TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_METRIC_PREFIX); + } else { _brokerMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, timer, 30_000L, TimeUnit.MILLISECONDS); - }).forEach(timer -> assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, - EXPORTED_METRIC_PREFIX)); + assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + } } - @Test - public void gaugeTest() { - //global gauges - Stream.of(BrokerGauge.values()).filter(BrokerGauge::isGlobal) - .peek(gauge -> _brokerMetrics.setOrUpdateGlobalGauge(gauge, () -> 5L)) - .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX)); - //local gauges - Stream.of(BrokerGauge.values()).filter(gauge -> !gauge.isGlobal()).peek(gauge -> { + @Test(dataProvider = "brokerGauges") + public void gaugeTest(BrokerGauge gauge) { + if (gauge.isGlobal()) { + _brokerMetrics.setOrUpdateGlobalGauge(gauge, () -> 5L); + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); + } else { if (gauge == BrokerGauge.REQUEST_SIZE) { _brokerMetrics.setOrUpdateTableGauge(LABEL_VAL_RAW_TABLENAME, gauge, 5L); - } else { - _brokerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L); - } - }).forEach(gauge -> { - if (gauge == BrokerGauge.REQUEST_SIZE) { assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); } else { + _brokerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L); assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); } - }); + } } - @Test - public void meterTest() { + @Test(dataProvider = "brokerMeters") + public void meterTest(BrokerMeter meter) { List globalMetersWithExceptionsPrefix = List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, BrokerMeter.QUERY_REJECTED_EXCEPTIONS, BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, BrokerMeter.RESOURCE_MISSING_EXCEPTIONS); - Stream.of(BrokerMeter.values()).filter(BrokerMeter::isGlobal) - .peek(meter -> _brokerMetrics.addMeteredGlobalValue(meter, 5L)).forEach(meter -> { - if (globalMetersWithExceptionsPrefix.contains(meter)) { - String exportedMeterPrefix = String.format("%s_%s", EXPORTED_METRIC_PREFIX_EXCEPTIONS, - StringUtils.remove(meter.getMeterName(), "Exceptions")); - assertMeterExportedCorrectly(exportedMeterPrefix, EXPORTED_METRIC_PREFIX); - } else { - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); - } - }); - List localMetersThatAcceptRawTableName = List.of(BrokerMeter.QUERIES, BrokerMeter.NO_SERVER_FOUND_EXCEPTIONS, BrokerMeter.DOCUMENTS_SCANNED, BrokerMeter.ENTRIES_SCANNED_IN_FILTER, BrokerMeter.BROKER_RESPONSES_WITH_UNAVAILABLE_SEGMENTS, @@ -135,20 +118,25 @@ public void meterTest() { BrokerMeter.ENTRIES_SCANNED_POST_FILTER, BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE, BrokerMeter.QUERY_QUOTA_EXCEEDED); - Stream.of(BrokerMeter.values()).filter(meter -> !meter.isGlobal()).peek(meter -> { - if (localMetersThatAcceptRawTableName.contains(meter)) { - _brokerMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 5L); + if (meter.isGlobal()) { + _brokerMetrics.addMeteredGlobalValue(meter, 5L); + if (globalMetersWithExceptionsPrefix.contains(meter)) { + String exportedMeterPrefix = String.format("%s_%s", EXPORTED_METRIC_PREFIX_EXCEPTIONS, + StringUtils.remove(meter.getMeterName(), "Exceptions")); + assertMeterExportedCorrectly(exportedMeterPrefix, EXPORTED_METRIC_PREFIX); } else { - _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); } - }).forEach(meter -> { + } else { if (localMetersThatAcceptRawTableName.contains(meter)) { + _brokerMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 5L); assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); } else { + _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); } - }); + } } @Override @@ -159,4 +147,19 @@ protected SimpleHttpResponse getExportedPromMetrics() { throw new RuntimeException(e); } } + + @DataProvider(name = "brokerTimers") + public Object[] brokerTimers() { + return BrokerTimer.values(); + } + + @DataProvider(name = "brokerMeters") + public Object[] brokerMeters() { + return BrokerMeter.values(); + } + + @DataProvider(name = "brokerGauges") + public Object[] brokerGauges() { + return BrokerGauge.values(); + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java index 44e36e6b5846..32b839f83858 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java @@ -35,7 +35,9 @@ import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; @@ -74,155 +76,124 @@ public void setup() _httpClient = new HttpClient(); } - @Test - public void timerTest() { - //global timers - Stream.of(ControllerTimer.values()).filter(ControllerTimer::isGlobal) - .peek(controllerTimer -> _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS)) - .forEach( - controllerTimer -> assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX)); + @Test(dataProvider = "controllerTimers") + public void timerTest(ControllerTimer controllerTimer) { - //local timers - Stream.of(ControllerTimer.values()).filter(controllerTimer -> !controllerTimer.isGlobal()).peek(controllerTimer -> { + if (controllerTimer.isGlobal()) { + _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX); + } else { _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); _controllerMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - }).forEach(controllerTimer -> { assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); - }); + } } - @Test - public void meterTest() { + @Test(dataProvider = "controllerMeters") + public void meterTest(ControllerMeter meter) { //global meters - Arrays.stream(ControllerMeter.values()).filter(ControllerMeter::isGlobal) - .peek(controllerMeter -> _controllerMetrics.addMeteredGlobalValue(controllerMeter, 5L)) - .forEach(controllerMeter -> { - String meterName = controllerMeter.getMeterName(); - //some meters contain a "controller" prefix. For example, controllerInstancePostError. These meters are - // exported as 'pinot_controller_pinot_controller_InstancePostError'. So we strip the 'controller' from - // 'controllerInstancePostError' - String strippedMeterName = StringUtils.remove(meterName, "controller"); - assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX); - }); - - //local meters - Arrays.stream(ControllerMeter.values()).filter(controllerMeter -> !controllerMeter.isGlobal()) - .peek(controllerMeter -> { - if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR - || controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - _controllerMetrics.addMeteredTableValue(TASK_TYPE, controllerMeter, 1L); - } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { - _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + TASK_TYPE, controllerMeter, 1L); - } else { - _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, controllerMeter, 5L); - _controllerMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, controllerMeter, 5L); - } - }).forEach(controllerMeter -> { - String meterName = controllerMeter.getMeterName(); - String strippedMeterName = StringUtils.remove(meterName, "controller"); - if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, List.of(LABEL_KEY_TABLE, TASK_TYPE), EXPORTED_METRIC_PREFIX); - } else if (controllerMeter == ControllerMeter.PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_PERIODIC_TASK_TABLE_TABLETYPE, - EXPORTED_METRIC_PREFIX); - } else if (controllerMeter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - assertMeterExportedCorrectly(String.format("%s_%s", strippedMeterName, TASK_TYPE), EXPORTED_METRIC_PREFIX); - } else if (controllerMeter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { - assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); - } else { - assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); - } - }); + if (meter.isGlobal()) { + _controllerMetrics.addMeteredGlobalValue(meter, 5L); + String meterName = meter.getMeterName(); + //some meters contain a "controller" prefix. For example, controllerInstancePostError. These meters are + // exported as 'pinot_controller_pinot_controller_InstancePostError'. So we strip the 'controller' from + // 'controllerInstancePostError' + String strippedMeterName = StringUtils.remove(meterName, "controller"); + assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX); + } else { + if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR + || meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { + _controllerMetrics.addMeteredTableValue(TASK_TYPE, meter, 1L); + } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { + _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + TASK_TYPE, meter, 1L); + } else { + _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); + _controllerMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 5L); + } + String meterName = meter.getMeterName(); + String strippedMeterName = StringUtils.remove(meterName, "controller"); + if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { + assertMeterExportedCorrectly(meterName, List.of(LABEL_KEY_TABLE, TASK_TYPE), EXPORTED_METRIC_PREFIX); + } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { + assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_PERIODIC_TASK_TABLE_TABLETYPE, EXPORTED_METRIC_PREFIX); + } else if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { + assertMeterExportedCorrectly(String.format("%s_%s", strippedMeterName, TASK_TYPE), EXPORTED_METRIC_PREFIX); + } else if (meter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { + assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + } + } } - @Test - public void gaugeTest() { + @Test(dataProvider = "controllerGauges") + public void gaugeTest(ControllerGauge controllerGauge) { //that accept global gauge with suffix List globalGaugesWithTaskType = List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, ControllerGauge.NUM_MINION_SUBTASKS_WAITING, ControllerGauge.NUM_MINION_SUBTASKS_ERROR, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); - Arrays.stream(ControllerGauge.values()).filter(ControllerGauge::isGlobal).filter(globalGaugesWithTaskType::contains) - .peek(controllerGauge -> _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE, 1L)) - .filter(controllerGauge -> controllerGauge != ControllerGauge.VERSION).forEach(controllerGauge -> { - String strippedMetricName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedMetricName, List.of(LABEL_KEY_TASK_TYPE, TASK_TYPE), - EXPORTED_METRIC_PREFIX); - }); - - //remaining guages are set without any suffix - Arrays.stream(ControllerGauge.values()).filter(ControllerGauge::isGlobal) - .filter(controllerGauge -> !globalGaugesWithTaskType.contains(controllerGauge)) - .peek(controllerGauge -> _controllerMetrics.setValueOfGlobalGauge(controllerGauge, 1L)) - .forEach(controllerGauge -> { - String strippedMetricName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedMetricName, EXPORTED_METRIC_PREFIX); - }); - //local gauges that accept partition List gaugesAcceptingPartition = List.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS); - gaugesAcceptingPartition.stream().peek( - controllerGauge -> _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, - 10L)) - .forEach(controllerGauge -> { - String strippedGaugeName = getStrippedMetricName(controllerGauge); - ArrayList exportedLabels = new ArrayList<>(EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE); - assertGaugeExportedCorrectly(strippedGaugeName, exportedLabels, EXPORTED_METRIC_PREFIX); - }); - //these accept task type List gaugesAcceptingTaskType = List.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR); - gaugesAcceptingTaskType.stream() - .peek(gauge -> _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, TASK_TYPE, gauge, () -> 50L)) - .forEach( - gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_TABLETYPE_TASKTYPE, - EXPORTED_METRIC_PREFIX)); - List gaugesAcceptingRawTableName = List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); - gaugesAcceptingRawTableName.stream() - .peek(gauge -> _controllerMetrics.setValueOfTableGauge(LABEL_VAL_RAW_TABLENAME, gauge, 5L)).forEach( - gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, - EXPORTED_METRIC_PREFIX)); - - //ad-hoc - _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TABLE_NAME_WITH_TYPE, TASK_TYPE), - ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED, 5L); - assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(), - EXPORTED_LABELS_FOR_TABLENAMEANDTYPE_AND_TASKTYPE, EXPORTED_METRIC_PREFIX); - - _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TASK_TYPE, TaskState.IN_PROGRESS), - ControllerGauge.TASK_STATUS, 5); - assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), EXPORTED_LABELS_FOR_TASK_TYPE_AND_STATUS, - EXPORTED_METRIC_PREFIX); - - //all remaining gauges - Stream.of(ControllerGauge.values()).filter( - gauge -> getRemaining(gauge, gaugesAcceptingPartition, gaugesAcceptingTaskType, - gaugesAcceptingRawTableName)) - .filter(gauge -> gauge != ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED && gauge != ControllerGauge.TASK_STATUS) - .peek(gauge -> _controllerMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L)).forEach(gauge -> { - assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - }); - } - private boolean getRemaining(ControllerGauge gauge, List gaugesAcceptingPartition, - List gaugesAcceptingTaskType, List gaugesAcceptingRawTableName) { - return !gauge.isGlobal() && !gaugesAcceptingPartition.contains(gauge) && !gaugesAcceptingTaskType.contains(gauge) - && !gaugesAcceptingRawTableName.contains(gauge); + if (controllerGauge.isGlobal()) { + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE, 1L); + if (globalGaugesWithTaskType.contains(controllerGauge)) { + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE, 1L); + String strippedMetricName = getStrippedMetricName(controllerGauge); + assertGaugeExportedCorrectly(strippedMetricName, List.of(LABEL_KEY_TASK_TYPE, TASK_TYPE), + EXPORTED_METRIC_PREFIX); + } else { + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, 1L); + String strippedMetricName = getStrippedMetricName(controllerGauge); + assertGaugeExportedCorrectly(strippedMetricName, EXPORTED_METRIC_PREFIX); + } + } else { + if (gaugesAcceptingPartition.contains(controllerGauge)) { + _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, 10L); + String strippedGaugeName = getStrippedMetricName(controllerGauge); + ArrayList exportedLabels = new ArrayList<>(EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE); + assertGaugeExportedCorrectly(strippedGaugeName, exportedLabels, EXPORTED_METRIC_PREFIX); + } else if (gaugesAcceptingTaskType.contains(controllerGauge)) { + _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, TASK_TYPE, controllerGauge, () -> 50L); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_TABLETYPE_TASKTYPE, + EXPORTED_METRIC_PREFIX); + } else if (gaugesAcceptingRawTableName.contains(controllerGauge)) { + _controllerMetrics.setValueOfTableGauge(LABEL_VAL_RAW_TABLENAME, controllerGauge, 5L); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX); + } else if (controllerGauge == ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED) { + _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TABLE_NAME_WITH_TYPE, TASK_TYPE), + ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED, 5L); + assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(), + EXPORTED_LABELS_FOR_TABLENAMEANDTYPE_AND_TASKTYPE, EXPORTED_METRIC_PREFIX); + } else if (controllerGauge == ControllerGauge.TASK_STATUS) { + _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TASK_TYPE, TaskState.IN_PROGRESS), + ControllerGauge.TASK_STATUS, 5); + assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), + EXPORTED_LABELS_FOR_TASK_TYPE_AND_STATUS, EXPORTED_METRIC_PREFIX); + } else { + _controllerMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, controllerGauge, 5L); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + } + } } private static String getStrippedMetricName(ControllerGauge controllerGauge) { @@ -237,4 +208,24 @@ protected SimpleHttpResponse getExportedPromMetrics() { throw new RuntimeException(e); } } + + @DataProvider(name = "controllerTimers") + public Object[] controllerTimers() { + return ControllerTimer.values(); + } + + @DataProvider(name = "controllerMeters") + public Object[] controllerMeters() { + return ControllerMeter.values(); + } + + @DataProvider(name = "controllerGauges") + public Object[] controllerGauges() { + return ControllerGauge.values(); + } + + @AfterClass + public void cleanup() { + _httpServer.close(); + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java index f20f887f7f2e..bf1f44b5895d 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java @@ -31,7 +31,9 @@ import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; @@ -74,73 +76,70 @@ public void setup() { _httpClient = new HttpClient(); } - @Test - public void timerTest() { - Stream.of(MinionTimer.values()).peek(timer -> { - _minionMetrics.addTimedValue(LABEL_VAL_TASKTYPE, timer, 30L, TimeUnit.MILLISECONDS); - _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, LABEL_VAL_TASKTYPE, timer, 30L, TimeUnit.MILLISECONDS); - }).forEach(timer -> { - assertTimerExportedCorrectly(timer.getTimerName(), List.of(LABEL_KEY_ID, LABEL_VAL_TASKTYPE), + @Test(dataProvider = "minionTimers") + public void timerTest(MinionTimer timer) { + + _minionMetrics.addTimedValue(LABEL_VAL_TASKTYPE, timer, 30L, TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(timer.getTimerName(), List.of(LABEL_KEY_ID, LABEL_VAL_TASKTYPE), + EXPORTED_METRIC_PREFIX); + + _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, LABEL_VAL_TASKTYPE, timer, 30L, TimeUnit.MILLISECONDS); + + if (timer == MinionTimer.TASK_THREAD_CPU_TIME_NS) { + assertTimerExportedCorrectly(timer.getTimerName(), + List.of(LABEL_KEY_DATABASE, LABEL_VAL_TABLENAME_WITH_TYPE_REALTIME, LABEL_KEY_TABLE, + "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); + } else { + assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); - if (timer == MinionTimer.TASK_THREAD_CPU_TIME_NS) { - assertTimerExportedCorrectly(timer.getTimerName(), - List.of(LABEL_KEY_DATABASE, LABEL_VAL_TABLENAME_WITH_TYPE_REALTIME, LABEL_KEY_TABLE, - "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); - } else { - assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, - EXPORTED_METRIC_PREFIX); - } - }); + } } - @Test - public void meterTest() { - //global meters - Stream.of(MinionMeter.values()).filter(MinionMeter::isGlobal) - .peek(meter -> _minionMetrics.addMeteredGlobalValue(meter, 5L)) - .forEach(meter -> assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX)); - //local meters - Stream.of(MinionMeter.values()).filter(meter -> !meter.isGlobal()) - .filter(meter -> meter.getMeterName().startsWith(METER_PREFIX_NO_TASKS)).peek(meter -> { - _minionMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 1L); - _minionMetrics.addMeteredValue(LABEL_VAL_TASKTYPE, meter, 1L); - }).forEach(meter -> { - assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, LABEL_VAL_RAW_TABLENAME), - EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, LABEL_VAL_TASKTYPE), - EXPORTED_METRIC_PREFIX); - }); - - List metersAcceptingTableNameWithType = - List.of(MinionMeter.SEGMENT_UPLOAD_FAIL_COUNT, MinionMeter.SEGMENT_DOWNLOAD_FAIL_COUNT); - - metersAcceptingTableNameWithType.stream() - .peek(meter -> _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 1L)).forEach( - meter -> assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, TABLE_NAME_WITH_TYPE), - EXPORTED_METRIC_PREFIX)); - - Stream.of(MinionMeter.values()).filter(meter -> !meter.isGlobal()) - .filter(meter -> !meter.getMeterName().startsWith("numberTasks")) - .filter(meter -> !metersAcceptingTableNameWithType.contains(meter)).peek(meter -> { - _minionMetrics.addMeteredGlobalValue(meter, 1L); - _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, LABEL_VAL_TASKTYPE, meter, 1L); - }).forEach(meter -> { - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, - EXPORTED_METRIC_PREFIX); - }); + @Test(dataProvider = "minionMeters") + public void meterTest(MinionMeter meter) { + if (meter.isGlobal()) { + validateGlobalMeters(meter); + } else { + validateMetersWithLabels(meter); + } } - @Test - public void gaugeTest() { - Stream.of(MinionGauge.values()).filter(MinionGauge::isGlobal).peek(gauge -> { - _minionMetrics.setValueOfGlobalGauge(gauge, 1L); - }).forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX)); + private void validateMetersWithLabels(MinionMeter meter) { + if (meter.getMeterName().startsWith(METER_PREFIX_NO_TASKS)) { + _minionMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 1L); + _minionMetrics.addMeteredValue(LABEL_VAL_TASKTYPE, meter, 1L); + assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, LABEL_VAL_RAW_TABLENAME), + EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, LABEL_VAL_TASKTYPE), + EXPORTED_METRIC_PREFIX); + } else if (meter == MinionMeter.SEGMENT_UPLOAD_FAIL_COUNT || meter == MinionMeter.SEGMENT_DOWNLOAD_FAIL_COUNT) { + _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 1L); + assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, TABLE_NAME_WITH_TYPE), + EXPORTED_METRIC_PREFIX); + } else { + _minionMetrics.addMeteredGlobalValue(meter, 1L); + _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, LABEL_VAL_TASKTYPE, meter, 1L); + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, + EXPORTED_METRIC_PREFIX); + } + } - Stream.of(MinionGauge.values()).filter(gauge -> !gauge.isGlobal()) - .peek(gauge -> _minionMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 1L)).forEach( - gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX)); + private void validateGlobalMeters(MinionMeter meter) { + _minionMetrics.addMeteredGlobalValue(meter, 5L); + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); + } + + @Test(dataProvider = "minionGauges") + public void gaugeTest(MinionGauge gauge) { + if (gauge.isGlobal()) { + _minionMetrics.setValueOfGlobalGauge(gauge, 1L); + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); + } else { + _minionMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 1L); + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + } } @Override @@ -151,4 +150,24 @@ protected SimpleHttpResponse getExportedPromMetrics() { throw new RuntimeException(e); } } + + @DataProvider(name = "minionTimers") + public Object[] minionTimers() { + return MinionTimer.values(); + } + + @DataProvider(name = "minionMeters") + public Object[] minionMeters() { + return MinionMeter.values(); + } + + @DataProvider(name = "minionGauges") + public Object[] minionGauges() { + return MinionGauge.values(); + } + + @AfterClass + public void cleanup() { + _httpServer.close(); + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java index 6deded3a575e..130ea2b1b796 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java @@ -139,9 +139,9 @@ protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String e List promMetrics; try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - Assert.assertTrue( - promMetrics.contains(PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value")), - exportedGaugePrefix); + PromMetric expectedMetric = PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value"); + Assert.assertTrue(promMetrics.contains(expectedMetric), + "Cannot find gauge: " + expectedMetric + " in exported metrics"); } catch (Exception e) { throw new RuntimeException(e); } @@ -152,9 +152,10 @@ protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, List promMetrics; try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - Assert.assertTrue(promMetrics.contains( - PromMetric.withNameAndLabels(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value", labels)), - exportedGaugePrefix); + PromMetric expectedGauge = + PromMetric.withNameAndLabels(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value", labels); + Assert.assertTrue(promMetrics.contains(expectedGauge), + "Cannot find gauge: " + expectedGauge + " in exported metrics"); } catch (Exception e) { throw new RuntimeException(e); } @@ -196,9 +197,9 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, String e throw new RuntimeException(e); } for (String meterType : METER_TYPES) { - Assert.assertTrue( - promMetrics.contains(PromMetric.withName(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType)), - exportedMeterPrefix); + PromMetric expectedMetric = PromMetric.withName(exportedMetricPrefix + exportedMeterPrefix + "_" + meterType); + Assert.assertTrue(promMetrics.contains(expectedMetric), + "Cannot find metric: " + expectedMetric + " in the exported metrics"); } } @@ -208,9 +209,10 @@ protected void assertMeterExportedCorrectly(String exportedMeterPrefix, List parseExportedPromMetrics(String response) protected abstract SimpleHttpResponse getExportedPromMetrics(); - protected abstract void timerTest(); - - protected abstract void gaugeTest(); - - protected abstract void meterTest(); - public static class PromMetric { private final String _metricName; private final Map _labels; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java index f0dc533903f4..f3724fd3e34b 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java @@ -32,7 +32,10 @@ import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; @@ -42,7 +45,6 @@ public class ServerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; private ServerMetrics _serverMetrics; - private HTTPServer _httpServer; @BeforeClass @@ -71,143 +73,127 @@ public void setup() /** * This test validates each timer defined in {@link ServerTimer} */ - @Test - public void timerTest() { - - for (ServerTimer serverTimer : ServerTimer.values()) { - if (serverTimer.isGlobal()) { - _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS); - } else { - _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); - _serverMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); - } + @Test(dataProvider = "serverTimers") + public void timerTest(ServerTimer serverTimer) { + if (serverTimer.isGlobal()) { + _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS); + } else { + _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); + _serverMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); } - - for (ServerTimer serverTimer : ServerTimer.values()) { - if (serverTimer.isGlobal()) { - assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_METRIC_PREFIX); - } else { - assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, - EXPORTED_METRIC_PREFIX); - } + if (serverTimer.isGlobal()) { + assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_METRIC_PREFIX); + } else { + assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX); } } /** * This test validates each meter defined in {@link ServerMeter} */ - @Test - public void meterTest() { + @Test(dataProvider = "serverMeters") + public void meterTest(ServerMeter serverMeter) { + //first, assert on all global meters - Arrays.stream(ServerMeter.values()).filter(ServerMeter::isGlobal).peek(this::addGlobalMeter) - .forEach(serverMeter -> { - //we cannot use raw meter names for all meters as exported metrics don't follow any convention currently. - // For example, meters that track realtime exceptions start with prefix "realtime_exceptions" - if (meterTrackingRealtimeExceptions(serverMeter)) { - assertMeterExportedCorrectly(getRealtimeExceptionMeterName(serverMeter)); - } else { - assertMeterExportedCorrectly(serverMeter.getMeterName()); - } - }); - - //these meters accept the clientId - List metersAcceptingClientId = - List.of(ServerMeter.REALTIME_ROWS_CONSUMED, ServerMeter.REALTIME_ROWS_SANITIZED, - ServerMeter.REALTIME_ROWS_FETCHED, ServerMeter.REALTIME_ROWS_FILTERED, - ServerMeter.INVALID_REALTIME_ROWS_DROPPED, ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, - ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, ServerMeter.ROWS_WITH_ERRORS); - - metersAcceptingClientId.stream().peek(meter -> addMeterWithLables(meter, CLIENT_ID)) - .forEach(meter -> assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_CLIENT_ID)); - - //these meters accept raw table name - List metersAcceptingRawTableNames = - List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, - ServerMeter.SEGMENT_UPLOAD_TIMEOUT); - - metersAcceptingRawTableNames.stream().peek(meter -> addMeterWithLables(meter, LABEL_VAL_RAW_TABLENAME)) - .forEach(meter -> { - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME); - }); - - //remaining all meters accept tableNameWithType - Arrays.stream(ServerMeter.values()).filter( - serverMeter -> !serverMeter.isGlobal() && !metersAcceptingRawTableNames.contains(serverMeter) - && !metersAcceptingClientId.contains(serverMeter)).forEach(serverMeter -> { - addMeterWithLables(serverMeter, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE); - }); + if (serverMeter.isGlobal()) { + addGlobalMeter(serverMeter); + //we cannot use raw meter names for all meters as exported metrics don't follow any convention currently. + // For example, meters that track realtime exceptions start with prefix "realtime_exceptions" + if (meterTrackingRealtimeExceptions(serverMeter)) { + assertMeterExportedCorrectly(getRealtimeExceptionMeterName(serverMeter)); + } else { + assertMeterExportedCorrectly(serverMeter.getMeterName()); + } + } else { + + //these meters accept the clientId + List metersAcceptingClientId = + List.of(ServerMeter.REALTIME_ROWS_CONSUMED, ServerMeter.REALTIME_ROWS_SANITIZED, + ServerMeter.REALTIME_ROWS_FETCHED, ServerMeter.REALTIME_ROWS_FILTERED, + ServerMeter.INVALID_REALTIME_ROWS_DROPPED, ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, + ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, ServerMeter.ROWS_WITH_ERRORS); + + //these meters accept raw table name + List metersAcceptingRawTableNames = + List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, + ServerMeter.SEGMENT_UPLOAD_TIMEOUT); + + if (metersAcceptingClientId.contains(serverMeter)) { + addMeterWithLables(serverMeter, CLIENT_ID); + assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_CLIENT_ID); + } else if (metersAcceptingRawTableNames.contains(serverMeter)) { + addMeterWithLables(serverMeter, LABEL_VAL_RAW_TABLENAME); + assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME); + } else { + addMeterWithLables(serverMeter, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE); + } + } } /** * This test validates each gauge defined in {@link ServerGauge} */ - @Test - public void gaugeTest() { + @Test(dataProvider = "serverGauges") + public void gaugeTest(ServerGauge serverGauge) { int partition = 3; long someVal = 100L; - //global gauges - Stream.of(ServerGauge.values()).filter(ServerGauge::isGlobal) - .peek(gauge -> _serverMetrics.setValueOfGlobalGauge(gauge, 10L)) - .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX)); - - //local gauges - //gauges that accept clientId - List gaugesAcceptingClientId = - List.of(ServerGauge.LLC_PARTITION_CONSUMING, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, - ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS); - - gaugesAcceptingClientId.stream() - .peek(gauge -> _serverMetrics.setValueOfTableGauge(CLIENT_ID, gauge, TimeUnit.MILLISECONDS.toSeconds(someVal))) - .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_CLIENT_ID, - EXPORTED_METRIC_PREFIX)); - - //gauges accepting partition - List gaugesAcceptingPartition = - List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, - ServerGauge.REALTIME_INGESTION_OFFSET_LAG, ServerGauge.REALTIME_INGESTION_DELAY_MS, - ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS); - - gaugesAcceptingPartition.stream() - .peek(gauge -> _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, gauge, someVal)) - .forEach(gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), - EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE, EXPORTED_METRIC_PREFIX)); - - //gauges accepting raw table name - List gaugesAcceptingRawTableName = - List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, - ServerGauge.LUCENE_INDEXING_DELAY_MS, ServerGauge.LUCENE_INDEXING_DELAY_DOCS); - - gaugesAcceptingRawTableName.stream() - .peek(gauge -> _serverMetrics.setValueOfTableGauge(LABEL_VAL_RAW_TABLENAME, gauge, 5L)).forEach( - gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, - EXPORTED_METRIC_PREFIX)); - - //all remaining gauges - Stream.of(ServerGauge.values()).filter(gauge -> !gauge.isGlobal()).filter( - gauge -> (!gaugesAcceptingClientId.contains(gauge) && !gaugesAcceptingPartition.contains(gauge) - && !gaugesAcceptingRawTableName.contains(gauge))) - .peek(gauge -> _serverMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, gauge, someVal)).forEach( - gauge -> assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX)); + if (serverGauge.isGlobal()) { + _serverMetrics.setValueOfGlobalGauge(serverGauge, 10L); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_METRIC_PREFIX); + } else { + //gauges that accept clientId + List gaugesAcceptingClientId = + List.of(ServerGauge.LLC_PARTITION_CONSUMING, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, + ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS); + + List gaugesAcceptingPartition = + List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, + ServerGauge.REALTIME_INGESTION_OFFSET_LAG, ServerGauge.REALTIME_INGESTION_DELAY_MS, + ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS, + ServerGauge.DEDUP_PRIMARY_KEYS_COUNT); + + List gaugesAcceptingRawTableName = + List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, + ServerGauge.LUCENE_INDEXING_DELAY_MS, ServerGauge.LUCENE_INDEXING_DELAY_DOCS); + + if (gaugesAcceptingClientId.contains(serverGauge)) { + _serverMetrics.setValueOfTableGauge(CLIENT_ID, serverGauge, TimeUnit.MILLISECONDS.toSeconds(someVal)); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_LABELS_FOR_CLIENT_ID, + EXPORTED_METRIC_PREFIX); + } else if (gaugesAcceptingPartition.contains(serverGauge)) { + _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, serverGauge, someVal); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), + EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE, EXPORTED_METRIC_PREFIX); + } else if (gaugesAcceptingRawTableName.contains(serverGauge)) { + _serverMetrics.setValueOfTableGauge(LABEL_VAL_RAW_TABLENAME, serverGauge, 5L); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + EXPORTED_METRIC_PREFIX); + } else { + _serverMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, serverGauge, someVal); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + EXPORTED_METRIC_PREFIX); + } + } //this gauge is currently exported as: `pinot_server_3_Value{database="dedupPrimaryKeysCount", // table="dedupPrimaryKeysCount.myTable",tableType="REALTIME",}`. We add an explicit test for it to maintain // backward compatibility. todo: ServerGauge.DEDUP_PRIMARY_KEYS_COUNT should be moved to // gaugesThatAcceptPartition. It should be exported as: `pinot_server_dedupPrimaryKeysCount_Value{partition="3", // table="myTable",tableType="REALTIME",}` - _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, 5L); - assertGaugeExportedCorrectly(String.valueOf(partition), - List.of(LABEL_KEY_DATABASE, "dedupPrimaryKeysCount", LABEL_KEY_TABLE, "dedupPrimaryKeysCount.myTable", - LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME), EXPORTED_METRIC_PREFIX); +// _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, 5L); +// assertGaugeExportedCorrectly(String.valueOf(partition), +// List.of(LABEL_KEY_DATABASE, "dedupPrimaryKeysCount", LABEL_KEY_TABLE, "dedupPrimaryKeysCount.myTable", +// LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME), EXPORTED_METRIC_PREFIX); } public void addGlobalMeter(ServerMeter serverMeter) { @@ -245,4 +231,24 @@ protected SimpleHttpResponse getExportedPromMetrics() { throw new RuntimeException(e); } } + + @DataProvider(name = "serverTimers") + public Object[] serverTimers() { + return ServerTimer.values(); // Provide all values of ServerTimer enum + } + + @DataProvider(name = "serverMeters") + public Object[] serverMeter() { + return ServerMeter.values(); // Provide all values of ServerTimer enum + } + + @DataProvider(name = "serverGauges") + public Object[] serverGauge() { + return ServerGauge.values(); // Provide all values of ServerTimer enum + } + + @AfterClass + public void cleanup() { + _httpServer.close(); + } } From a300a3499aafd684f7cbe75c71e26f408f1320d0 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Sun, 20 Oct 2024 14:23:52 +0530 Subject: [PATCH 28/45] WIP --- ....java => BrokerPrometheusMetricsTest.java} | 24 +-- ...a => ControllerPrometheusMetricsTest.java} | 51 +++--- ....java => MinionPrometheusMetricsTest.java} | 23 +-- ...t.java => PinotPrometheusMetricsTest.java} | 114 ++++++------ ....java => ServerPrometheusMetricsTest.java} | 165 ++++++++---------- 5 files changed, 187 insertions(+), 190 deletions(-) rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{BrokerJMXToPromMetricsTest.java => BrokerPrometheusMetricsTest.java} (85%) rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{ControllerJMXToPromMetricsTest.java => ControllerPrometheusMetricsTest.java} (85%) rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{MinionJMXToPromMetricsTest.java => MinionPrometheusMetricsTest.java} (87%) rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{PinotJMXToPromMetricsTest.java => PinotPrometheusMetricsTest.java} (78%) rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{ServerJMXToPromMetricsTest.java => ServerPrometheusMetricsTest.java} (57%) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java similarity index 85% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java index a6aeecb7a3a1..a8317969ccb0 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java @@ -24,9 +24,7 @@ import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; import java.util.List; -import java.util.Objects; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; @@ -40,7 +38,7 @@ import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; -public class BrokerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { +public class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_broker_"; @@ -79,8 +77,10 @@ public void timerTest(BrokerTimer timer) { _brokerMetrics.addTimedValue(timer, 30_000, TimeUnit.MILLISECONDS); assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_METRIC_PREFIX); } else { - _brokerMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, timer, 30_000L, TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + _brokerMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, timer, 30_000L, + TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(timer.getTimerName(), ExportedLabels.TABLENAME, + EXPORTED_METRIC_PREFIX); } } @@ -91,11 +91,12 @@ public void gaugeTest(BrokerGauge gauge) { assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); } else { if (gauge == BrokerGauge.REQUEST_SIZE) { - _brokerMetrics.setOrUpdateTableGauge(LABEL_VAL_RAW_TABLENAME, gauge, 5L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + _brokerMetrics.setOrUpdateTableGauge(ExportedLabelValues.TABLENAME, gauge, 5L); + assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME, + EXPORTED_METRIC_PREFIX); } else { _brokerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } } @@ -129,11 +130,12 @@ public void meterTest(BrokerMeter meter) { } } else { if (localMetersThatAcceptRawTableName.contains(meter)) { - _brokerMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 5L); - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + _brokerMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 5L); + assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME, + EXPORTED_METRIC_PREFIX); } else { _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java similarity index 85% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java index 32b839f83858..dd257b7b2ddf 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java @@ -24,10 +24,8 @@ import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.helix.task.TaskState; import org.apache.pinot.common.utils.SimpleHttpResponse; @@ -43,7 +41,7 @@ import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; -public class ControllerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { +public class ControllerPrometheusMetricsTest extends PinotPrometheusMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; private static final String LABEL_KEY_TASK_TYPE = "taskType"; private static final String TASK_TYPE = "ClusterHealthCheck"; @@ -84,10 +82,11 @@ public void timerTest(ControllerTimer controllerTimer) { assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX); } else { _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - _controllerMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + _controllerMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, controllerTimer, 30_000L, + TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(controllerTimer.getTimerName(), + ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(controllerTimer.getTimerName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } } @@ -111,23 +110,28 @@ public void meterTest(ControllerMeter meter) { _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + TASK_TYPE, meter, 1L); } else { _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); - _controllerMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 5L); + _controllerMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 5L); } String meterName = meter.getMeterName(); String strippedMeterName = StringUtils.remove(meterName, "controller"); if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, List.of(LABEL_KEY_TABLE, TASK_TYPE), EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, List.of(ExportedLabelKeys.TABLE, TASK_TYPE), + EXPORTED_METRIC_PREFIX); } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_PERIODIC_TASK_TABLE_TABLETYPE, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, ExportedLabels.PERIODIC_TASK_TABLE_TABLETYPE, + EXPORTED_METRIC_PREFIX); } else if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { assertMeterExportedCorrectly(String.format("%s_%s", strippedMeterName, TASK_TYPE), EXPORTED_METRIC_PREFIX); } else if (meter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { - assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME, + EXPORTED_METRIC_PREFIX); } else { - assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(strippedMeterName, EXPORTED_LABELS_FOR_RAW_TABLE_NAME, EXPORTED_METRIC_PREFIX); } } } @@ -168,30 +172,31 @@ public void gaugeTest(ControllerGauge controllerGauge) { if (gaugesAcceptingPartition.contains(controllerGauge)) { _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, 10L); String strippedGaugeName = getStrippedMetricName(controllerGauge); - ArrayList exportedLabels = new ArrayList<>(EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE); + ArrayList exportedLabels = + new ArrayList<>(ExportedLabels.PARTITION_TABLE_NAME_AND_TYPE); assertGaugeExportedCorrectly(strippedGaugeName, exportedLabels, EXPORTED_METRIC_PREFIX); } else if (gaugesAcceptingTaskType.contains(controllerGauge)) { _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, TASK_TYPE, controllerGauge, () -> 50L); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_TABLETYPE_TASKTYPE, - EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), + ExportedLabels.TABLENAME_TABLETYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); } else if (gaugesAcceptingRawTableName.contains(controllerGauge)) { - _controllerMetrics.setValueOfTableGauge(LABEL_VAL_RAW_TABLENAME, controllerGauge, 5L); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + _controllerMetrics.setValueOfTableGauge(ExportedLabelValues.TABLENAME, controllerGauge, 5L); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } else if (controllerGauge == ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED) { _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TABLE_NAME_WITH_TYPE, TASK_TYPE), ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED, 5L); assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(), - EXPORTED_LABELS_FOR_TABLENAMEANDTYPE_AND_TASKTYPE, EXPORTED_METRIC_PREFIX); + ExportedLabels.TABLENAME_WITHTYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); } else if (controllerGauge == ControllerGauge.TASK_STATUS) { _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TASK_TYPE, TaskState.IN_PROGRESS), ControllerGauge.TASK_STATUS, 5); assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), - EXPORTED_LABELS_FOR_TASK_TYPE_AND_STATUS, EXPORTED_METRIC_PREFIX); + ExportedLabels.STATUS_TASKTYPE, EXPORTED_METRIC_PREFIX); } else { _controllerMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, controllerGauge, 5L); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), + ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java similarity index 87% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java index bf1f44b5895d..85e3615d56f3 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java @@ -25,7 +25,6 @@ import java.net.URI; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; @@ -39,7 +38,7 @@ import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; -public class MinionJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { +public class MinionPrometheusMetricsTest extends PinotPrometheusMetricsTest { private static final String EXPORTED_METRIC_PREFIX = "pinot_minion_"; private static final String LABEL_KEY_TASKTYPE = "taskType"; @@ -51,8 +50,9 @@ public class MinionJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { private HTTPServer _httpServer; private static final List EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE = - List.of(LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME, - LABEL_KEY_TASKTYPE, LABEL_VAL_TASKTYPE); + List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.TABLENAME, + ExportedLabelKeys.TABLETYPE, ExportedLabelValues.TABLETYPE_REALTIME, LABEL_KEY_TASKTYPE, + LABEL_VAL_TASKTYPE); @BeforeClass public void setup() { @@ -87,8 +87,8 @@ public void timerTest(MinionTimer timer) { if (timer == MinionTimer.TASK_THREAD_CPU_TIME_NS) { assertTimerExportedCorrectly(timer.getTimerName(), - List.of(LABEL_KEY_DATABASE, LABEL_VAL_TABLENAME_WITH_TYPE_REALTIME, LABEL_KEY_TABLE, - "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); + List.of(ExportedLabelKeys.DATABASE, ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME, + ExportedLabelKeys.TABLE, "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); } else { assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); @@ -106,17 +106,20 @@ public void meterTest(MinionMeter meter) { private void validateMetersWithLabels(MinionMeter meter) { if (meter.getMeterName().startsWith(METER_PREFIX_NO_TASKS)) { - _minionMetrics.addMeteredTableValue(LABEL_VAL_RAW_TABLENAME, meter, 1L); + _minionMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 1L); + assertMeterExportedCorrectly(meter.getMeterName(), + List.of(LABEL_KEY_ID, ExportedLabelValues.TABLENAME), EXPORTED_METRIC_PREFIX); + _minionMetrics.addMeteredValue(LABEL_VAL_TASKTYPE, meter, 1L); - assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, LABEL_VAL_RAW_TABLENAME), - EXPORTED_METRIC_PREFIX); assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, LABEL_VAL_TASKTYPE), EXPORTED_METRIC_PREFIX); } else if (meter == MinionMeter.SEGMENT_UPLOAD_FAIL_COUNT || meter == MinionMeter.SEGMENT_DOWNLOAD_FAIL_COUNT) { + _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 1L); assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, TABLE_NAME_WITH_TYPE), EXPORTED_METRIC_PREFIX); } else { + //all remaining meters are also being used as global meters, check their usage _minionMetrics.addMeteredGlobalValue(meter, 1L); _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, LABEL_VAL_TASKTYPE, meter, 1L); assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); @@ -137,7 +140,7 @@ public void gaugeTest(MinionGauge gauge) { assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); } else { _minionMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 1L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java similarity index 78% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java index 130ea2b1b796..a96045694906 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java @@ -44,32 +44,21 @@ import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; +import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelKeys.*; +import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.*; -public abstract class PinotJMXToPromMetricsTest { +public abstract class PinotPrometheusMetricsTest { + + //this dir contains the JMX exporter configs for each Pinot component (broker, server, controller, minion) + private static final String JMX_EXPORTER_CONFIG_PARENT_DIR = "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs"; + + //this map is a mapping of pinot components to their JMX exporter config files. They can be found at: + // docker/images/pinot/etc/jmx_prometheus_javaagent/configs private static final Map PINOT_COMPONENT_CONFIG_FILE_MAP = Map.of(PinotComponent.CONTROLLER, "controller.yml", PinotComponent.SERVER, "server.yml", PinotComponent.MINION, "minion.yml", PinotComponent.BROKER, "broker.yml"); - - private static final String CONFIG_DIR = "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs"; - protected HttpClient _httpClient; - - protected static final String LABEL_KEY_TABLE = "table"; - protected static final String LABEL_VAL_RAW_TABLENAME = "myTable"; - protected static final String LABEL_KEY_TABLETYPE = "tableType"; - protected static final String LABEL_VAL_TABLETYPE_REALTIME = "REALTIME"; - protected static final String LABEL_VAL_TABLENAME_WITH_TYPE_REALTIME = - TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(LABEL_VAL_RAW_TABLENAME); - protected static final String LABEL_KEY_DATABASE = "database"; - protected static final String LABEL_KEY_TOPIC = "topic"; - protected static final String LABEL_KEY_PARTITION = "partition"; - protected static final String LABEL_KEY_CONTROLLER_TASKTYPE = "taskType"; - protected static final String LABEL_VAL_CONTROLLER_TASKTYPE_CHC = "ClusterHealthCheck"; - protected static final String LABEL_KEY_CONTROLLER_PERIODIC_TASK = "periodicTask"; - protected static final String LABEL_KEY_STATUS = "status"; - protected static final String LABEL_VAL_STATUS_IN_PROGRESS = "IN_PROGRESS"; - protected static final List METER_TYPES = List.of("Count", "FiveMinuteRate", "MeanRate", "OneMinuteRate", "FifteenMinuteRate"); @@ -79,51 +68,20 @@ public abstract class PinotJMXToPromMetricsTest { "Min"); protected static final String TABLE_NAME_WITH_TYPE = - TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(LABEL_VAL_RAW_TABLENAME); + TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(ExportedLabelValues.TABLENAME); protected static final String KAFKA_TOPIC = "myTopic"; protected static final String PARTITION_GROUP_ID = "partitionGroupId"; protected static final String CLIENT_ID = String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC, PARTITION_GROUP_ID); - protected static final String TABLE_STREAM_NAME = String.format("%s_%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC); - protected static final List EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE = - List.of(LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME); - - protected static final List EXPORTED_LABELS_FOR_RAW_TABLE_NAME = - List.of(LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME); - - protected static final List EXPORTED_LABELS_FOR_CLIENT_ID = - List.of(LABEL_KEY_PARTITION, PARTITION_GROUP_ID, LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, - TableType.REALTIME.toString(), LABEL_KEY_TOPIC, KAFKA_TOPIC); - - protected static final List EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE = - List.of(LABEL_KEY_PARTITION, "3", LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, - TableType.REALTIME.toString()); - - protected static final List EXPORTED_LABELS_FOR_TABLE_TABLETYPE_TASKTYPE = - List.of(LABEL_KEY_TABLE, LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME, - LABEL_KEY_CONTROLLER_TASKTYPE, LABEL_VAL_CONTROLLER_TASKTYPE_CHC); - - protected static final List EXPORTED_LABELS_FOR_TABLENAMEANDTYPE_AND_TASKTYPE = - List.of(LABEL_KEY_TABLE, LABEL_VAL_TABLENAME_WITH_TYPE_REALTIME, LABEL_KEY_CONTROLLER_TASKTYPE, - LABEL_VAL_CONTROLLER_TASKTYPE_CHC); - - protected static final List EXPORTED_LABELS_FOR_TASK_TYPE_AND_STATUS = - List.of(LABEL_KEY_STATUS, LABEL_VAL_STATUS_IN_PROGRESS, LABEL_KEY_CONTROLLER_TASKTYPE, - LABEL_VAL_CONTROLLER_TASKTYPE_CHC); - - protected static final List EXPORTED_LABELS_PERIODIC_TASK_TABLE_TABLETYPE = - List.of(LABEL_KEY_CONTROLLER_PERIODIC_TASK, LABEL_VAL_CONTROLLER_TASKTYPE_CHC, LABEL_KEY_TABLE, - LABEL_VAL_RAW_TABLENAME, LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME); - /** - * For impl, see: https://github.com/prometheus/jmx_exporter/blob/a3b9443564ff5a78c25fd6566396fda2b7cbf216 + * For impl, see: ... * /jmx_prometheus_javaagent/src/main/java/io/prometheus/jmx/JavaAgent.java#L48 * @param pinotComponent the Pinot component to start the server for * @return the corresponding HTTP server on a random unoccupied port */ protected HTTPServer startExporter(PinotComponent pinotComponent) { - String args = String.format("%s:%s/%s", 0, CONFIG_DIR, PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); + String args = String.format("%s:%s/%s", 0, JMX_EXPORTER_CONFIG_PARENT_DIR, PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); try { JMXExporterConfig config = parseExporterConfig(args, "0.0.0.0"); CollectorRegistry registry = new CollectorRegistry(); @@ -372,4 +330,52 @@ static class JMXExporterConfig { public enum PinotComponent { SERVER, BROKER, CONTROLLER, MINION } + + public static class ExportedLabels { + public static final List TABLENAME_TABLETYPE = + List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME); + + public static final List TABLENAME = List.of(TABLE, ExportedLabelValues.TABLENAME); + + public static final List CLIENT_ID = + List.of(PARTITION, PARTITION_GROUP_ID, TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, + TableType.REALTIME.toString(), TOPIC, KAFKA_TOPIC); + + public static final List PARTITION_TABLE_NAME_AND_TYPE = + List.of(PARTITION, "3", TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TableType.REALTIME.toString()); + + public static final List TABLENAME_TABLETYPE_TASKTYPE = + List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME, TASKTYPE, + CONTROLLER_PERIODIC_TASK_CHC); + + public static final List TABLENAME_WITHTYPE_TASKTYPE = + List.of(TABLE, TABLENAME_WITH_TYPE_REALTIME, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC); + + public static final List STATUS_TASKTYPE = + List.of(STATUS, IN_PROGRESS, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC); + + public static final List PERIODIC_TASK_TABLE_TABLETYPE = + List.of(PERIODIC_TASK, CONTROLLER_PERIODIC_TASK_CHC, TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, + TABLETYPE_REALTIME); + } + + public static class ExportedLabelKeys { + public static final String TABLE = "table"; + public static final String TABLETYPE = "tableType"; + public static final String TOPIC = "topic"; + public static final String PARTITION = "partition"; + public static final String TASKTYPE = "taskType"; + public static final String PERIODIC_TASK = "periodicTask"; + public static final String STATUS = "status"; + public static final String DATABASE = "database"; + } + + public static class ExportedLabelValues { + public static final String TABLENAME = "myTable"; + public static final String TABLETYPE_REALTIME = "REALTIME"; + public static final String TABLENAME_WITH_TYPE_REALTIME = + TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(TABLENAME); + public static final String CONTROLLER_PERIODIC_TASK_CHC = "ClusterHealthCheck"; + public static final String IN_PROGRESS = "IN_PROGRESS"; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java similarity index 57% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java index f3724fd3e34b..6f7d1d79f8c8 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerJMXToPromMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java @@ -23,10 +23,8 @@ import com.yammer.metrics.reporting.JmxReporter; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; -import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; @@ -35,15 +33,44 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; -import org.testng.annotations.Parameters; import org.testng.annotations.Test; import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; -public class ServerJMXToPromMetricsTest extends PinotJMXToPromMetricsTest { - +public class ServerPrometheusMetricsTest extends PinotPrometheusMetricsTest { + //all exported server metrics have this prefix private static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; + + private static final List METERS_ACCEPTING_CLIENT_ID = + List.of(ServerMeter.REALTIME_ROWS_CONSUMED, ServerMeter.REALTIME_ROWS_SANITIZED, + ServerMeter.REALTIME_ROWS_FETCHED, ServerMeter.REALTIME_ROWS_FILTERED, + ServerMeter.INVALID_REALTIME_ROWS_DROPPED, ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, + ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, ServerMeter.ROWS_WITH_ERRORS); + + private static final List METERS_ACCEPTING_RAW_TABLE_NAMES = + List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, + ServerMeter.SEGMENT_UPLOAD_TIMEOUT); + + //gauges that accept clientId + private static final List GAUGES_ACCEPTING_CLIENT_ID = + List.of(ServerGauge.LLC_PARTITION_CONSUMING, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, + ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS); + + private static final List GAUGES_ACCEPTING_PARTITION = + List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, + ServerGauge.REALTIME_INGESTION_OFFSET_LAG, ServerGauge.REALTIME_INGESTION_DELAY_MS, + ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS, + ServerGauge.DEDUP_PRIMARY_KEYS_COUNT); + + private static final List GAUGES_ACCEPTING_RAW_TABLE_NAME = + List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, + ServerGauge.LUCENE_INDEXING_DELAY_MS, ServerGauge.LUCENE_INDEXING_DELAY_DOCS); + private ServerMetrics _serverMetrics; private HTTPServer _httpServer; @@ -70,36 +97,25 @@ public void setup() _httpClient = new HttpClient(); } - /** - * This test validates each timer defined in {@link ServerTimer} - */ @Test(dataProvider = "serverTimers") public void timerTest(ServerTimer serverTimer) { if (serverTimer.isGlobal()) { _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS); - } else { - _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); - _serverMetrics.addTimedTableValue(LABEL_VAL_RAW_TABLENAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); - } - if (serverTimer.isGlobal()) { assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_METRIC_PREFIX); } else { - assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); + _serverMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); + + assertTimerExportedCorrectly(serverTimer.getTimerName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(serverTimer.getTimerName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } } - /** - * This test validates each meter defined in {@link ServerMeter} - */ @Test(dataProvider = "serverMeters") public void meterTest(ServerMeter serverMeter) { - - //first, assert on all global meters if (serverMeter.isGlobal()) { - addGlobalMeter(serverMeter); + _serverMetrics.addMeteredGlobalValue(serverMeter, 4L); //we cannot use raw meter names for all meters as exported metrics don't follow any convention currently. // For example, meters that track realtime exceptions start with prefix "realtime_exceptions" if (meterTrackingRealtimeExceptions(serverMeter)) { @@ -108,100 +124,65 @@ public void meterTest(ServerMeter serverMeter) { assertMeterExportedCorrectly(serverMeter.getMeterName()); } } else { - - //these meters accept the clientId - List metersAcceptingClientId = - List.of(ServerMeter.REALTIME_ROWS_CONSUMED, ServerMeter.REALTIME_ROWS_SANITIZED, - ServerMeter.REALTIME_ROWS_FETCHED, ServerMeter.REALTIME_ROWS_FILTERED, - ServerMeter.INVALID_REALTIME_ROWS_DROPPED, ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, - ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, ServerMeter.ROWS_WITH_ERRORS); - - //these meters accept raw table name - List metersAcceptingRawTableNames = - List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, - ServerMeter.SEGMENT_UPLOAD_TIMEOUT); - - if (metersAcceptingClientId.contains(serverMeter)) { + if (METERS_ACCEPTING_CLIENT_ID.contains(serverMeter)) { addMeterWithLables(serverMeter, CLIENT_ID); - assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_CLIENT_ID); - } else if (metersAcceptingRawTableNames.contains(serverMeter)) { - addMeterWithLables(serverMeter, LABEL_VAL_RAW_TABLENAME); - assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME); + assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.CLIENT_ID); + } else if (METERS_ACCEPTING_RAW_TABLE_NAMES.contains(serverMeter)) { + addMeterWithLables(serverMeter, ExportedLabelValues.TABLENAME); + assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.TABLENAME); } else { + //we pass tableNameWithType to all remaining meters addMeterWithLables(serverMeter, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly(serverMeter.getMeterName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE); + assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE); } } } - /** - * This test validates each gauge defined in {@link ServerGauge} - */ @Test(dataProvider = "serverGauges") public void gaugeTest(ServerGauge serverGauge) { - - int partition = 3; - long someVal = 100L; - if (serverGauge.isGlobal()) { _serverMetrics.setValueOfGlobalGauge(serverGauge, 10L); assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_METRIC_PREFIX); } else { - //gauges that accept clientId - List gaugesAcceptingClientId = - List.of(ServerGauge.LLC_PARTITION_CONSUMING, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, - ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS); - - List gaugesAcceptingPartition = - List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, - ServerGauge.REALTIME_INGESTION_OFFSET_LAG, ServerGauge.REALTIME_INGESTION_DELAY_MS, - ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS, - ServerGauge.DEDUP_PRIMARY_KEYS_COUNT); - - List gaugesAcceptingRawTableName = - List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, - ServerGauge.LUCENE_INDEXING_DELAY_MS, ServerGauge.LUCENE_INDEXING_DELAY_DOCS); - - if (gaugesAcceptingClientId.contains(serverGauge)) { - _serverMetrics.setValueOfTableGauge(CLIENT_ID, serverGauge, TimeUnit.MILLISECONDS.toSeconds(someVal)); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_LABELS_FOR_CLIENT_ID, + if (serverGauge == ServerGauge.DEDUP_PRIMARY_KEYS_COUNT) { + //this gauge is currently exported as: `pinot_server_3_Value{database="dedupPrimaryKeysCount", + // table="dedupPrimaryKeysCount.myTable",tableType="REALTIME",}`. We add an explicit test for it to maintain + // backward compatibility. todo: ServerGauge.DEDUP_PRIMARY_KEYS_COUNT should be moved to + // gaugesThatAcceptPartition. It should be exported as: + // `pinot_server_dedupPrimaryKeysCount_Value{partition="3", table="myTable",tableType="REALTIME",}` + addPartitionGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); + assertGaugeExportedCorrectly(String.valueOf(3), + List.of(ExportedLabelKeys.DATABASE, serverGauge.getGaugeName(), ExportedLabelKeys.TABLE, + "dedupPrimaryKeysCount.myTable", ExportedLabelKeys.TABLETYPE, ExportedLabelValues.TABLETYPE_REALTIME), EXPORTED_METRIC_PREFIX); - } else if (gaugesAcceptingPartition.contains(serverGauge)) { - _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, serverGauge, someVal); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), - EXPORTED_LABELS_FOR_PARTITION_TABLE_NAME_AND_TYPE, EXPORTED_METRIC_PREFIX); - } else if (gaugesAcceptingRawTableName.contains(serverGauge)) { - _serverMetrics.setValueOfTableGauge(LABEL_VAL_RAW_TABLENAME, serverGauge, 5L); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_LABELS_FOR_RAW_TABLE_NAME, + } else if (GAUGES_ACCEPTING_CLIENT_ID.contains(serverGauge)) { + addGaugeWithLabels(serverGauge, CLIENT_ID); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.CLIENT_ID, EXPORTED_METRIC_PREFIX); + } else if (GAUGES_ACCEPTING_PARTITION.contains(serverGauge)) { + addPartitionGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.PARTITION_TABLE_NAME_AND_TYPE, EXPORTED_METRIC_PREFIX); + } else if (GAUGES_ACCEPTING_RAW_TABLE_NAME.contains(serverGauge)) { + addGaugeWithLabels(serverGauge, ExportedLabelValues.TABLENAME); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } else { - _serverMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, serverGauge, someVal); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_LABELS_FOR_TABLE_NAME_TABLE_TYPE, + addGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } } + } - //this gauge is currently exported as: `pinot_server_3_Value{database="dedupPrimaryKeysCount", - // table="dedupPrimaryKeysCount.myTable",tableType="REALTIME",}`. We add an explicit test for it to maintain - // backward compatibility. todo: ServerGauge.DEDUP_PRIMARY_KEYS_COUNT should be moved to - // gaugesThatAcceptPartition. It should be exported as: `pinot_server_dedupPrimaryKeysCount_Value{partition="3", - // table="myTable",tableType="REALTIME",}` -// _serverMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, partition, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT, 5L); -// assertGaugeExportedCorrectly(String.valueOf(partition), -// List.of(LABEL_KEY_DATABASE, "dedupPrimaryKeysCount", LABEL_KEY_TABLE, "dedupPrimaryKeysCount.myTable", -// LABEL_KEY_TABLETYPE, LABEL_VAL_TABLETYPE_REALTIME), EXPORTED_METRIC_PREFIX); + private void addGaugeWithLabels(ServerGauge serverGauge, String labels) { + _serverMetrics.setValueOfTableGauge(labels, serverGauge, 100L); } - public void addGlobalMeter(ServerMeter serverMeter) { - _serverMetrics.addMeteredGlobalValue(serverMeter, 4L); + private void addPartitionGaugeWithLabels(ServerGauge serverGauge, String labels) { + _serverMetrics.setValueOfPartitionGauge(labels, 3, serverGauge, 100L); } - public void addMeterWithLables(ServerMeter serverMeter, String label) { - _serverMetrics.addMeteredTableValue(label, serverMeter, 4L); + public void addMeterWithLables(ServerMeter serverMeter, String labels) { + _serverMetrics.addMeteredTableValue(labels, serverMeter, 4L); } private boolean meterTrackingRealtimeExceptions(ServerMeter serverMeter) { From 6dc98d8b0ffdf22ff11064278a921758479d5afa Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Sun, 20 Oct 2024 15:09:26 +0530 Subject: [PATCH 29/45] WIP --- .../metrics/BrokerPrometheusMetricsTest.java | 34 ++-- .../ControllerPrometheusMetricsTest.java | 151 +++++++++--------- .../metrics/MinionPrometheusMetricsTest.java | 56 +++---- .../metrics/PinotPrometheusMetricsTest.java | 15 +- .../metrics/ServerPrometheusMetricsTest.java | 10 +- 5 files changed, 133 insertions(+), 133 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java index a8317969ccb0..0086eebd44c7 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java @@ -44,6 +44,21 @@ public class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { private static final String EXPORTED_METRIC_PREFIX_EXCEPTIONS = "exceptions"; + + List GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX = + List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, + BrokerMeter.QUERY_REJECTED_EXCEPTIONS, BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, + BrokerMeter.RESOURCE_MISSING_EXCEPTIONS); + + List METERS_ACCEPTING_RAW_TABLENAME = + List.of(BrokerMeter.QUERIES, BrokerMeter.NO_SERVER_FOUND_EXCEPTIONS, BrokerMeter.DOCUMENTS_SCANNED, + BrokerMeter.ENTRIES_SCANNED_IN_FILTER, BrokerMeter.BROKER_RESPONSES_WITH_UNAVAILABLE_SEGMENTS, + BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, + BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, + BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED, BrokerMeter.BROKER_RESPONSES_WITH_TIMEOUTS, + BrokerMeter.ENTRIES_SCANNED_POST_FILTER, BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE, + BrokerMeter.QUERY_QUOTA_EXCEEDED); + private BrokerMetrics _brokerMetrics; private HTTPServer _httpServer; @@ -104,24 +119,9 @@ public void gaugeTest(BrokerGauge gauge) { @Test(dataProvider = "brokerMeters") public void meterTest(BrokerMeter meter) { - - List globalMetersWithExceptionsPrefix = - List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, - BrokerMeter.QUERY_REJECTED_EXCEPTIONS, BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, - BrokerMeter.RESOURCE_MISSING_EXCEPTIONS); - - List localMetersThatAcceptRawTableName = - List.of(BrokerMeter.QUERIES, BrokerMeter.NO_SERVER_FOUND_EXCEPTIONS, BrokerMeter.DOCUMENTS_SCANNED, - BrokerMeter.ENTRIES_SCANNED_IN_FILTER, BrokerMeter.BROKER_RESPONSES_WITH_UNAVAILABLE_SEGMENTS, - BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, - BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, - BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED, BrokerMeter.BROKER_RESPONSES_WITH_TIMEOUTS, - BrokerMeter.ENTRIES_SCANNED_POST_FILTER, BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE, - BrokerMeter.QUERY_QUOTA_EXCEEDED); - if (meter.isGlobal()) { _brokerMetrics.addMeteredGlobalValue(meter, 5L); - if (globalMetersWithExceptionsPrefix.contains(meter)) { + if (GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX.contains(meter)) { String exportedMeterPrefix = String.format("%s_%s", EXPORTED_METRIC_PREFIX_EXCEPTIONS, StringUtils.remove(meter.getMeterName(), "Exceptions")); assertMeterExportedCorrectly(exportedMeterPrefix, EXPORTED_METRIC_PREFIX); @@ -129,7 +129,7 @@ public void meterTest(BrokerMeter meter) { assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); } } else { - if (localMetersThatAcceptRawTableName.contains(meter)) { + if (METERS_ACCEPTING_RAW_TABLENAME.contains(meter)) { _brokerMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 5L); assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java index dd257b7b2ddf..981d4c577fe0 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java @@ -23,7 +23,6 @@ import com.yammer.metrics.reporting.JmxReporter; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; @@ -42,12 +41,29 @@ public class ControllerPrometheusMetricsTest extends PinotPrometheusMetricsTest { + //all exported controller metrics have this prefix private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; private static final String LABEL_KEY_TASK_TYPE = "taskType"; - private static final String TASK_TYPE = "ClusterHealthCheck"; - private ControllerMetrics _controllerMetrics; - private int _exporterPort; + //that accept global gauge with suffix + List GLOBAL_GAUGES_ACCEPTING_TASKTYPE = + List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, + ControllerGauge.NUM_MINION_SUBTASKS_WAITING, ControllerGauge.NUM_MINION_SUBTASKS_ERROR, + ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); + + //local gauges that accept partition + List GAUGES_ACCEPTING_PARTITION = + List.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS); + + //these accept task type + List GAUGES_ACCEPTING_TASKTYPE = + List.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, + ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, + ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR); + + List GAUGES_ACCEPTING_RAW_TABLENAME = List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); + + private ControllerMetrics _controllerMetrics; private HTTPServer _httpServer; @@ -76,7 +92,6 @@ public void setup() @Test(dataProvider = "controllerTimers") public void timerTest(ControllerTimer controllerTimer) { - if (controllerTimer.isGlobal()) { _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS); assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX); @@ -84,16 +99,15 @@ public void timerTest(ControllerTimer controllerTimer) { _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); _controllerMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(controllerTimer.getTimerName(), - ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly(controllerTimer.getTimerName(), ExportedLabels.TABLENAME, + + assertTimerExportedCorrectly(controllerTimer.getTimerName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(controllerTimer.getTimerName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } } @Test(dataProvider = "controllerMeters") public void meterTest(ControllerMeter meter) { - //global meters if (meter.isGlobal()) { _controllerMetrics.addMeteredGlobalValue(meter, 5L); String meterName = meter.getMeterName(); @@ -103,104 +117,87 @@ public void meterTest(ControllerMeter meter) { String strippedMeterName = StringUtils.remove(meterName, "controller"); assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX); } else { - if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR - || meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - _controllerMetrics.addMeteredTableValue(TASK_TYPE, meter, 1L); - } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { - _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE + "." + TASK_TYPE, meter, 1L); - } else { - _controllerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); - _controllerMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 5L); - } + String meterName = meter.getMeterName(); String strippedMeterName = StringUtils.remove(meterName, "controller"); + if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, List.of(ExportedLabelKeys.TABLE, TASK_TYPE), - EXPORTED_METRIC_PREFIX); - } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { - assertMeterExportedCorrectly(meterName, ExportedLabels.PERIODIC_TASK_TABLE_TABLETYPE, - EXPORTED_METRIC_PREFIX); + addMeterWithLabels(meter, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT); + assertMeterExportedCorrectly(meterName, + List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); } else if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - assertMeterExportedCorrectly(String.format("%s_%s", strippedMeterName, TASK_TYPE), EXPORTED_METRIC_PREFIX); - } else if (meter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { - assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME, + addMeterWithLabels(meter, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT); + assertMeterExportedCorrectly( + String.format("%s_%s", strippedMeterName, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); + } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { + addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE + "." + ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT); + assertMeterExportedCorrectly(meterName, ExportedLabels.PERIODIC_TASK_TABLE_TABLETYPE, EXPORTED_METRIC_PREFIX); } else { - assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME, - EXPORTED_METRIC_PREFIX); + addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE); + addMeterWithLabels(meter, ExportedLabelValues.TABLENAME); + if (meter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { + assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); + } } } } @Test(dataProvider = "controllerGauges") public void gaugeTest(ControllerGauge controllerGauge) { - //that accept global gauge with suffix - List globalGaugesWithTaskType = - List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, - ControllerGauge.NUM_MINION_SUBTASKS_WAITING, ControllerGauge.NUM_MINION_SUBTASKS_ERROR, - ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); - - //local gauges that accept partition - List gaugesAcceptingPartition = - List.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS); - - //these accept task type - List gaugesAcceptingTaskType = - List.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, - ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, - ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR); - - List gaugesAcceptingRawTableName = List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); - if (controllerGauge.isGlobal()) { - _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE, 1L); - if (globalGaugesWithTaskType.contains(controllerGauge)) { - _controllerMetrics.setValueOfGlobalGauge(controllerGauge, TASK_TYPE, 1L); + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, 1L); + //some global gauges also accept the taskType (which should not be). todo: this should be fixed + if (GLOBAL_GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) { + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, 1L); String strippedMetricName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedMetricName, List.of(LABEL_KEY_TASK_TYPE, TASK_TYPE), - EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(strippedMetricName, + List.of(LABEL_KEY_TASK_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); } else { _controllerMetrics.setValueOfGlobalGauge(controllerGauge, 1L); String strippedMetricName = getStrippedMetricName(controllerGauge); assertGaugeExportedCorrectly(strippedMetricName, EXPORTED_METRIC_PREFIX); } } else { - if (gaugesAcceptingPartition.contains(controllerGauge)) { + if (GAUGES_ACCEPTING_PARTITION.contains(controllerGauge)) { _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, 10L); String strippedGaugeName = getStrippedMetricName(controllerGauge); - ArrayList exportedLabels = - new ArrayList<>(ExportedLabels.PARTITION_TABLE_NAME_AND_TYPE); - assertGaugeExportedCorrectly(strippedGaugeName, exportedLabels, EXPORTED_METRIC_PREFIX); - } else if (gaugesAcceptingTaskType.contains(controllerGauge)) { - _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, TASK_TYPE, controllerGauge, () -> 50L); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), - ExportedLabels.TABLENAME_TABLETYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); - } else if (gaugesAcceptingRawTableName.contains(controllerGauge)) { - _controllerMetrics.setValueOfTableGauge(ExportedLabelValues.TABLENAME, controllerGauge, 5L); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME, + assertGaugeExportedCorrectly(strippedGaugeName, ExportedLabels.PARTITION_TABLE_NAME_AND_TYPE, + EXPORTED_METRIC_PREFIX); + } else if (GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) { + _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, + controllerGauge, () -> 50L); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); + } else if (GAUGES_ACCEPTING_RAW_TABLENAME.contains(controllerGauge)) { + addGaugeWithLabels(controllerGauge, ExportedLabelValues.TABLENAME); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } else if (controllerGauge == ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED) { - _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TABLE_NAME_WITH_TYPE, TASK_TYPE), - ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED, 5L); + addGaugeWithLabels(controllerGauge, + String.format("%s.%s", TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT)); assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(), ExportedLabels.TABLENAME_WITHTYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); } else if (controllerGauge == ControllerGauge.TASK_STATUS) { - _controllerMetrics.setValueOfTableGauge(String.format("%s.%s", TASK_TYPE, TaskState.IN_PROGRESS), - ControllerGauge.TASK_STATUS, 5); - assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), - ExportedLabels.STATUS_TASKTYPE, EXPORTED_METRIC_PREFIX); + addGaugeWithLabels(controllerGauge, + String.format("%s.%s", ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, TaskState.IN_PROGRESS)); + assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), ExportedLabels.STATUS_TASKTYPE, + EXPORTED_METRIC_PREFIX); } else { - _controllerMetrics.setValueOfTableGauge(TABLE_NAME_WITH_TYPE, controllerGauge, 5L); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), - ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + addGaugeWithLabels(controllerGauge, TABLE_NAME_WITH_TYPE); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); } } } + private void addGaugeWithLabels(ControllerGauge gauge, String labels) { + _controllerMetrics.setValueOfTableGauge(labels, gauge, 5L); + } + private static String getStrippedMetricName(ControllerGauge controllerGauge) { return StringUtils.remove(controllerGauge.getGaugeName(), "controller"); } @@ -233,4 +230,8 @@ public Object[] controllerGauges() { public void cleanup() { _httpServer.close(); } + + private void addMeterWithLabels(ControllerMeter meter, String labels) { + _controllerMetrics.addMeteredTableValue(labels, meter, 1L); + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java index 85e3615d56f3..9de1597a42e2 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java @@ -39,21 +39,12 @@ public class MinionPrometheusMetricsTest extends PinotPrometheusMetricsTest { - + //all exported minion metrics have this prefix private static final String EXPORTED_METRIC_PREFIX = "pinot_minion_"; - private static final String LABEL_KEY_TASKTYPE = "taskType"; - private static final String LABEL_VAL_TASKTYPE = "SegmentImportTask"; - private static final String LABEL_KEY_ID = "id"; private static final String METER_PREFIX_NO_TASKS = "numberTasks"; - private MinionMetrics _minionMetrics; private HTTPServer _httpServer; - private static final List EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE = - List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.TABLENAME, - ExportedLabelKeys.TABLETYPE, ExportedLabelValues.TABLETYPE_REALTIME, LABEL_KEY_TASKTYPE, - LABEL_VAL_TASKTYPE); - @BeforeClass public void setup() { @@ -79,18 +70,19 @@ public void setup() { @Test(dataProvider = "minionTimers") public void timerTest(MinionTimer timer) { - _minionMetrics.addTimedValue(LABEL_VAL_TASKTYPE, timer, 30L, TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(timer.getTimerName(), List.of(LABEL_KEY_ID, LABEL_VAL_TASKTYPE), - EXPORTED_METRIC_PREFIX); + _minionMetrics.addTimedValue(ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, timer, 30L, TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(timer.getTimerName(), + List.of(ExportedLabelKeys.ID, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); - _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, LABEL_VAL_TASKTYPE, timer, 30L, TimeUnit.MILLISECONDS); + _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, timer, 30L, + TimeUnit.MILLISECONDS); if (timer == MinionTimer.TASK_THREAD_CPU_TIME_NS) { assertTimerExportedCorrectly(timer.getTimerName(), - List.of(ExportedLabelKeys.DATABASE, ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME, - ExportedLabelKeys.TABLE, "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); + List.of(ExportedLabelKeys.DATABASE, ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME, ExportedLabelKeys.TABLE, + "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); } else { - assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, + assertTimerExportedCorrectly(timer.getTimerName(), ExportedLabels.EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); } } @@ -107,32 +99,28 @@ public void meterTest(MinionMeter meter) { private void validateMetersWithLabels(MinionMeter meter) { if (meter.getMeterName().startsWith(METER_PREFIX_NO_TASKS)) { _minionMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 1L); - assertMeterExportedCorrectly(meter.getMeterName(), - List.of(LABEL_KEY_ID, ExportedLabelValues.TABLENAME), EXPORTED_METRIC_PREFIX); - - _minionMetrics.addMeteredValue(LABEL_VAL_TASKTYPE, meter, 1L); - assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, LABEL_VAL_TASKTYPE), + assertMeterExportedCorrectly(meter.getMeterName(), List.of(ExportedLabelKeys.ID, ExportedLabelValues.TABLENAME), EXPORTED_METRIC_PREFIX); + + _minionMetrics.addMeteredValue(ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, meter, 1L); + assertMeterExportedCorrectly(meter.getMeterName(), + List.of(ExportedLabelKeys.ID, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); } else if (meter == MinionMeter.SEGMENT_UPLOAD_FAIL_COUNT || meter == MinionMeter.SEGMENT_DOWNLOAD_FAIL_COUNT) { _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 1L); - assertMeterExportedCorrectly(meter.getMeterName(), List.of(LABEL_KEY_ID, TABLE_NAME_WITH_TYPE), + assertMeterExportedCorrectly(meter.getMeterName(), List.of(ExportedLabelKeys.ID, TABLE_NAME_WITH_TYPE), EXPORTED_METRIC_PREFIX); } else { //all remaining meters are also being used as global meters, check their usage _minionMetrics.addMeteredGlobalValue(meter, 1L); - _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, LABEL_VAL_TASKTYPE, meter, 1L); + _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, meter, + 1L); assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, + assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); } } - private void validateGlobalMeters(MinionMeter meter) { - _minionMetrics.addMeteredGlobalValue(meter, 5L); - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); - } - @Test(dataProvider = "minionGauges") public void gaugeTest(MinionGauge gauge) { if (gauge.isGlobal()) { @@ -140,8 +128,7 @@ public void gaugeTest(MinionGauge gauge) { assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); } else { _minionMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 1L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } } @@ -173,4 +160,9 @@ public Object[] minionGauges() { public void cleanup() { _httpServer.close(); } + + private void validateGlobalMeters(MinionMeter meter) { + _minionMetrics.addMeteredGlobalValue(meter, 5L); + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java index a96045694906..94be0e27dd18 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java @@ -346,21 +346,27 @@ public static class ExportedLabels { public static final List TABLENAME_TABLETYPE_TASKTYPE = List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME, TASKTYPE, - CONTROLLER_PERIODIC_TASK_CHC); + MINION_TASK_SEGMENT_IMPORT); public static final List TABLENAME_WITHTYPE_TASKTYPE = - List.of(TABLE, TABLENAME_WITH_TYPE_REALTIME, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC); + List.of(TABLE, TABLENAME_WITH_TYPE_REALTIME, TASKTYPE, MINION_TASK_SEGMENT_IMPORT); public static final List STATUS_TASKTYPE = - List.of(STATUS, IN_PROGRESS, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC); + List.of(STATUS, IN_PROGRESS, TASKTYPE, MINION_TASK_SEGMENT_IMPORT); public static final List PERIODIC_TASK_TABLE_TABLETYPE = - List.of(PERIODIC_TASK, CONTROLLER_PERIODIC_TASK_CHC, TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, + List.of(PERIODIC_TASK, MINION_TASK_SEGMENT_IMPORT, TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME); + + public static final List EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE = + List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.TABLENAME, ExportedLabelKeys.TABLETYPE, + ExportedLabelValues.TABLETYPE_REALTIME, ExportedLabelKeys.TASKTYPE, + ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT); } public static class ExportedLabelKeys { public static final String TABLE = "table"; + public static final String ID = "id"; public static final String TABLETYPE = "tableType"; public static final String TOPIC = "topic"; public static final String PARTITION = "partition"; @@ -376,6 +382,7 @@ public static class ExportedLabelValues { public static final String TABLENAME_WITH_TYPE_REALTIME = TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(TABLENAME); public static final String CONTROLLER_PERIODIC_TASK_CHC = "ClusterHealthCheck"; + public static final String MINION_TASK_SEGMENT_IMPORT = "SegmentImportTask"; public static final String IN_PROGRESS = "IN_PROGRESS"; } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java index 6f7d1d79f8c8..e6d40e369cb4 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java @@ -125,14 +125,14 @@ public void meterTest(ServerMeter serverMeter) { } } else { if (METERS_ACCEPTING_CLIENT_ID.contains(serverMeter)) { - addMeterWithLables(serverMeter, CLIENT_ID); + addMeterWithLabels(serverMeter, CLIENT_ID); assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.CLIENT_ID); } else if (METERS_ACCEPTING_RAW_TABLE_NAMES.contains(serverMeter)) { - addMeterWithLables(serverMeter, ExportedLabelValues.TABLENAME); + addMeterWithLabels(serverMeter, ExportedLabelValues.TABLENAME); assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.TABLENAME); } else { //we pass tableNameWithType to all remaining meters - addMeterWithLables(serverMeter, TABLE_NAME_WITH_TYPE); + addMeterWithLabels(serverMeter, TABLE_NAME_WITH_TYPE); assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE); } } @@ -145,7 +145,7 @@ public void gaugeTest(ServerGauge serverGauge) { assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_METRIC_PREFIX); } else { if (serverGauge == ServerGauge.DEDUP_PRIMARY_KEYS_COUNT) { - //this gauge is currently exported as: `pinot_server_3_Value{database="dedupPrimaryKeysCount", + //this gauge is currently exported as: `pinot_server_${partitionId}_Value{database="dedupPrimaryKeysCount", // table="dedupPrimaryKeysCount.myTable",tableType="REALTIME",}`. We add an explicit test for it to maintain // backward compatibility. todo: ServerGauge.DEDUP_PRIMARY_KEYS_COUNT should be moved to // gaugesThatAcceptPartition. It should be exported as: @@ -181,7 +181,7 @@ private void addPartitionGaugeWithLabels(ServerGauge serverGauge, String labels) _serverMetrics.setValueOfPartitionGauge(labels, 3, serverGauge, 100L); } - public void addMeterWithLables(ServerMeter serverMeter, String labels) { + public void addMeterWithLabels(ServerMeter serverMeter, String labels) { _serverMetrics.addMeteredTableValue(labels, serverMeter, 4L); } From 1f20b203da95680713563f8edadd4722016b2f9a Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Sun, 20 Oct 2024 16:48:46 +0530 Subject: [PATCH 30/45] WIP --- pinot-common/pom.xml | 5 ++ .../metrics/BrokerPrometheusMetricsTest.java | 46 ++++------ .../metrics/PinotPrometheusMetricsTest.java | 83 +++++++++++++++++-- .../test/resources/metrics/testConfig.json | 3 + 4 files changed, 100 insertions(+), 37 deletions(-) create mode 100644 pinot-common/src/test/resources/metrics/testConfig.json diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index ea5cbc30c4a9..8f0c09634cb6 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -273,6 +273,11 @@ equalsverifier test + + org.apache.pinot + pinot-dropwizard + test + diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java index 0086eebd44c7..14b19e0701ee 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java @@ -22,13 +22,17 @@ import com.yammer.metrics.core.MetricsRegistry; import com.yammer.metrics.reporting.JmxReporter; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; +import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.util.List; import java.util.concurrent.TimeUnit; +import javax.inject.Inject; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; import org.testng.annotations.BeforeClass; @@ -44,13 +48,12 @@ public class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { private static final String EXPORTED_METRIC_PREFIX_EXCEPTIONS = "exceptions"; - - List GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX = + private static final List GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX = List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, BrokerMeter.QUERY_REJECTED_EXCEPTIONS, BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, BrokerMeter.RESOURCE_MISSING_EXCEPTIONS); - List METERS_ACCEPTING_RAW_TABLENAME = + private static final List METERS_ACCEPTING_RAW_TABLENAME = List.of(BrokerMeter.QUERIES, BrokerMeter.NO_SERVER_FOUND_EXCEPTIONS, BrokerMeter.DOCUMENTS_SCANNED, BrokerMeter.ENTRIES_SCANNED_IN_FILTER, BrokerMeter.BROKER_RESPONSES_WITH_UNAVAILABLE_SEGMENTS, BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, @@ -63,26 +66,13 @@ public class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { private HTTPServer _httpServer; + private PinotMetricsFactory _pinotMetricsFactory; + @BeforeClass public void setup() throws Exception { - _httpServer = startExporter(PinotComponent.BROKER); - - PinotConfiguration pinotConfiguration = new PinotConfiguration(); - pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, - "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); - PinotMetricUtils.init(pinotConfiguration); - - // Initialize ServerMetrics with the registry - YammerMetricsRegistry yammerMetricsRegistry = new YammerMetricsRegistry(); - _brokerMetrics = new BrokerMetrics(yammerMetricsRegistry); - - // Enable JMX reporting - MetricsRegistry metricsRegistry = (MetricsRegistry) yammerMetricsRegistry.getMetricsRegistry(); - JmxReporter jmxReporter = new JmxReporter(metricsRegistry); - jmxReporter.start(); - + _brokerMetrics = new BrokerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); _httpClient = new HttpClient(); } @@ -92,10 +82,8 @@ public void timerTest(BrokerTimer timer) { _brokerMetrics.addTimedValue(timer, 30_000, TimeUnit.MILLISECONDS); assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_METRIC_PREFIX); } else { - _brokerMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, timer, 30_000L, - TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(timer.getTimerName(), ExportedLabels.TABLENAME, - EXPORTED_METRIC_PREFIX); + _brokerMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, timer, 30_000L, TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(timer.getTimerName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } } @@ -107,12 +95,10 @@ public void gaugeTest(BrokerGauge gauge) { } else { if (gauge == BrokerGauge.REQUEST_SIZE) { _brokerMetrics.setOrUpdateTableGauge(ExportedLabelValues.TABLENAME, gauge, 5L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME, - EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } else { _brokerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } } } @@ -131,12 +117,10 @@ public void meterTest(BrokerMeter meter) { } else { if (METERS_ACCEPTING_RAW_TABLENAME.contains(meter)) { _brokerMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 5L); - assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME, - EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } else { _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); - assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java index 94be0e27dd18..6077613f98ec 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java @@ -19,7 +19,9 @@ package org.apache.pinot.common.metrics; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Objects; +import com.google.common.io.Resources; import io.prometheus.jmx.JmxCollector; import io.prometheus.jmx.common.http.HTTPServerFactory; import io.prometheus.jmx.shaded.io.prometheus.client.CollectorRegistry; @@ -30,6 +32,11 @@ import java.io.IOException; import java.io.StringReader; import java.net.InetSocketAddress; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -37,21 +44,32 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; +import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; import org.apache.pinot.spi.config.table.TableType; +import org.apache.pinot.spi.env.PinotConfiguration; +import org.apache.pinot.spi.metrics.PinotMetricUtils; +import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelKeys.*; import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.*; +import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; public abstract class PinotPrometheusMetricsTest { //this dir contains the JMX exporter configs for each Pinot component (broker, server, controller, minion) - private static final String JMX_EXPORTER_CONFIG_PARENT_DIR = "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs"; + private static final String JMX_EXPORTER_CONFIG_PARENT_DIR = + "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs"; //this map is a mapping of pinot components to their JMX exporter config files. They can be found at: // docker/images/pinot/etc/jmx_prometheus_javaagent/configs @@ -74,18 +92,70 @@ public abstract class PinotPrometheusMetricsTest { protected static final String PARTITION_GROUP_ID = "partitionGroupId"; protected static final String CLIENT_ID = String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC, PARTITION_GROUP_ID); + + private PinotMetricsFactory _pinotMetricsFactory; + + @BeforeClass + public void setupBase() + throws Exception { + JsonNode jsonNode = JsonUtils.DEFAULT_READER.readTree(loadResourceAsString("metrics/testConfig.json")); + String pinotMetricsFactory = jsonNode.get("pinotMetricsFactory").toString(); + switch (pinotMetricsFactory) { + case "\"YammerMetricsFactory\"": + _pinotMetricsFactory = new YammerMetricsFactory(); + break; + case "\"DropwizardMetricsFactory\"": + _pinotMetricsFactory = new DropwizardMetricsFactory(); + break; + default: + throw new IllegalArgumentException("Unknow metrics factory specified in test config: " + pinotMetricsFactory + + ", supported ones are: YammerMetricsFactory and DropwizardMetricsFactory"); + } + PinotConfiguration pinotConfiguration = new PinotConfiguration(); + pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, _pinotMetricsFactory.getClass().getCanonicalName()); + PinotMetricUtils.init(pinotConfiguration); + + _pinotMetricsFactory.makePinotJmxReporter(_pinotMetricsFactory.getPinotMetricsRegistry()).start(); + } + + private String loadResourceAsString(String resourceFileName) { + URL url = Resources.getResource(resourceFileName); + try { + return Resources.toString(url, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @DataProvider + public Object[][] configs() + throws IOException { + try (Stream configs = Files.list(Paths.get("src/test/resources/testConfigs/testConfig.json"))) { + return configs.map(path -> { + try { + return Files.readAllBytes(path); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).map(config -> new Object[]{config}).toArray(Object[][]::new); + } + } + /** - * For impl, see: ... + * For impl, see: + * ... * /jmx_prometheus_javaagent/src/main/java/io/prometheus/jmx/JavaAgent.java#L48 * @param pinotComponent the Pinot component to start the server for * @return the corresponding HTTP server on a random unoccupied port */ protected HTTPServer startExporter(PinotComponent pinotComponent) { - String args = String.format("%s:%s/%s", 0, JMX_EXPORTER_CONFIG_PARENT_DIR, PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); + String args = String.format("%s:%s/%s", 0, JMX_EXPORTER_CONFIG_PARENT_DIR, + PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); try { JMXExporterConfig config = parseExporterConfig(args, "0.0.0.0"); CollectorRegistry registry = new CollectorRegistry(); - (new JmxCollector(new File(config._file), JmxCollector.Mode.AGENT)).register(registry); + JmxCollector jmxCollector = new JmxCollector(new File(config._file), JmxCollector.Mode.AGENT); + jmxCollector.register(registry); DefaultExports.register(registry); return (new HTTPServerFactory()).createHTTPServer(config._socket, registry, true, new File(config._file)); } catch (Exception e) { @@ -124,8 +194,9 @@ protected void assertTimerExportedCorrectly(String exportedTimerPrefix, String e try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); for (String meterType : TIMER_TYPES) { - Assert.assertTrue( - promMetrics.contains(PromMetric.withName(exportedMetricPrefix + exportedTimerPrefix + "_" + meterType))); + PromMetric expectedTimer = PromMetric.withName(exportedMetricPrefix + exportedTimerPrefix + "_" + meterType); + Assert.assertTrue(promMetrics.contains(expectedTimer), + "Cannot find timer: " + expectedTimer + " in exported metrics"); } } catch (Exception e) { throw new RuntimeException(e); diff --git a/pinot-common/src/test/resources/metrics/testConfig.json b/pinot-common/src/test/resources/metrics/testConfig.json new file mode 100644 index 000000000000..da29c6a919f7 --- /dev/null +++ b/pinot-common/src/test/resources/metrics/testConfig.json @@ -0,0 +1,3 @@ +{ + "pinotMetricsFactory": "YammerMetricsFactory" +} \ No newline at end of file From eff9b92ee13eaa78fd2ab44379212a97a154eafd Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Sun, 20 Oct 2024 21:36:49 +0530 Subject: [PATCH 31/45] WIP --- .../metrics/BrokerPrometheusMetricsTest.java | 3 -- .../metrics/PinotPrometheusMetricsTest.java | 46 +++++++++++++------ .../test/resources/metrics/testConfig.json | 7 ++- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java index 14b19e0701ee..6f2b1c5901f9 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java @@ -66,14 +66,11 @@ public class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { private HTTPServer _httpServer; - private PinotMetricsFactory _pinotMetricsFactory; - @BeforeClass public void setup() throws Exception { _httpServer = startExporter(PinotComponent.BROKER); _brokerMetrics = new BrokerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); - _httpClient = new HttpClient(); } @Test(dataProvider = "brokerTimers") diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java index 6077613f98ec..622ffd0581b0 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java @@ -38,6 +38,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -67,15 +68,13 @@ public abstract class PinotPrometheusMetricsTest { - //this dir contains the JMX exporter configs for each Pinot component (broker, server, controller, minion) - private static final String JMX_EXPORTER_CONFIG_PARENT_DIR = - "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs"; + private static final String CONFIG_KEY_JMX_EXPORTER_PARENT_DIR = "jmxExporterConfigsParentDir"; + private static final String CONFIG_KEY_PINOT_METRICS_FACTORY = "pinotMetricsFactory"; + private static final String CONFIG_KEY_CONTROLLER_CONFIG_FILE_NAME = "controllerConfigFileName"; + private static final String CONFIG_KEY_SERVER_CONFIG_FILE_NAME = "serverConfigFileName"; + private static final String CONFIG_KEY_BROKER_CONFIG_FILE_NAME = "brokerConfigFileName"; + private static final String CONFIG_KEY_MINION_CONFIG_FILE_NAME = "minionConfigFileName"; - //this map is a mapping of pinot components to their JMX exporter config files. They can be found at: - // docker/images/pinot/etc/jmx_prometheus_javaagent/configs - private static final Map PINOT_COMPONENT_CONFIG_FILE_MAP = - Map.of(PinotComponent.CONTROLLER, "controller.yml", PinotComponent.SERVER, "server.yml", PinotComponent.MINION, - "minion.yml", PinotComponent.BROKER, "broker.yml"); protected HttpClient _httpClient; protected static final List METER_TYPES = List.of("Count", "FiveMinuteRate", "MeanRate", "OneMinuteRate", "FifteenMinuteRate"); @@ -93,18 +92,33 @@ public abstract class PinotPrometheusMetricsTest { protected static final String CLIENT_ID = String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC, PARTITION_GROUP_ID); - private PinotMetricsFactory _pinotMetricsFactory; + protected String _exporterConfigsParentDir; + + protected PinotMetricsFactory _pinotMetricsFactory; + + protected Map _pinotComponentToConfigFileMap = new HashMap<>(); @BeforeClass public void setupBase() throws Exception { + //read test configuration JsonNode jsonNode = JsonUtils.DEFAULT_READER.readTree(loadResourceAsString("metrics/testConfig.json")); - String pinotMetricsFactory = jsonNode.get("pinotMetricsFactory").toString(); + _exporterConfigsParentDir = jsonNode.get(CONFIG_KEY_JMX_EXPORTER_PARENT_DIR).textValue(); + _pinotComponentToConfigFileMap.put(PinotComponent.CONTROLLER, + jsonNode.get(CONFIG_KEY_CONTROLLER_CONFIG_FILE_NAME).textValue()); + _pinotComponentToConfigFileMap.put(PinotComponent.SERVER, + jsonNode.get(CONFIG_KEY_SERVER_CONFIG_FILE_NAME).textValue()); + _pinotComponentToConfigFileMap.put(PinotComponent.BROKER, + jsonNode.get(CONFIG_KEY_BROKER_CONFIG_FILE_NAME).textValue()); + _pinotComponentToConfigFileMap.put(PinotComponent.MINION, + jsonNode.get(CONFIG_KEY_MINION_CONFIG_FILE_NAME).textValue()); + + String pinotMetricsFactory = jsonNode.get(CONFIG_KEY_PINOT_METRICS_FACTORY).textValue(); switch (pinotMetricsFactory) { - case "\"YammerMetricsFactory\"": + case "YammerMetricsFactory": _pinotMetricsFactory = new YammerMetricsFactory(); break; - case "\"DropwizardMetricsFactory\"": + case "DropwizardMetricsFactory": _pinotMetricsFactory = new DropwizardMetricsFactory(); break; default: @@ -112,10 +126,12 @@ public void setupBase() + ", supported ones are: YammerMetricsFactory and DropwizardMetricsFactory"); } PinotConfiguration pinotConfiguration = new PinotConfiguration(); - pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, _pinotMetricsFactory.getClass().getCanonicalName()); + pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, + _pinotMetricsFactory.getClass().getCanonicalName()); PinotMetricUtils.init(pinotConfiguration); _pinotMetricsFactory.makePinotJmxReporter(_pinotMetricsFactory.getPinotMetricsRegistry()).start(); + _httpClient = new HttpClient(); } private String loadResourceAsString(String resourceFileName) { @@ -149,8 +165,8 @@ public Object[][] configs() * @return the corresponding HTTP server on a random unoccupied port */ protected HTTPServer startExporter(PinotComponent pinotComponent) { - String args = String.format("%s:%s/%s", 0, JMX_EXPORTER_CONFIG_PARENT_DIR, - PINOT_COMPONENT_CONFIG_FILE_MAP.get(pinotComponent)); + String args = + String.format("%s:%s/%s", 0, _exporterConfigsParentDir, _pinotComponentToConfigFileMap.get(pinotComponent)); try { JMXExporterConfig config = parseExporterConfig(args, "0.0.0.0"); CollectorRegistry registry = new CollectorRegistry(); diff --git a/pinot-common/src/test/resources/metrics/testConfig.json b/pinot-common/src/test/resources/metrics/testConfig.json index da29c6a919f7..5bdc0a6d66f6 100644 --- a/pinot-common/src/test/resources/metrics/testConfig.json +++ b/pinot-common/src/test/resources/metrics/testConfig.json @@ -1,3 +1,8 @@ { - "pinotMetricsFactory": "YammerMetricsFactory" + "pinotMetricsFactory": "YammerMetricsFactory", + "jmxExporterConfigsParentDir": "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs", + "controllerConfigFileName": "controller.yml", + "serverConfigFileName": "server.yml", + "brokerConfigFileName": "broker.yml", + "minionConfigFileName": "minion.yml" } \ No newline at end of file From 6fdd03793fe1e44b9f3e8ae5aae67105bf9e742e Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Sun, 20 Oct 2024 22:27:36 +0530 Subject: [PATCH 32/45] WIP --- .../ControllerPrometheusMetricsTest.java | 52 ++++--------- .../metrics/MinionPrometheusMetricsTest.java | 18 +---- .../metrics/PinotPrometheusMetricsTest.java | 74 +++++++++---------- .../metrics/ServerPrometheusMetricsTest.java | 18 +---- 4 files changed, 52 insertions(+), 110 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java index 981d4c577fe0..86014f228795 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java @@ -19,8 +19,6 @@ package org.apache.pinot.common.metrics; -import com.yammer.metrics.core.MetricsRegistry; -import com.yammer.metrics.reporting.JmxReporter; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; import java.util.List; @@ -28,17 +26,11 @@ import org.apache.commons.lang3.StringUtils; import org.apache.helix.task.TaskState; import org.apache.pinot.common.utils.SimpleHttpResponse; -import org.apache.pinot.common.utils.http.HttpClient; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; -import org.apache.pinot.spi.env.PinotConfiguration; -import org.apache.pinot.spi.metrics.PinotMetricUtils; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; - public class ControllerPrometheusMetricsTest extends PinotPrometheusMetricsTest { //all exported controller metrics have this prefix @@ -70,24 +62,8 @@ public class ControllerPrometheusMetricsTest extends PinotPrometheusMetricsTest @BeforeClass public void setup() throws Exception { - _httpServer = startExporter(PinotComponent.CONTROLLER); - - PinotConfiguration pinotConfiguration = new PinotConfiguration(); - pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, - "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); - PinotMetricUtils.init(pinotConfiguration); - - // Initialize ControllerMetrics with the registry - YammerMetricsRegistry yammerMetricsRegistry = new YammerMetricsRegistry(); - _controllerMetrics = new ControllerMetrics(yammerMetricsRegistry); - - // Enable JMX reporting - MetricsRegistry metricsRegistry = (MetricsRegistry) yammerMetricsRegistry.getMetricsRegistry(); - JmxReporter jmxReporter = new JmxReporter(metricsRegistry); - jmxReporter.start(); - - _httpClient = new HttpClient(); + _controllerMetrics = new ControllerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); } @Test(dataProvider = "controllerTimers") @@ -122,16 +98,16 @@ public void meterTest(ControllerMeter meter) { String strippedMeterName = StringUtils.remove(meterName, "controller"); if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { - addMeterWithLabels(meter, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT); + addMeterWithLabels(meter, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); assertMeterExportedCorrectly(meterName, - List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); + List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), EXPORTED_METRIC_PREFIX); } else if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - addMeterWithLabels(meter, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT); + addMeterWithLabels(meter, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); assertMeterExportedCorrectly( - String.format("%s_%s", strippedMeterName, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), + String.format("%s_%s", strippedMeterName, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), EXPORTED_METRIC_PREFIX); } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { - addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE + "." + ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT); + addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE + "." + ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); assertMeterExportedCorrectly(meterName, ExportedLabels.PERIODIC_TASK_TABLE_TABLETYPE, EXPORTED_METRIC_PREFIX); } else { addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE); @@ -150,13 +126,13 @@ public void meterTest(ControllerMeter meter) { @Test(dataProvider = "controllerGauges") public void gaugeTest(ControllerGauge controllerGauge) { if (controllerGauge.isGlobal()) { - _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, 1L); + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, 1L); //some global gauges also accept the taskType (which should not be). todo: this should be fixed if (GLOBAL_GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) { - _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, 1L); + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, 1L); String strippedMetricName = getStrippedMetricName(controllerGauge); assertGaugeExportedCorrectly(strippedMetricName, - List.of(LABEL_KEY_TASK_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); + List.of(LABEL_KEY_TASK_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), EXPORTED_METRIC_PREFIX); } else { _controllerMetrics.setValueOfGlobalGauge(controllerGauge, 1L); String strippedMetricName = getStrippedMetricName(controllerGauge); @@ -169,21 +145,21 @@ public void gaugeTest(ControllerGauge controllerGauge) { assertGaugeExportedCorrectly(strippedGaugeName, ExportedLabels.PARTITION_TABLE_NAME_AND_TYPE, EXPORTED_METRIC_PREFIX); } else if (GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) { - _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, + _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, controllerGauge, () -> 50L); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE_TASKTYPE, + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); } else if (GAUGES_ACCEPTING_RAW_TABLENAME.contains(controllerGauge)) { addGaugeWithLabels(controllerGauge, ExportedLabelValues.TABLENAME); assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); } else if (controllerGauge == ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED) { addGaugeWithLabels(controllerGauge, - String.format("%s.%s", TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT)); + String.format("%s.%s", TABLE_NAME_WITH_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC)); assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(), - ExportedLabels.TABLENAME_WITHTYPE_TASKTYPE, EXPORTED_METRIC_PREFIX); + ExportedLabels.TABLENAME_WITHTYPE_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); } else if (controllerGauge == ControllerGauge.TASK_STATUS) { addGaugeWithLabels(controllerGauge, - String.format("%s.%s", ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, TaskState.IN_PROGRESS)); + String.format("%s.%s", ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, TaskState.IN_PROGRESS)); assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), ExportedLabels.STATUS_TASKTYPE, EXPORTED_METRIC_PREFIX); } else { diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java index 9de1597a42e2..8b96764e2387 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java @@ -47,24 +47,8 @@ public class MinionPrometheusMetricsTest extends PinotPrometheusMetricsTest { @BeforeClass public void setup() { - _httpServer = startExporter(PinotComponent.MINION); - - PinotConfiguration pinotConfiguration = new PinotConfiguration(); - pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, - "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); - PinotMetricUtils.init(pinotConfiguration); - - // Initialize ControllerMetrics with the registry - YammerMetricsRegistry yammerMetricsRegistry = new YammerMetricsRegistry(); - _minionMetrics = new MinionMetrics(yammerMetricsRegistry); - - // Enable JMX reporting - MetricsRegistry metricsRegistry = (MetricsRegistry) yammerMetricsRegistry.getMetricsRegistry(); - JmxReporter jmxReporter = new JmxReporter(metricsRegistry); - jmxReporter.start(); - - _httpClient = new HttpClient(); + _minionMetrics = new MinionMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); } @Test(dataProvider = "minionTimers") diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java index 622ffd0581b0..b4e324886231 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java @@ -34,9 +34,6 @@ import java.net.InetSocketAddress; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -45,7 +42,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; @@ -59,7 +55,6 @@ import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelKeys.*; import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.*; @@ -68,6 +63,7 @@ public abstract class PinotPrometheusMetricsTest { + //config keys private static final String CONFIG_KEY_JMX_EXPORTER_PARENT_DIR = "jmxExporterConfigsParentDir"; private static final String CONFIG_KEY_PINOT_METRICS_FACTORY = "pinotMetricsFactory"; private static final String CONFIG_KEY_CONTROLLER_CONFIG_FILE_NAME = "controllerConfigFileName"; @@ -75,31 +71,36 @@ public abstract class PinotPrometheusMetricsTest { private static final String CONFIG_KEY_BROKER_CONFIG_FILE_NAME = "brokerConfigFileName"; private static final String CONFIG_KEY_MINION_CONFIG_FILE_NAME = "minionConfigFileName"; - protected HttpClient _httpClient; - protected static final List METER_TYPES = + //each meter defined in code is exported with these measurements + private static final List METER_TYPES = List.of("Count", "FiveMinuteRate", "MeanRate", "OneMinuteRate", "FifteenMinuteRate"); - protected static final List TIMER_TYPES = + //each timer defined in code is exported with these measurements + private static final List TIMER_TYPES = List.of("Count", "FiveMinuteRate", "Max", "999thPercentile", "95thPercentile", "75thPercentile", "98thPercentile", "OneMinuteRate", "50thPercentile", "99thPercentile", "FifteenMinuteRate", "Mean", "StdDev", "MeanRate", "Min"); + //each gauge defined in code is exported with these measurements + private static final List GAUGE_TYPES = List.of("Value"); + protected static final String TABLE_NAME_WITH_TYPE = TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(ExportedLabelValues.TABLENAME); - protected static final String KAFKA_TOPIC = "myTopic"; protected static final String PARTITION_GROUP_ID = "partitionGroupId"; protected static final String CLIENT_ID = String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC, PARTITION_GROUP_ID); - protected String _exporterConfigsParentDir; - protected PinotMetricsFactory _pinotMetricsFactory; - protected Map _pinotComponentToConfigFileMap = new HashMap<>(); + protected HttpClient _httpClient; + + private String _exporterConfigsParentDir; + + private final Map _pinotComponentToConfigFileMap = new HashMap<>(); @BeforeClass - public void setupBase() + public void setupTest() throws Exception { //read test configuration JsonNode jsonNode = JsonUtils.DEFAULT_READER.readTree(loadResourceAsString("metrics/testConfig.json")); @@ -143,20 +144,6 @@ private String loadResourceAsString(String resourceFileName) { } } - @DataProvider - public Object[][] configs() - throws IOException { - try (Stream configs = Files.list(Paths.get("src/test/resources/testConfigs/testConfig.json"))) { - return configs.map(path -> { - try { - return Files.readAllBytes(path); - } catch (IOException e) { - throw new RuntimeException(e); - } - }).map(config -> new Object[]{config}).toArray(Object[][]::new); - } - } - /** * For impl, see: * ... @@ -183,9 +170,11 @@ protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, String e List promMetrics; try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - PromMetric expectedMetric = PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value"); - Assert.assertTrue(promMetrics.contains(expectedMetric), - "Cannot find gauge: " + expectedMetric + " in exported metrics"); + for (String gaugeType : GAUGE_TYPES) { + PromMetric expectedMetric = PromMetric.withName(exportedMetricPrefix + exportedGaugePrefix + "_" + gaugeType); + Assert.assertTrue(promMetrics.contains(expectedMetric), + "Cannot find gauge: " + expectedMetric + " in exported metrics"); + } } catch (Exception e) { throw new RuntimeException(e); } @@ -196,10 +185,12 @@ protected void assertGaugeExportedCorrectly(String exportedGaugePrefix, List promMetrics; try { promMetrics = parseExportedPromMetrics(getExportedPromMetrics().getResponse()); - PromMetric expectedGauge = - PromMetric.withNameAndLabels(exportedMetricPrefix + exportedGaugePrefix + "_" + "Value", labels); - Assert.assertTrue(promMetrics.contains(expectedGauge), - "Cannot find gauge: " + expectedGauge + " in exported metrics"); + for (String gaugeType : GAUGE_TYPES) { + PromMetric expectedGauge = + PromMetric.withNameAndLabels(exportedMetricPrefix + exportedGaugePrefix + "_" + gaugeType, labels); + Assert.assertTrue(promMetrics.contains(expectedGauge), + "Cannot find gauge: " + expectedGauge + " in exported metrics"); + } } catch (Exception e) { throw new RuntimeException(e); } @@ -431,18 +422,25 @@ public static class ExportedLabels { public static final List PARTITION_TABLE_NAME_AND_TYPE = List.of(PARTITION, "3", TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TableType.REALTIME.toString()); - public static final List TABLENAME_TABLETYPE_TASKTYPE = + public static final List TABLENAME_TABLETYPE_MINION_TASKTYPE = List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME, TASKTYPE, MINION_TASK_SEGMENT_IMPORT); - public static final List TABLENAME_WITHTYPE_TASKTYPE = + public static final List TABLENAME_TABLETYPE_CONTROLLER_TASKTYPE = + List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME, TASKTYPE, + CONTROLLER_PERIODIC_TASK_CHC); + + public static final List TABLENAME_WITHTYPE_MINION_TASKTYPE = List.of(TABLE, TABLENAME_WITH_TYPE_REALTIME, TASKTYPE, MINION_TASK_SEGMENT_IMPORT); + public static final List TABLENAME_WITHTYPE_CONTROLLER_TASKTYPE = + List.of(TABLE, TABLENAME_WITH_TYPE_REALTIME, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC); + public static final List STATUS_TASKTYPE = - List.of(STATUS, IN_PROGRESS, TASKTYPE, MINION_TASK_SEGMENT_IMPORT); + List.of(STATUS, IN_PROGRESS, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC); public static final List PERIODIC_TASK_TABLE_TABLETYPE = - List.of(PERIODIC_TASK, MINION_TASK_SEGMENT_IMPORT, TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, + List.of(PERIODIC_TASK, CONTROLLER_PERIODIC_TASK_CHC, TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME); public static final List EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE = diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java index e6d40e369cb4..efe22424c782 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java @@ -77,24 +77,8 @@ public class ServerPrometheusMetricsTest extends PinotPrometheusMetricsTest { @BeforeClass public void setup() throws Exception { - _httpServer = startExporter(PinotComponent.SERVER); - - PinotConfiguration pinotConfiguration = new PinotConfiguration(); - pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, - "org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory"); - PinotMetricUtils.init(pinotConfiguration); - - // Initialize ServerMetrics with the registry - YammerMetricsRegistry yammerMetricsRegistry = new YammerMetricsRegistry(); - _serverMetrics = new ServerMetrics(yammerMetricsRegistry); - - // Enable JMX reporting - MetricsRegistry metricsRegistry = (MetricsRegistry) yammerMetricsRegistry.getMetricsRegistry(); - JmxReporter jmxReporter = new JmxReporter(metricsRegistry); - jmxReporter.start(); - - _httpClient = new HttpClient(); + _serverMetrics = new ServerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); } @Test(dataProvider = "serverTimers") From 0c460ece41257846a1bac23e6281a3a53e2a4e84 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Sun, 20 Oct 2024 22:43:22 +0530 Subject: [PATCH 33/45] Addressed comments --- .../metrics/BrokerPrometheusMetricsTest.java | 12 --------- .../ControllerPrometheusMetricsTest.java | 24 ++++++++++-------- .../metrics/MinionPrometheusMetricsTest.java | 12 ++------- .../metrics/PinotPrometheusMetricsTest.java | 25 ++++++------------- .../metrics/ServerPrometheusMetricsTest.java | 16 ++++-------- 5 files changed, 28 insertions(+), 61 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java index 6f2b1c5901f9..3b1f291384da 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java @@ -19,28 +19,16 @@ package org.apache.pinot.common.metrics; -import com.yammer.metrics.core.MetricsRegistry; -import com.yammer.metrics.reporting.JmxReporter; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; -import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.util.List; import java.util.concurrent.TimeUnit; -import javax.inject.Inject; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; -import org.apache.pinot.common.utils.http.HttpClient; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; -import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; -import org.apache.pinot.spi.env.PinotConfiguration; -import org.apache.pinot.spi.metrics.PinotMetricUtils; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; - public class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java index 86014f228795..c01f994781c7 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java @@ -38,22 +38,23 @@ public class ControllerPrometheusMetricsTest extends PinotPrometheusMetricsTest private static final String LABEL_KEY_TASK_TYPE = "taskType"; //that accept global gauge with suffix - List GLOBAL_GAUGES_ACCEPTING_TASKTYPE = + private static final List GLOBAL_GAUGES_ACCEPTING_TASKTYPE = List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, ControllerGauge.NUM_MINION_SUBTASKS_WAITING, ControllerGauge.NUM_MINION_SUBTASKS_ERROR, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); //local gauges that accept partition - List GAUGES_ACCEPTING_PARTITION = + private static final List GAUGES_ACCEPTING_PARTITION = List.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS); //these accept task type - List GAUGES_ACCEPTING_TASKTYPE = + private static final List GAUGES_ACCEPTING_TASKTYPE = List.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR); - List GAUGES_ACCEPTING_RAW_TABLENAME = List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); + private static final List GAUGES_ACCEPTING_RAW_TABLENAME = + List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); private ControllerMetrics _controllerMetrics; @@ -108,7 +109,8 @@ public void meterTest(ControllerMeter meter) { EXPORTED_METRIC_PREFIX); } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE + "." + ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); - assertMeterExportedCorrectly(meterName, ExportedLabels.PERIODIC_TASK_TABLE_TABLETYPE, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, ExportedLabels.CONTROLLER_TASKTYPE_TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); } else { addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE); addMeterWithLabels(meter, ExportedLabelValues.TABLENAME); @@ -142,13 +144,13 @@ public void gaugeTest(ControllerGauge controllerGauge) { if (GAUGES_ACCEPTING_PARTITION.contains(controllerGauge)) { _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, 10L); String strippedGaugeName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedGaugeName, ExportedLabels.PARTITION_TABLE_NAME_AND_TYPE, + assertGaugeExportedCorrectly(strippedGaugeName, ExportedLabels.PARTITION_TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } else if (GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) { _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, controllerGauge, () -> 50L); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE_CONTROLLER_TASKTYPE, - EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), + ExportedLabels.TABLENAME_TABLETYPE_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); } else if (GAUGES_ACCEPTING_RAW_TABLENAME.contains(controllerGauge)) { addGaugeWithLabels(controllerGauge, ExportedLabelValues.TABLENAME); assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); @@ -156,12 +158,12 @@ public void gaugeTest(ControllerGauge controllerGauge) { addGaugeWithLabels(controllerGauge, String.format("%s.%s", TABLE_NAME_WITH_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC)); assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(), - ExportedLabels.TABLENAME_WITHTYPE_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); + ExportedLabels.TABLENAMEWITHTYPE_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); } else if (controllerGauge == ControllerGauge.TASK_STATUS) { addGaugeWithLabels(controllerGauge, String.format("%s.%s", ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, TaskState.IN_PROGRESS)); - assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), ExportedLabels.STATUS_TASKTYPE, - EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), + ExportedLabels.JOBSTATUS_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); } else { addGaugeWithLabels(controllerGauge, TABLE_NAME_WITH_TYPE); assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java index 8b96764e2387..3feeb12d3901 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java @@ -19,24 +19,16 @@ package org.apache.pinot.common.metrics; -import com.yammer.metrics.core.MetricsRegistry; -import com.yammer.metrics.reporting.JmxReporter; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.pinot.common.utils.SimpleHttpResponse; -import org.apache.pinot.common.utils.http.HttpClient; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; -import org.apache.pinot.spi.env.PinotConfiguration; -import org.apache.pinot.spi.metrics.PinotMetricUtils; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; - public class MinionPrometheusMetricsTest extends PinotPrometheusMetricsTest { //all exported minion metrics have this prefix @@ -66,7 +58,7 @@ public void timerTest(MinionTimer timer) { List.of(ExportedLabelKeys.DATABASE, ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME, ExportedLabelKeys.TABLE, "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); } else { - assertTimerExportedCorrectly(timer.getTimerName(), ExportedLabels.EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, + assertTimerExportedCorrectly(timer.getTimerName(), ExportedLabels.TABLENAME_TABLETYPE_MINION_TASKTYPE, EXPORTED_METRIC_PREFIX); } } @@ -100,7 +92,7 @@ private void validateMetersWithLabels(MinionMeter meter) { _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, meter, 1L); assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE, + assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE_MINION_TASKTYPE, EXPORTED_METRIC_PREFIX); } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java index b4e324886231..87df6109af88 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java @@ -410,40 +410,31 @@ public enum PinotComponent { } public static class ExportedLabels { - public static final List TABLENAME_TABLETYPE = - List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME); public static final List TABLENAME = List.of(TABLE, ExportedLabelValues.TABLENAME); - - public static final List CLIENT_ID = + public static final List TABLENAME_TABLETYPE = + List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME); + public static final List PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC = List.of(PARTITION, PARTITION_GROUP_ID, TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TableType.REALTIME.toString(), TOPIC, KAFKA_TOPIC); - - public static final List PARTITION_TABLE_NAME_AND_TYPE = + public static final List PARTITION_TABLENAME_TABLETYPE = List.of(PARTITION, "3", TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TableType.REALTIME.toString()); - public static final List TABLENAME_TABLETYPE_MINION_TASKTYPE = - List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME, TASKTYPE, - MINION_TASK_SEGMENT_IMPORT); - public static final List TABLENAME_TABLETYPE_CONTROLLER_TASKTYPE = List.of(TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC); - public static final List TABLENAME_WITHTYPE_MINION_TASKTYPE = - List.of(TABLE, TABLENAME_WITH_TYPE_REALTIME, TASKTYPE, MINION_TASK_SEGMENT_IMPORT); - - public static final List TABLENAME_WITHTYPE_CONTROLLER_TASKTYPE = + public static final List TABLENAMEWITHTYPE_CONTROLLER_TASKTYPE = List.of(TABLE, TABLENAME_WITH_TYPE_REALTIME, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC); - public static final List STATUS_TASKTYPE = + public static final List JOBSTATUS_CONTROLLER_TASKTYPE = List.of(STATUS, IN_PROGRESS, TASKTYPE, CONTROLLER_PERIODIC_TASK_CHC); - public static final List PERIODIC_TASK_TABLE_TABLETYPE = + public static final List CONTROLLER_TASKTYPE_TABLENAME_TABLETYPE = List.of(PERIODIC_TASK, CONTROLLER_PERIODIC_TASK_CHC, TABLE, ExportedLabelValues.TABLENAME, TABLETYPE, TABLETYPE_REALTIME); - public static final List EXPORTED_LABELS_TABLENAME_TYPE_TASKTYPE = + public static final List TABLENAME_TABLETYPE_MINION_TASKTYPE = List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.TABLENAME, ExportedLabelKeys.TABLETYPE, ExportedLabelValues.TABLETYPE_REALTIME, ExportedLabelKeys.TASKTYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java index efe22424c782..2d885cfee935 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java @@ -19,24 +19,16 @@ package org.apache.pinot.common.metrics; -import com.yammer.metrics.core.MetricsRegistry; -import com.yammer.metrics.reporting.JmxReporter; import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; import java.net.URI; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.pinot.common.utils.SimpleHttpResponse; -import org.apache.pinot.common.utils.http.HttpClient; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsRegistry; -import org.apache.pinot.spi.env.PinotConfiguration; -import org.apache.pinot.spi.metrics.PinotMetricUtils; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; - public class ServerPrometheusMetricsTest extends PinotPrometheusMetricsTest { //all exported server metrics have this prefix @@ -110,7 +102,8 @@ public void meterTest(ServerMeter serverMeter) { } else { if (METERS_ACCEPTING_CLIENT_ID.contains(serverMeter)) { addMeterWithLabels(serverMeter, CLIENT_ID); - assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.CLIENT_ID); + assertMeterExportedCorrectly(serverMeter.getMeterName(), + ExportedLabels.PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC); } else if (METERS_ACCEPTING_RAW_TABLE_NAMES.contains(serverMeter)) { addMeterWithLabels(serverMeter, ExportedLabelValues.TABLENAME); assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.TABLENAME); @@ -141,10 +134,11 @@ public void gaugeTest(ServerGauge serverGauge) { EXPORTED_METRIC_PREFIX); } else if (GAUGES_ACCEPTING_CLIENT_ID.contains(serverGauge)) { addGaugeWithLabels(serverGauge, CLIENT_ID); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.CLIENT_ID, EXPORTED_METRIC_PREFIX); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), + ExportedLabels.PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC, EXPORTED_METRIC_PREFIX); } else if (GAUGES_ACCEPTING_PARTITION.contains(serverGauge)) { addPartitionGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.PARTITION_TABLE_NAME_AND_TYPE, + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.PARTITION_TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } else if (GAUGES_ACCEPTING_RAW_TABLE_NAME.contains(serverGauge)) { addGaugeWithLabels(serverGauge, ExportedLabelValues.TABLENAME); From a62a113ec9355245b8803f86add3142038451b29 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Mon, 21 Oct 2024 12:14:56 +0530 Subject: [PATCH 34/45] Rearrange code --- .../metrics/BrokerPrometheusMetricsTest.java | 5 + .../ControllerPrometheusMetricsTest.java | 16 +- .../metrics/MinionPrometheusMetricsTest.java | 18 +- .../metrics/PinotPrometheusMetricsTest.java | 253 ++++++++++-------- .../metrics/ServerPrometheusMetricsTest.java | 46 ++-- 5 files changed, 186 insertions(+), 152 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java index 3b1f291384da..1bfd290bb5c7 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java @@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -133,4 +134,8 @@ public Object[] brokerMeters() { public Object[] brokerGauges() { return BrokerGauge.values(); } + @AfterClass + public void cleanup() { + _httpServer.close(); + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java index c01f994781c7..3e5251da6774 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java @@ -172,14 +172,6 @@ public void gaugeTest(ControllerGauge controllerGauge) { } } - private void addGaugeWithLabels(ControllerGauge gauge, String labels) { - _controllerMetrics.setValueOfTableGauge(labels, gauge, 5L); - } - - private static String getStrippedMetricName(ControllerGauge controllerGauge) { - return StringUtils.remove(controllerGauge.getGaugeName(), "controller"); - } - @Override protected SimpleHttpResponse getExportedPromMetrics() { try { @@ -209,6 +201,14 @@ public void cleanup() { _httpServer.close(); } + private void addGaugeWithLabels(ControllerGauge gauge, String labels) { + _controllerMetrics.setValueOfTableGauge(labels, gauge, 5L); + } + + private static String getStrippedMetricName(ControllerGauge controllerGauge) { + return StringUtils.remove(controllerGauge.getGaugeName(), "controller"); + } + private void addMeterWithLabels(ControllerMeter meter, String labels) { _controllerMetrics.addMeteredTableValue(labels, meter, 1L); } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java index 3feeb12d3901..b9539b1f0f85 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java @@ -108,15 +108,6 @@ public void gaugeTest(MinionGauge gauge) { } } - @Override - protected SimpleHttpResponse getExportedPromMetrics() { - try { - return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - @DataProvider(name = "minionTimers") public Object[] minionTimers() { return MinionTimer.values(); @@ -137,6 +128,15 @@ public void cleanup() { _httpServer.close(); } + @Override + protected SimpleHttpResponse getExportedPromMetrics() { + try { + return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private void validateGlobalMeters(MinionMeter meter) { _minionMetrics.addMeteredGlobalValue(meter, 5L); assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java index 87df6109af88..6a5a54f81124 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java @@ -63,6 +63,17 @@ public abstract class PinotPrometheusMetricsTest { + protected static final String TABLE_NAME_WITH_TYPE = + TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(ExportedLabelValues.TABLENAME); + protected static final String KAFKA_TOPIC = "myTopic"; + protected static final String PARTITION_GROUP_ID = "partitionGroupId"; + protected static final String CLIENT_ID = + String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC, PARTITION_GROUP_ID); + + protected PinotMetricsFactory _pinotMetricsFactory; + + protected HttpClient _httpClient; + //config keys private static final String CONFIG_KEY_JMX_EXPORTER_PARENT_DIR = "jmxExporterConfigsParentDir"; private static final String CONFIG_KEY_PINOT_METRICS_FACTORY = "pinotMetricsFactory"; @@ -84,17 +95,6 @@ public abstract class PinotPrometheusMetricsTest { //each gauge defined in code is exported with these measurements private static final List GAUGE_TYPES = List.of("Value"); - protected static final String TABLE_NAME_WITH_TYPE = - TableNameBuilder.forType(TableType.REALTIME).tableNameWithType(ExportedLabelValues.TABLENAME); - protected static final String KAFKA_TOPIC = "myTopic"; - protected static final String PARTITION_GROUP_ID = "partitionGroupId"; - protected static final String CLIENT_ID = - String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC, PARTITION_GROUP_ID); - - protected PinotMetricsFactory _pinotMetricsFactory; - - protected HttpClient _httpClient; - private String _exporterConfigsParentDir; private final Map _pinotComponentToConfigFileMap = new HashMap<>(); @@ -135,16 +135,10 @@ public void setupTest() _httpClient = new HttpClient(); } - private String loadResourceAsString(String resourceFileName) { - URL url = Resources.getResource(resourceFileName); - try { - return Resources.toString(url, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** + /** Pinot currently uses the JMX->Prom exporter to export metrics to Prometheus. Normally, this runs as an agent in + * the JVM. In this case however, we've got tests using four different config files (server.yml, broker.yml, + * controller.yml and minion.yml). Loading the same agent in the same JVM multiple times isn't allowed, we are + * copying the agent's code to some degree and starting up the HTTP servers manually. * For impl, see: * ... * /jmx_prometheus_javaagent/src/main/java/io/prometheus/jmx/JavaAgent.java#L48 @@ -272,98 +266,12 @@ protected List parseExportedPromMetrics(String response) } } + /** + * Reads the exported prometheus metrics from the JMX -> Prometheus server + * @return the corresponding GET response + */ protected abstract SimpleHttpResponse getExportedPromMetrics(); - public static class PromMetric { - private final String _metricName; - private final Map _labels; - - public String getMetricName() { - return _metricName; - } - - public Map getLabels() { - return _labels; - } - - private PromMetric(String metricName, Map labels) { - _metricName = metricName; - _labels = labels; - } - - public static PromMetric fromExportedMetric(String exportedMetric) { - int spaceIndex = exportedMetric.indexOf(' '); - String metricWithoutVal = exportedMetric.substring(0, spaceIndex); - int braceIndex = metricWithoutVal.indexOf('{'); - - if (braceIndex != -1) { - String metricName = metricWithoutVal.substring(0, braceIndex); - String labelsString = metricWithoutVal.substring(braceIndex + 1, metricWithoutVal.lastIndexOf('}')); - Map labels = parseLabels(labelsString); - return new PromMetric(metricName, labels); - } else { - return new PromMetric(metricWithoutVal, new LinkedHashMap<>()); - } - } - - public static PromMetric withName(String metricName) { - return new PromMetric(metricName, new LinkedHashMap<>()); - } - - public static PromMetric withNameAndLabels(String metricName, List labels) { - Map labelMap = new LinkedHashMap<>(); - for (int i = 0; i < labels.size(); i += 2) { - labelMap.put(labels.get(i), labels.get(i + 1)); - } - return new PromMetric(metricName, labelMap); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - PromMetric that = (PromMetric) o; - return metricNamesAreSimilar(that) && Objects.equal(_labels, that._labels); - } - - @Override - public int hashCode() { - return Objects.hashCode(_metricName, _labels); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(_metricName); - if (!_labels.isEmpty()) { - sb.append('{'); - sb.append(_labels.entrySet().stream().map(e -> e.getKey() + "=\"" + e.getValue() + "\"") - .collect(Collectors.joining(","))); - sb.append('}'); - } - return sb.toString(); - } - - private boolean metricNamesAreSimilar(PromMetric that) { - String processedMetricNameThis = StringUtils.remove(_metricName, "_"); - String processedMetricNameThat = StringUtils.remove(that._metricName, "_"); - return StringUtils.equalsIgnoreCase(processedMetricNameThis, processedMetricNameThat); - } - - private static Map parseLabels(String labelsString) { - return labelsString.isEmpty() ? new LinkedHashMap<>() - : java.util.Arrays.stream(labelsString.split(",")).map(kvPair -> kvPair.split("=")) - .collect(Collectors.toMap(kv -> kv[0], kv -> removeQuotes(kv[1]), (v1, v2) -> v2, LinkedHashMap::new)); - } - - private static String removeQuotes(String value) { - return value.startsWith("\"") ? value.substring(1, value.length() - 1) : value; - } - } - /* Implementation copied from: https://github .com/prometheus/jmx_exporter/blob/a3b9443564ff5a78c25fd6566396fda2b7cbf216/jmx_prometheus_javaagent/src/main/java @@ -391,7 +299,7 @@ private static JMXExporterConfig parseExporterConfig(String args, String ifc) { } } - static class JMXExporterConfig { + private static class JMXExporterConfig { String _host; int _port; String _file; @@ -405,6 +313,15 @@ static class JMXExporterConfig { } } + private String loadResourceAsString(String resourceFileName) { + URL url = Resources.getResource(resourceFileName); + try { + return Resources.toString(url, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public enum PinotComponent { SERVER, BROKER, CONTROLLER, MINION } @@ -461,4 +378,116 @@ public static class ExportedLabelValues { public static final String MINION_TASK_SEGMENT_IMPORT = "SegmentImportTask"; public static final String IN_PROGRESS = "IN_PROGRESS"; } + + /* + * Represents an exported Prometheus metric. A Prom metric looks like: + * pinot_server_realtimeRowsSanitized_Count{app="pinot", cluster_name="pinot", + * component="pinot-server", component_name="server-default-tenant-1"} + */ + private static class PromMetric { + private final String _metricName; + private final Map _labels; + + public String getMetricName() { + return _metricName; + } + + public Map getLabels() { + return _labels; + } + + //make constructor private so that it can be instantiated only with the factory methods + private PromMetric(String metricName, Map labels) { + _metricName = metricName; + _labels = labels; + } + + /** + * Create an instance of {@link PromMetric} from an exported Prometheus metric + * @param exportedMetric the exported Prom metric (name + labels) + * @return the corresponding {@link PromMetric} + */ + public static PromMetric fromExportedMetric(String exportedMetric) { + int spaceIndex = exportedMetric.indexOf(' '); + String metricWithoutVal = exportedMetric.substring(0, spaceIndex); + int braceIndex = metricWithoutVal.indexOf('{'); + + if (braceIndex != -1) { + String metricName = metricWithoutVal.substring(0, braceIndex); + String labelsString = metricWithoutVal.substring(braceIndex + 1, metricWithoutVal.lastIndexOf('}')); + Map labels = parseLabels(labelsString); + return new PromMetric(metricName, labels); + } else { + return new PromMetric(metricWithoutVal, new LinkedHashMap<>()); + } + } + + /** + * Creates an instance of {@link PromMetric} with a name and an empty label list + * @param metricName the metric name + * @return the corresponding PromMetric + */ + public static PromMetric withName(String metricName) { + return new PromMetric(metricName, new LinkedHashMap<>()); + } + + /** + * Creates an instance of {@link PromMetric} with a name and an label list + * @param metricName the metric name + * @param labels the labels + * @return the corresponding PromMetric + */ + public static PromMetric withNameAndLabels(String metricName, List labels) { + Map labelMap = new LinkedHashMap<>(); + for (int i = 0; i < labels.size(); i += 2) { + labelMap.put(labels.get(i), labels.get(i + 1)); + } + return new PromMetric(metricName, labelMap); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PromMetric that = (PromMetric) o; + return metricNamesAreSimilar(that) && Objects.equal(_labels, that._labels); + } + + @Override + public int hashCode() { + return Objects.hashCode(_metricName, _labels); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(_metricName); + if (!_labels.isEmpty()) { + sb.append('{'); + sb.append(_labels.entrySet().stream().map(e -> e.getKey() + "=\"" + e.getValue() + "\"") + .collect(Collectors.joining(","))); + sb.append('}'); + } + return sb.toString(); + } + + private boolean metricNamesAreSimilar(PromMetric that) { + String processedMetricNameThis = StringUtils.remove(_metricName, "_"); + String processedMetricNameThat = StringUtils.remove(that._metricName, "_"); + return StringUtils.equalsIgnoreCase(processedMetricNameThis, processedMetricNameThat); + } + + private static Map parseLabels(String labelsString) { + return labelsString.isEmpty() ? new LinkedHashMap<>() + : java.util.Arrays.stream(labelsString.split(",")).map(kvPair -> kvPair.split("=")) + .collect(Collectors.toMap(kv -> kv[0], kv -> removeQuotes(kv[1]), (v1, v2) -> v2, LinkedHashMap::new)); + } + + private static String removeQuotes(String value) { + return value.startsWith("\"") ? value.substring(1, value.length() - 1) : value; + } + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java index 2d885cfee935..fb8a89869661 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java @@ -163,23 +163,24 @@ public void addMeterWithLabels(ServerMeter serverMeter, String labels) { _serverMetrics.addMeteredTableValue(labels, serverMeter, 4L); } - private boolean meterTrackingRealtimeExceptions(ServerMeter serverMeter) { - return serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS - || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS - || serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS || serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS; + @DataProvider(name = "serverTimers") + public Object[] serverTimers() { + return ServerTimer.values(); // Provide all values of ServerTimer enum } - private String getRealtimeExceptionMeterName(ServerMeter serverMeter) { - String meterName = serverMeter.getMeterName(); - return "realtime_exceptions_" + meterName.substring(0, meterName.lastIndexOf("Exceptions")); + @DataProvider(name = "serverMeters") + public Object[] serverMeter() { + return ServerMeter.values(); // Provide all values of ServerTimer enum } - private void assertMeterExportedCorrectly(String exportedMeterName) { - assertMeterExportedCorrectly(exportedMeterName, EXPORTED_METRIC_PREFIX); + @DataProvider(name = "serverGauges") + public Object[] serverGauge() { + return ServerGauge.values(); // Provide all values of ServerTimer enum } - private void assertMeterExportedCorrectly(String exportedMeterName, List labels) { - assertMeterExportedCorrectly(exportedMeterName, labels, EXPORTED_METRIC_PREFIX); + @AfterClass + public void cleanup() { + _httpServer.close(); } @Override @@ -191,23 +192,22 @@ protected SimpleHttpResponse getExportedPromMetrics() { } } - @DataProvider(name = "serverTimers") - public Object[] serverTimers() { - return ServerTimer.values(); // Provide all values of ServerTimer enum + private boolean meterTrackingRealtimeExceptions(ServerMeter serverMeter) { + return serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS + || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS + || serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS || serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS; } - @DataProvider(name = "serverMeters") - public Object[] serverMeter() { - return ServerMeter.values(); // Provide all values of ServerTimer enum + private String getRealtimeExceptionMeterName(ServerMeter serverMeter) { + String meterName = serverMeter.getMeterName(); + return "realtime_exceptions_" + meterName.substring(0, meterName.lastIndexOf("Exceptions")); } - @DataProvider(name = "serverGauges") - public Object[] serverGauge() { - return ServerGauge.values(); // Provide all values of ServerTimer enum + private void assertMeterExportedCorrectly(String exportedMeterName) { + assertMeterExportedCorrectly(exportedMeterName, EXPORTED_METRIC_PREFIX); } - @AfterClass - public void cleanup() { - _httpServer.close(); + private void assertMeterExportedCorrectly(String exportedMeterName, List labels) { + assertMeterExportedCorrectly(exportedMeterName, labels, EXPORTED_METRIC_PREFIX); } } From 2daac0bb36fd42c08ec3a690a294760f4f37631a Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 12:53:09 +0530 Subject: [PATCH 35/45] Separate tests for Yammer and Dropwizard metrics factory --- .../metrics/BrokerPrometheusMetricsTest.java | 90 +--------- .../ControllerPrometheusMetricsTest.java | 159 +----------------- .../metrics/MinionPrometheusMetricsTest.java | 101 +---------- .../metrics/PinotPrometheusMetricsTest.java | 51 +++--- .../metrics/ServerPrometheusMetricsTest.java | 150 +---------------- .../YammerBrokerPrometheusMetricsTest.java | 80 +++++++++ ...YammerControllerPrometheusMetricsTest.java | 144 ++++++++++++++++ .../YammerMinionPrometheusMetricsTest.java | 94 +++++++++++ .../YammerServerPrometheusMetricsTest.java | 134 +++++++++++++++ 9 files changed, 511 insertions(+), 492 deletions(-) create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerBrokerPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerControllerPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerMinionPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerServerPrometheusMetricsTest.java diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java index 1bfd290bb5c7..cefa4f754d05 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java @@ -19,30 +19,22 @@ package org.apache.pinot.common.metrics; -import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; -import java.net.URI; import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.StringUtils; -import org.apache.pinot.common.utils.SimpleHttpResponse; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; -public class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { +public abstract class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { - private static final String EXPORTED_METRIC_PREFIX = "pinot_broker_"; + protected static final String EXPORTED_METRIC_PREFIX = "pinot_broker_"; - private static final String EXPORTED_METRIC_PREFIX_EXCEPTIONS = "exceptions"; + protected static final String EXPORTED_METRIC_PREFIX_EXCEPTIONS = "exceptions"; - private static final List GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX = + protected static final List GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX = List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, BrokerMeter.QUERY_REJECTED_EXCEPTIONS, BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, BrokerMeter.RESOURCE_MISSING_EXCEPTIONS); - private static final List METERS_ACCEPTING_RAW_TABLENAME = + protected static final List METERS_ACCEPTING_RAW_TABLENAME = List.of(BrokerMeter.QUERIES, BrokerMeter.NO_SERVER_FOUND_EXCEPTIONS, BrokerMeter.DOCUMENTS_SCANNED, BrokerMeter.ENTRIES_SCANNED_IN_FILTER, BrokerMeter.BROKER_RESPONSES_WITH_UNAVAILABLE_SEGMENTS, BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, @@ -51,73 +43,9 @@ public class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { BrokerMeter.ENTRIES_SCANNED_POST_FILTER, BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE, BrokerMeter.QUERY_QUOTA_EXCEEDED); - private BrokerMetrics _brokerMetrics; - - private HTTPServer _httpServer; - - @BeforeClass - public void setup() - throws Exception { - _httpServer = startExporter(PinotComponent.BROKER); - _brokerMetrics = new BrokerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); - } - - @Test(dataProvider = "brokerTimers") - public void timerTest(BrokerTimer timer) { - if (timer.isGlobal()) { - _brokerMetrics.addTimedValue(timer, 30_000, TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_METRIC_PREFIX); - } else { - _brokerMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, timer, 30_000L, TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(timer.getTimerName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - } - } - - @Test(dataProvider = "brokerGauges") - public void gaugeTest(BrokerGauge gauge) { - if (gauge.isGlobal()) { - _brokerMetrics.setOrUpdateGlobalGauge(gauge, () -> 5L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); - } else { - if (gauge == BrokerGauge.REQUEST_SIZE) { - _brokerMetrics.setOrUpdateTableGauge(ExportedLabelValues.TABLENAME, gauge, 5L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - } else { - _brokerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); - } - } - } - - @Test(dataProvider = "brokerMeters") - public void meterTest(BrokerMeter meter) { - if (meter.isGlobal()) { - _brokerMetrics.addMeteredGlobalValue(meter, 5L); - if (GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX.contains(meter)) { - String exportedMeterPrefix = String.format("%s_%s", EXPORTED_METRIC_PREFIX_EXCEPTIONS, - StringUtils.remove(meter.getMeterName(), "Exceptions")); - assertMeterExportedCorrectly(exportedMeterPrefix, EXPORTED_METRIC_PREFIX); - } else { - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); - } - } else { - if (METERS_ACCEPTING_RAW_TABLENAME.contains(meter)) { - _brokerMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 5L); - assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - } else { - _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); - assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); - } - } - } - @Override - protected SimpleHttpResponse getExportedPromMetrics() { - try { - return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); - } catch (Exception e) { - throw new RuntimeException(e); - } + protected PinotComponent getPinotComponent() { + return PinotComponent.BROKER; } @DataProvider(name = "brokerTimers") @@ -134,8 +62,4 @@ public Object[] brokerMeters() { public Object[] brokerGauges() { return BrokerGauge.values(); } - @AfterClass - public void cleanup() { - _httpServer.close(); - } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java index 3e5251da6774..c1657535bb79 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java @@ -19,166 +19,42 @@ package org.apache.pinot.common.metrics; -import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; -import java.net.URI; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.helix.task.TaskState; -import org.apache.pinot.common.utils.SimpleHttpResponse; -import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class ControllerPrometheusMetricsTest extends PinotPrometheusMetricsTest { +public abstract class ControllerPrometheusMetricsTest extends PinotPrometheusMetricsTest { //all exported controller metrics have this prefix - private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; - private static final String LABEL_KEY_TASK_TYPE = "taskType"; + protected static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; + protected static final String LABEL_KEY_TASK_TYPE = "taskType"; //that accept global gauge with suffix - private static final List GLOBAL_GAUGES_ACCEPTING_TASKTYPE = + protected static final List GLOBAL_GAUGES_ACCEPTING_TASKTYPE = List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, ControllerGauge.NUM_MINION_SUBTASKS_WAITING, ControllerGauge.NUM_MINION_SUBTASKS_ERROR, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); //local gauges that accept partition - private static final List GAUGES_ACCEPTING_PARTITION = + protected static final List GAUGES_ACCEPTING_PARTITION = List.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS); //these accept task type - private static final List GAUGES_ACCEPTING_TASKTYPE = + protected static final List GAUGES_ACCEPTING_TASKTYPE = List.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR); - private static final List GAUGES_ACCEPTING_RAW_TABLENAME = + protected static final List GAUGES_ACCEPTING_RAW_TABLENAME = List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); - private ControllerMetrics _controllerMetrics; - - private HTTPServer _httpServer; - - @BeforeClass - public void setup() - throws Exception { - _httpServer = startExporter(PinotComponent.CONTROLLER); - _controllerMetrics = new ControllerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); - } - - @Test(dataProvider = "controllerTimers") - public void timerTest(ControllerTimer controllerTimer) { - if (controllerTimer.isGlobal()) { - _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX); - } else { - _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); - _controllerMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, controllerTimer, 30_000L, - TimeUnit.MILLISECONDS); - - assertTimerExportedCorrectly(controllerTimer.getTimerName(), ExportedLabels.TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly(controllerTimer.getTimerName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - } - } - - @Test(dataProvider = "controllerMeters") - public void meterTest(ControllerMeter meter) { - if (meter.isGlobal()) { - _controllerMetrics.addMeteredGlobalValue(meter, 5L); - String meterName = meter.getMeterName(); - //some meters contain a "controller" prefix. For example, controllerInstancePostError. These meters are - // exported as 'pinot_controller_pinot_controller_InstancePostError'. So we strip the 'controller' from - // 'controllerInstancePostError' - String strippedMeterName = StringUtils.remove(meterName, "controller"); - assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX); - } else { - - String meterName = meter.getMeterName(); - String strippedMeterName = StringUtils.remove(meterName, "controller"); - - if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { - addMeterWithLabels(meter, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); - assertMeterExportedCorrectly(meterName, - List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), EXPORTED_METRIC_PREFIX); - } else if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { - addMeterWithLabels(meter, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); - assertMeterExportedCorrectly( - String.format("%s_%s", strippedMeterName, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), - EXPORTED_METRIC_PREFIX); - } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { - addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE + "." + ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); - assertMeterExportedCorrectly(meterName, ExportedLabels.CONTROLLER_TASKTYPE_TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); - } else { - addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE); - addMeterWithLabels(meter, ExportedLabelValues.TABLENAME); - if (meter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { - assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - } else { - assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - } - } - } - } - - @Test(dataProvider = "controllerGauges") - public void gaugeTest(ControllerGauge controllerGauge) { - if (controllerGauge.isGlobal()) { - _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, 1L); - //some global gauges also accept the taskType (which should not be). todo: this should be fixed - if (GLOBAL_GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) { - _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, 1L); - String strippedMetricName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedMetricName, - List.of(LABEL_KEY_TASK_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), EXPORTED_METRIC_PREFIX); - } else { - _controllerMetrics.setValueOfGlobalGauge(controllerGauge, 1L); - String strippedMetricName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedMetricName, EXPORTED_METRIC_PREFIX); - } - } else { - if (GAUGES_ACCEPTING_PARTITION.contains(controllerGauge)) { - _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, 10L); - String strippedGaugeName = getStrippedMetricName(controllerGauge); - assertGaugeExportedCorrectly(strippedGaugeName, ExportedLabels.PARTITION_TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); - } else if (GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) { - _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, - controllerGauge, () -> 50L); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), - ExportedLabels.TABLENAME_TABLETYPE_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); - } else if (GAUGES_ACCEPTING_RAW_TABLENAME.contains(controllerGauge)) { - addGaugeWithLabels(controllerGauge, ExportedLabelValues.TABLENAME); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - } else if (controllerGauge == ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED) { - addGaugeWithLabels(controllerGauge, - String.format("%s.%s", TABLE_NAME_WITH_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC)); - assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(), - ExportedLabels.TABLENAMEWITHTYPE_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); - } else if (controllerGauge == ControllerGauge.TASK_STATUS) { - addGaugeWithLabels(controllerGauge, - String.format("%s.%s", ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, TaskState.IN_PROGRESS)); - assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), - ExportedLabels.JOBSTATUS_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); - } else { - addGaugeWithLabels(controllerGauge, TABLE_NAME_WITH_TYPE); - assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); - } - } - } - @Override - protected SimpleHttpResponse getExportedPromMetrics() { - try { - return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); - } catch (Exception e) { - throw new RuntimeException(e); - } + protected PinotComponent getPinotComponent() { + return PinotComponent.CONTROLLER; } @DataProvider(name = "controllerTimers") @@ -195,21 +71,4 @@ public Object[] controllerMeters() { public Object[] controllerGauges() { return ControllerGauge.values(); } - - @AfterClass - public void cleanup() { - _httpServer.close(); - } - - private void addGaugeWithLabels(ControllerGauge gauge, String labels) { - _controllerMetrics.setValueOfTableGauge(labels, gauge, 5L); - } - - private static String getStrippedMetricName(ControllerGauge controllerGauge) { - return StringUtils.remove(controllerGauge.getGaugeName(), "controller"); - } - - private void addMeterWithLabels(ControllerMeter meter, String labels) { - _controllerMetrics.addMeteredTableValue(labels, meter, 1L); - } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java index b9539b1f0f85..998cae8e12ad 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java @@ -19,94 +19,17 @@ package org.apache.pinot.common.metrics; -import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; -import java.net.URI; import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.pinot.common.utils.SimpleHttpResponse; -import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class MinionPrometheusMetricsTest extends PinotPrometheusMetricsTest { +public abstract class MinionPrometheusMetricsTest extends PinotPrometheusMetricsTest { //all exported minion metrics have this prefix - private static final String EXPORTED_METRIC_PREFIX = "pinot_minion_"; - private static final String METER_PREFIX_NO_TASKS = "numberTasks"; - private MinionMetrics _minionMetrics; - private HTTPServer _httpServer; - - @BeforeClass - public void setup() { - _httpServer = startExporter(PinotComponent.MINION); - _minionMetrics = new MinionMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); - } - - @Test(dataProvider = "minionTimers") - public void timerTest(MinionTimer timer) { - - _minionMetrics.addTimedValue(ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, timer, 30L, TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(timer.getTimerName(), - List.of(ExportedLabelKeys.ID, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); - - _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, timer, 30L, - TimeUnit.MILLISECONDS); - - if (timer == MinionTimer.TASK_THREAD_CPU_TIME_NS) { - assertTimerExportedCorrectly(timer.getTimerName(), - List.of(ExportedLabelKeys.DATABASE, ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME, ExportedLabelKeys.TABLE, - "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); - } else { - assertTimerExportedCorrectly(timer.getTimerName(), ExportedLabels.TABLENAME_TABLETYPE_MINION_TASKTYPE, - EXPORTED_METRIC_PREFIX); - } - } - - @Test(dataProvider = "minionMeters") - public void meterTest(MinionMeter meter) { - if (meter.isGlobal()) { - validateGlobalMeters(meter); - } else { - validateMetersWithLabels(meter); - } - } - - private void validateMetersWithLabels(MinionMeter meter) { - if (meter.getMeterName().startsWith(METER_PREFIX_NO_TASKS)) { - _minionMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 1L); - assertMeterExportedCorrectly(meter.getMeterName(), List.of(ExportedLabelKeys.ID, ExportedLabelValues.TABLENAME), - EXPORTED_METRIC_PREFIX); - - _minionMetrics.addMeteredValue(ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, meter, 1L); - assertMeterExportedCorrectly(meter.getMeterName(), - List.of(ExportedLabelKeys.ID, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); - } else if (meter == MinionMeter.SEGMENT_UPLOAD_FAIL_COUNT || meter == MinionMeter.SEGMENT_DOWNLOAD_FAIL_COUNT) { - - _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 1L); - assertMeterExportedCorrectly(meter.getMeterName(), List.of(ExportedLabelKeys.ID, TABLE_NAME_WITH_TYPE), - EXPORTED_METRIC_PREFIX); - } else { - //all remaining meters are also being used as global meters, check their usage - _minionMetrics.addMeteredGlobalValue(meter, 1L); - _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, meter, - 1L); - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); - assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE_MINION_TASKTYPE, - EXPORTED_METRIC_PREFIX); - } - } - - @Test(dataProvider = "minionGauges") - public void gaugeTest(MinionGauge gauge) { - if (gauge.isGlobal()) { - _minionMetrics.setValueOfGlobalGauge(gauge, 1L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); - } else { - _minionMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 1L); - assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); - } - } + protected static final String EXPORTED_METRIC_PREFIX = "pinot_minion_"; + protected static final String METER_PREFIX_NO_TASKS = "numberTasks"; @DataProvider(name = "minionTimers") public Object[] minionTimers() { @@ -123,22 +46,8 @@ public Object[] minionGauges() { return MinionGauge.values(); } - @AfterClass - public void cleanup() { - _httpServer.close(); - } - @Override - protected SimpleHttpResponse getExportedPromMetrics() { - try { - return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private void validateGlobalMeters(MinionMeter meter) { - _minionMetrics.addMeteredGlobalValue(meter, 5L); - assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); + protected PinotComponent getPinotComponent() { + return PinotComponent.MINION; } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java index 6a5a54f81124..5fb7f79298c4 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.io.StringReader; import java.net.InetSocketAddress; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -54,10 +55,14 @@ import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelKeys.*; -import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.*; +import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC; +import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.IN_PROGRESS; +import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME; +import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.TABLETYPE_REALTIME; import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; @@ -70,8 +75,6 @@ public abstract class PinotPrometheusMetricsTest { protected static final String CLIENT_ID = String.format("%s-%s-%s", TABLE_NAME_WITH_TYPE, KAFKA_TOPIC, PARTITION_GROUP_ID); - protected PinotMetricsFactory _pinotMetricsFactory; - protected HttpClient _httpClient; //config keys @@ -99,6 +102,8 @@ public abstract class PinotPrometheusMetricsTest { private final Map _pinotComponentToConfigFileMap = new HashMap<>(); + private HTTPServer _httpServer; + @BeforeClass public void setupTest() throws Exception { @@ -114,25 +119,21 @@ public void setupTest() _pinotComponentToConfigFileMap.put(PinotComponent.MINION, jsonNode.get(CONFIG_KEY_MINION_CONFIG_FILE_NAME).textValue()); - String pinotMetricsFactory = jsonNode.get(CONFIG_KEY_PINOT_METRICS_FACTORY).textValue(); - switch (pinotMetricsFactory) { - case "YammerMetricsFactory": - _pinotMetricsFactory = new YammerMetricsFactory(); - break; - case "DropwizardMetricsFactory": - _pinotMetricsFactory = new DropwizardMetricsFactory(); - break; - default: - throw new IllegalArgumentException("Unknow metrics factory specified in test config: " + pinotMetricsFactory - + ", supported ones are: YammerMetricsFactory and DropwizardMetricsFactory"); - } PinotConfiguration pinotConfiguration = new PinotConfiguration(); + + PinotMetricsFactory pinotMetricsFactory = getPinotMetricsFactory(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, - _pinotMetricsFactory.getClass().getCanonicalName()); + pinotMetricsFactory.getClass().getCanonicalName()); PinotMetricUtils.init(pinotConfiguration); - _pinotMetricsFactory.makePinotJmxReporter(_pinotMetricsFactory.getPinotMetricsRegistry()).start(); + pinotMetricsFactory.makePinotJmxReporter(pinotMetricsFactory.getPinotMetricsRegistry()).start(); _httpClient = new HttpClient(); + _httpServer = startExporter(getPinotComponent()); + } + + @AfterClass + public void cleanup() { + _httpServer.close(); } /** Pinot currently uses the JMX->Prom exporter to export metrics to Prometheus. Normally, this runs as an agent in @@ -266,11 +267,17 @@ protected List parseExportedPromMetrics(String response) } } - /** - * Reads the exported prometheus metrics from the JMX -> Prometheus server - * @return the corresponding GET response - */ - protected abstract SimpleHttpResponse getExportedPromMetrics(); + protected abstract PinotComponent getPinotComponent(); + + protected SimpleHttpResponse getExportedPromMetrics() { + try { + return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected abstract PinotMetricsFactory getPinotMetricsFactory(); /* Implementation copied from: https://github diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java index fb8a89869661..cf8778f6d4e3 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java @@ -19,33 +19,29 @@ package org.apache.pinot.common.metrics; -import io.prometheus.jmx.shaded.io.prometheus.client.exporter.HTTPServer; -import java.net.URI; import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.pinot.common.utils.SimpleHttpResponse; -import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class ServerPrometheusMetricsTest extends PinotPrometheusMetricsTest { +public abstract class ServerPrometheusMetricsTest extends PinotPrometheusMetricsTest { //all exported server metrics have this prefix - private static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; + protected static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; - private static final List METERS_ACCEPTING_CLIENT_ID = + protected static final List METERS_ACCEPTING_CLIENT_ID = List.of(ServerMeter.REALTIME_ROWS_CONSUMED, ServerMeter.REALTIME_ROWS_SANITIZED, ServerMeter.REALTIME_ROWS_FETCHED, ServerMeter.REALTIME_ROWS_FILTERED, ServerMeter.INVALID_REALTIME_ROWS_DROPPED, ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, ServerMeter.ROWS_WITH_ERRORS); - private static final List METERS_ACCEPTING_RAW_TABLE_NAMES = + protected static final List METERS_ACCEPTING_RAW_TABLE_NAMES = List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, ServerMeter.SEGMENT_UPLOAD_TIMEOUT); //gauges that accept clientId - private static final List GAUGES_ACCEPTING_CLIENT_ID = + protected static final List GAUGES_ACCEPTING_CLIENT_ID = List.of(ServerGauge.LLC_PARTITION_CONSUMING, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, @@ -53,116 +49,16 @@ public class ServerPrometheusMetricsTest extends PinotPrometheusMetricsTest { ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS); - private static final List GAUGES_ACCEPTING_PARTITION = + protected static final List GAUGES_ACCEPTING_PARTITION = List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, ServerGauge.REALTIME_INGESTION_OFFSET_LAG, ServerGauge.REALTIME_INGESTION_DELAY_MS, ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT); - private static final List GAUGES_ACCEPTING_RAW_TABLE_NAME = + protected static final List GAUGES_ACCEPTING_RAW_TABLE_NAME = List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, ServerGauge.LUCENE_INDEXING_DELAY_MS, ServerGauge.LUCENE_INDEXING_DELAY_DOCS); - private ServerMetrics _serverMetrics; - private HTTPServer _httpServer; - - @BeforeClass - public void setup() - throws Exception { - _httpServer = startExporter(PinotComponent.SERVER); - _serverMetrics = new ServerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); - } - - @Test(dataProvider = "serverTimers") - public void timerTest(ServerTimer serverTimer) { - if (serverTimer.isGlobal()) { - _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS); - assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_METRIC_PREFIX); - } else { - _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); - _serverMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); - - assertTimerExportedCorrectly(serverTimer.getTimerName(), ExportedLabels.TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); - assertTimerExportedCorrectly(serverTimer.getTimerName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - } - } - - @Test(dataProvider = "serverMeters") - public void meterTest(ServerMeter serverMeter) { - if (serverMeter.isGlobal()) { - _serverMetrics.addMeteredGlobalValue(serverMeter, 4L); - //we cannot use raw meter names for all meters as exported metrics don't follow any convention currently. - // For example, meters that track realtime exceptions start with prefix "realtime_exceptions" - if (meterTrackingRealtimeExceptions(serverMeter)) { - assertMeterExportedCorrectly(getRealtimeExceptionMeterName(serverMeter)); - } else { - assertMeterExportedCorrectly(serverMeter.getMeterName()); - } - } else { - if (METERS_ACCEPTING_CLIENT_ID.contains(serverMeter)) { - addMeterWithLabels(serverMeter, CLIENT_ID); - assertMeterExportedCorrectly(serverMeter.getMeterName(), - ExportedLabels.PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC); - } else if (METERS_ACCEPTING_RAW_TABLE_NAMES.contains(serverMeter)) { - addMeterWithLabels(serverMeter, ExportedLabelValues.TABLENAME); - assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.TABLENAME); - } else { - //we pass tableNameWithType to all remaining meters - addMeterWithLabels(serverMeter, TABLE_NAME_WITH_TYPE); - assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE); - } - } - } - - @Test(dataProvider = "serverGauges") - public void gaugeTest(ServerGauge serverGauge) { - if (serverGauge.isGlobal()) { - _serverMetrics.setValueOfGlobalGauge(serverGauge, 10L); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_METRIC_PREFIX); - } else { - if (serverGauge == ServerGauge.DEDUP_PRIMARY_KEYS_COUNT) { - //this gauge is currently exported as: `pinot_server_${partitionId}_Value{database="dedupPrimaryKeysCount", - // table="dedupPrimaryKeysCount.myTable",tableType="REALTIME",}`. We add an explicit test for it to maintain - // backward compatibility. todo: ServerGauge.DEDUP_PRIMARY_KEYS_COUNT should be moved to - // gaugesThatAcceptPartition. It should be exported as: - // `pinot_server_dedupPrimaryKeysCount_Value{partition="3", table="myTable",tableType="REALTIME",}` - addPartitionGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); - assertGaugeExportedCorrectly(String.valueOf(3), - List.of(ExportedLabelKeys.DATABASE, serverGauge.getGaugeName(), ExportedLabelKeys.TABLE, - "dedupPrimaryKeysCount.myTable", ExportedLabelKeys.TABLETYPE, ExportedLabelValues.TABLETYPE_REALTIME), - EXPORTED_METRIC_PREFIX); - } else if (GAUGES_ACCEPTING_CLIENT_ID.contains(serverGauge)) { - addGaugeWithLabels(serverGauge, CLIENT_ID); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), - ExportedLabels.PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC, EXPORTED_METRIC_PREFIX); - } else if (GAUGES_ACCEPTING_PARTITION.contains(serverGauge)) { - addPartitionGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.PARTITION_TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); - } else if (GAUGES_ACCEPTING_RAW_TABLE_NAME.contains(serverGauge)) { - addGaugeWithLabels(serverGauge, ExportedLabelValues.TABLENAME); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); - } else { - addGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); - assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, - EXPORTED_METRIC_PREFIX); - } - } - } - - private void addGaugeWithLabels(ServerGauge serverGauge, String labels) { - _serverMetrics.setValueOfTableGauge(labels, serverGauge, 100L); - } - - private void addPartitionGaugeWithLabels(ServerGauge serverGauge, String labels) { - _serverMetrics.setValueOfPartitionGauge(labels, 3, serverGauge, 100L); - } - - public void addMeterWithLabels(ServerMeter serverMeter, String labels) { - _serverMetrics.addMeteredTableValue(labels, serverMeter, 4L); - } - @DataProvider(name = "serverTimers") public Object[] serverTimers() { return ServerTimer.values(); // Provide all values of ServerTimer enum @@ -178,36 +74,8 @@ public Object[] serverGauge() { return ServerGauge.values(); // Provide all values of ServerTimer enum } - @AfterClass - public void cleanup() { - _httpServer.close(); - } - @Override - protected SimpleHttpResponse getExportedPromMetrics() { - try { - return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private boolean meterTrackingRealtimeExceptions(ServerMeter serverMeter) { - return serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS - || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS - || serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS || serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS; - } - - private String getRealtimeExceptionMeterName(ServerMeter serverMeter) { - String meterName = serverMeter.getMeterName(); - return "realtime_exceptions_" + meterName.substring(0, meterName.lastIndexOf("Exceptions")); - } - - private void assertMeterExportedCorrectly(String exportedMeterName) { - assertMeterExportedCorrectly(exportedMeterName, EXPORTED_METRIC_PREFIX); - } - - private void assertMeterExportedCorrectly(String exportedMeterName, List labels) { - assertMeterExportedCorrectly(exportedMeterName, labels, EXPORTED_METRIC_PREFIX); + protected PinotComponent getPinotComponent() { + return PinotComponent.SERVER; } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerBrokerPrometheusMetricsTest.java new file mode 100644 index 000000000000..555fdb3d588b --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerBrokerPrometheusMetricsTest.java @@ -0,0 +1,80 @@ +package org.apache.pinot.common.metrics; + +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +public class YammerBrokerPrometheusMetricsTest extends BrokerPrometheusMetricsTest { + + private BrokerMetrics _brokerMetrics; + + @BeforeClass + public void setup() + throws Exception { + _brokerMetrics = new BrokerMetrics(getPinotMetricsFactory().getPinotMetricsRegistry()); + } + + @Test(dataProvider = "brokerTimers") + public void timerTest(BrokerTimer timer) { + if (timer.isGlobal()) { + _brokerMetrics.addTimedValue(timer, 30_000, TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(timer.getTimerName(), EXPORTED_METRIC_PREFIX); + } else { + _brokerMetrics.addTimedTableValue(PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME, timer, 30_000L, + TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(timer.getTimerName(), PinotPrometheusMetricsTest.ExportedLabels.TABLENAME, + EXPORTED_METRIC_PREFIX); + } + } + + @Test(dataProvider = "brokerGauges") + public void gaugeTest(BrokerGauge gauge) { + if (gauge.isGlobal()) { + _brokerMetrics.setOrUpdateGlobalGauge(gauge, () -> 5L); + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); + } else { + if (gauge == BrokerGauge.REQUEST_SIZE) { + _brokerMetrics.setOrUpdateTableGauge(PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME, gauge, 5L); + assertGaugeExportedCorrectly(gauge.getGaugeName(), PinotPrometheusMetricsTest.ExportedLabels.TABLENAME, + EXPORTED_METRIC_PREFIX); + } else { + _brokerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 5L); + assertGaugeExportedCorrectly(gauge.getGaugeName(), + PinotPrometheusMetricsTest.ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + } + } + } + + @Test(dataProvider = "brokerMeters") + public void meterTest(BrokerMeter meter) { + if (meter.isGlobal()) { + _brokerMetrics.addMeteredGlobalValue(meter, 5L); + if (GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX.contains(meter)) { + String exportedMeterPrefix = String.format("%s_%s", EXPORTED_METRIC_PREFIX_EXCEPTIONS, + StringUtils.remove(meter.getMeterName(), "Exceptions")); + assertMeterExportedCorrectly(exportedMeterPrefix, EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); + } + } else { + if (METERS_ACCEPTING_RAW_TABLENAME.contains(meter)) { + _brokerMetrics.addMeteredTableValue(PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME, meter, 5L); + assertMeterExportedCorrectly(meter.getMeterName(), PinotPrometheusMetricsTest.ExportedLabels.TABLENAME, + EXPORTED_METRIC_PREFIX); + } else { + _brokerMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 5L); + assertMeterExportedCorrectly(meter.getMeterName(), + PinotPrometheusMetricsTest.ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + } + } + } + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new YammerMetricsFactory(); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerControllerPrometheusMetricsTest.java new file mode 100644 index 000000000000..d34ab15b78bf --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerControllerPrometheusMetricsTest.java @@ -0,0 +1,144 @@ +package org.apache.pinot.common.metrics; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; +import org.apache.helix.task.TaskState; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +public class YammerControllerPrometheusMetricsTest extends ControllerPrometheusMetricsTest { + + private ControllerMetrics _controllerMetrics; + + @BeforeClass + public void setup() + throws Exception { + _controllerMetrics = new ControllerMetrics(getPinotMetricsFactory().getPinotMetricsRegistry()); + } + + @Test(dataProvider = "controllerTimers") + public void timerTest(ControllerTimer controllerTimer) { + if (controllerTimer.isGlobal()) { + _controllerMetrics.addTimedValue(controllerTimer, 30_000, TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(controllerTimer.getTimerName(), EXPORTED_METRIC_PREFIX); + } else { + _controllerMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, controllerTimer, 30_000L, TimeUnit.MILLISECONDS); + _controllerMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, controllerTimer, 30_000L, + TimeUnit.MILLISECONDS); + + assertTimerExportedCorrectly(controllerTimer.getTimerName(), ExportedLabels.TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(controllerTimer.getTimerName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); + } + } + + @Test(dataProvider = "controllerMeters") + public void meterTest(ControllerMeter meter) { + if (meter.isGlobal()) { + _controllerMetrics.addMeteredGlobalValue(meter, 5L); + String meterName = meter.getMeterName(); + //some meters contain a "controller" prefix. For example, controllerInstancePostError. These meters are + // exported as 'pinot_controller_pinot_controller_InstancePostError'. So we strip the 'controller' from + // 'controllerInstancePostError' + String strippedMeterName = StringUtils.remove(meterName, "controller"); + assertMeterExportedCorrectly(strippedMeterName, EXPORTED_METRIC_PREFIX); + } else { + + String meterName = meter.getMeterName(); + String strippedMeterName = StringUtils.remove(meterName, "controller"); + + if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR) { + addMeterWithLabels(meter, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); + assertMeterExportedCorrectly(meterName, + List.of(ExportedLabelKeys.TABLE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), EXPORTED_METRIC_PREFIX); + } else if (meter == ControllerMeter.CONTROLLER_PERIODIC_TASK_RUN) { + addMeterWithLabels(meter, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); + assertMeterExportedCorrectly( + String.format("%s_%s", strippedMeterName, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), + EXPORTED_METRIC_PREFIX); + } else if (meter == ControllerMeter.PERIODIC_TASK_ERROR) { + addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE + "." + ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC); + assertMeterExportedCorrectly(meterName, ExportedLabels.CONTROLLER_TASKTYPE_TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); + } else { + addMeterWithLabels(meter, TABLE_NAME_WITH_TYPE); + addMeterWithLabels(meter, ExportedLabelValues.TABLENAME); + if (meter == ControllerMeter.CONTROLLER_TABLE_SEGMENT_UPLOAD_ERROR) { + assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meterName, ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); + } else { + assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(strippedMeterName, ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); + } + } + } + } + + @Test(dataProvider = "controllerGauges") + public void gaugeTest(ControllerGauge controllerGauge) { + if (controllerGauge.isGlobal()) { + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, 1L); + //some global gauges also accept the taskType (which should not be). todo: this should be fixed + if (GLOBAL_GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) { + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, 1L); + String strippedMetricName = getStrippedMetricName(controllerGauge); + assertGaugeExportedCorrectly(strippedMetricName, + List.of(LABEL_KEY_TASK_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC), EXPORTED_METRIC_PREFIX); + } else { + _controllerMetrics.setValueOfGlobalGauge(controllerGauge, 1L); + String strippedMetricName = getStrippedMetricName(controllerGauge); + assertGaugeExportedCorrectly(strippedMetricName, EXPORTED_METRIC_PREFIX); + } + } else { + if (GAUGES_ACCEPTING_PARTITION.contains(controllerGauge)) { + _controllerMetrics.setValueOfPartitionGauge(TABLE_NAME_WITH_TYPE, 3, controllerGauge, 10L); + String strippedGaugeName = getStrippedMetricName(controllerGauge); + assertGaugeExportedCorrectly(strippedGaugeName, ExportedLabels.PARTITION_TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); + } else if (GAUGES_ACCEPTING_TASKTYPE.contains(controllerGauge)) { + _controllerMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, + controllerGauge, () -> 50L); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), + ExportedLabels.TABLENAME_TABLETYPE_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); + } else if (GAUGES_ACCEPTING_RAW_TABLENAME.contains(controllerGauge)) { + addGaugeWithLabels(controllerGauge, ExportedLabelValues.TABLENAME); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); + } else if (controllerGauge == ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED) { + addGaugeWithLabels(controllerGauge, + String.format("%s.%s", TABLE_NAME_WITH_TYPE, ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC)); + assertGaugeExportedCorrectly(ControllerGauge.CRON_SCHEDULER_JOB_SCHEDULED.getGaugeName(), + ExportedLabels.TABLENAMEWITHTYPE_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); + } else if (controllerGauge == ControllerGauge.TASK_STATUS) { + addGaugeWithLabels(controllerGauge, + String.format("%s.%s", ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC, TaskState.IN_PROGRESS)); + assertGaugeExportedCorrectly(ControllerGauge.TASK_STATUS.getGaugeName(), + ExportedLabels.JOBSTATUS_CONTROLLER_TASKTYPE, EXPORTED_METRIC_PREFIX); + } else { + addGaugeWithLabels(controllerGauge, TABLE_NAME_WITH_TYPE); + assertGaugeExportedCorrectly(controllerGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); + } + } + } + + private void addGaugeWithLabels(ControllerGauge gauge, String labels) { + _controllerMetrics.setValueOfTableGauge(labels, gauge, 5L); + } + + private static String getStrippedMetricName(ControllerGauge controllerGauge) { + return StringUtils.remove(controllerGauge.getGaugeName(), "controller"); + } + + private void addMeterWithLabels(ControllerMeter meter, String labels) { + _controllerMetrics.addMeteredTableValue(labels, meter, 1L); + } + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new YammerMetricsFactory(); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerMinionPrometheusMetricsTest.java new file mode 100644 index 000000000000..0b47984a4764 --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerMinionPrometheusMetricsTest.java @@ -0,0 +1,94 @@ +package org.apache.pinot.common.metrics; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +public class YammerMinionPrometheusMetricsTest extends MinionPrometheusMetricsTest { + + private MinionMetrics _minionMetrics; + + @BeforeClass + public void setup() { + _minionMetrics = new MinionMetrics(getPinotMetricsFactory().getPinotMetricsRegistry()); + } + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new YammerMetricsFactory(); + } + + @Test(dataProvider = "minionTimers") + public void timerTest(MinionTimer timer) { + + _minionMetrics.addTimedValue(ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, timer, 30L, TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(timer.getTimerName(), + List.of(ExportedLabelKeys.ID, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); + + _minionMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, timer, 30L, + TimeUnit.MILLISECONDS); + + if (timer == MinionTimer.TASK_THREAD_CPU_TIME_NS) { + assertTimerExportedCorrectly(timer.getTimerName(), + List.of(ExportedLabelKeys.DATABASE, ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME, ExportedLabelKeys.TABLE, + "myTable_REALTIME.SegmentImportTask"), EXPORTED_METRIC_PREFIX); + } else { + assertTimerExportedCorrectly(timer.getTimerName(), ExportedLabels.TABLENAME_TABLETYPE_MINION_TASKTYPE, + EXPORTED_METRIC_PREFIX); + } + } + + @Test(dataProvider = "minionMeters") + public void meterTest(MinionMeter meter) { + if (meter.isGlobal()) { + validateGlobalMeters(meter); + } else { + validateMetersWithLabels(meter); + } + } + + private void validateGlobalMeters(MinionMeter meter) { + _minionMetrics.addMeteredGlobalValue(meter, 5L); + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); + } + + private void validateMetersWithLabels(MinionMeter meter) { + if (meter.getMeterName().startsWith(METER_PREFIX_NO_TASKS)) { + _minionMetrics.addMeteredTableValue(ExportedLabelValues.TABLENAME, meter, 1L); + assertMeterExportedCorrectly(meter.getMeterName(), List.of(ExportedLabelKeys.ID, ExportedLabelValues.TABLENAME), + EXPORTED_METRIC_PREFIX); + + _minionMetrics.addMeteredValue(ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, meter, 1L); + assertMeterExportedCorrectly(meter.getMeterName(), + List.of(ExportedLabelKeys.ID, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT), EXPORTED_METRIC_PREFIX); + } else if (meter == MinionMeter.SEGMENT_UPLOAD_FAIL_COUNT || meter == MinionMeter.SEGMENT_DOWNLOAD_FAIL_COUNT) { + + _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, meter, 1L); + assertMeterExportedCorrectly(meter.getMeterName(), List.of(ExportedLabelKeys.ID, TABLE_NAME_WITH_TYPE), + EXPORTED_METRIC_PREFIX); + } else { + //all remaining meters are also being used as global meters, check their usage + _minionMetrics.addMeteredGlobalValue(meter, 1L); + _minionMetrics.addMeteredTableValue(TABLE_NAME_WITH_TYPE, ExportedLabelValues.MINION_TASK_SEGMENT_IMPORT, meter, + 1L); + assertMeterExportedCorrectly(meter.getMeterName(), EXPORTED_METRIC_PREFIX); + assertMeterExportedCorrectly(meter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE_MINION_TASKTYPE, + EXPORTED_METRIC_PREFIX); + } + } + + @Test(dataProvider = "minionGauges") + public void gaugeTest(MinionGauge gauge) { + if (gauge.isGlobal()) { + _minionMetrics.setValueOfGlobalGauge(gauge, 1L); + assertGaugeExportedCorrectly(gauge.getGaugeName(), EXPORTED_METRIC_PREFIX); + } else { + _minionMetrics.setOrUpdateTableGauge(TABLE_NAME_WITH_TYPE, gauge, 1L); + assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); + } + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerServerPrometheusMetricsTest.java new file mode 100644 index 000000000000..9995e3e8c23e --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerServerPrometheusMetricsTest.java @@ -0,0 +1,134 @@ +package org.apache.pinot.common.metrics; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +public class YammerServerPrometheusMetricsTest extends ServerPrometheusMetricsTest { + + protected ServerMetrics _serverMetrics; + + @BeforeClass + public void setup() + throws Exception { + _serverMetrics = new ServerMetrics(getPinotMetricsFactory().getPinotMetricsRegistry()); + } + + @Test(dataProvider = "serverTimers") + public void timerTest(ServerTimer serverTimer) { + if (serverTimer.isGlobal()) { + _serverMetrics.addTimedValue(serverTimer, 30_000, TimeUnit.MILLISECONDS); + assertTimerExportedCorrectly(serverTimer.getTimerName(), EXPORTED_METRIC_PREFIX); + } else { + _serverMetrics.addTimedTableValue(TABLE_NAME_WITH_TYPE, serverTimer, 30_000L, TimeUnit.MILLISECONDS); + _serverMetrics.addTimedTableValue(ExportedLabelValues.TABLENAME, serverTimer, 30_000L, TimeUnit.MILLISECONDS); + + assertTimerExportedCorrectly(serverTimer.getTimerName(), ExportedLabels.TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); + assertTimerExportedCorrectly(serverTimer.getTimerName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); + } + } + + @Test(dataProvider = "serverMeters") + public void meterTest(ServerMeter serverMeter) { + if (serverMeter.isGlobal()) { + _serverMetrics.addMeteredGlobalValue(serverMeter, 4L); + //we cannot use raw meter names for all meters as exported metrics don't follow any convention currently. + // For example, meters that track realtime exceptions start with prefix "realtime_exceptions" + if (meterTrackingRealtimeExceptions(serverMeter)) { + assertMeterExportedCorrectly(getRealtimeExceptionMeterName(serverMeter)); + } else { + assertMeterExportedCorrectly(serverMeter.getMeterName()); + } + } else { + if (METERS_ACCEPTING_CLIENT_ID.contains(serverMeter)) { + addMeterWithLabels(serverMeter, CLIENT_ID); + assertMeterExportedCorrectly(serverMeter.getMeterName(), + ExportedLabels.PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC); + } else if (METERS_ACCEPTING_RAW_TABLE_NAMES.contains(serverMeter)) { + addMeterWithLabels(serverMeter, ExportedLabelValues.TABLENAME); + assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.TABLENAME); + } else { + //we pass tableNameWithType to all remaining meters + addMeterWithLabels(serverMeter, TABLE_NAME_WITH_TYPE); + assertMeterExportedCorrectly(serverMeter.getMeterName(), ExportedLabels.TABLENAME_TABLETYPE); + } + } + } + + @Test(dataProvider = "serverGauges") + public void gaugeTest(ServerGauge serverGauge) { + if (serverGauge.isGlobal()) { + _serverMetrics.setValueOfGlobalGauge(serverGauge, 10L); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), EXPORTED_METRIC_PREFIX); + } else { + if (serverGauge == ServerGauge.DEDUP_PRIMARY_KEYS_COUNT) { + //this gauge is currently exported as: `pinot_server_${partitionId}_Value{database="dedupPrimaryKeysCount", + // table="dedupPrimaryKeysCount.myTable",tableType="REALTIME",}`. We add an explicit test for it to maintain + // backward compatibility. todo: ServerGauge.DEDUP_PRIMARY_KEYS_COUNT should be moved to + // gaugesThatAcceptPartition. It should be exported as: + // `pinot_server_dedupPrimaryKeysCount_Value{partition="3", table="myTable",tableType="REALTIME",}` + addPartitionGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); + assertGaugeExportedCorrectly(String.valueOf(3), + List.of(ExportedLabelKeys.DATABASE, serverGauge.getGaugeName(), ExportedLabelKeys.TABLE, + "dedupPrimaryKeysCount.myTable", ExportedLabelKeys.TABLETYPE, ExportedLabelValues.TABLETYPE_REALTIME), + EXPORTED_METRIC_PREFIX); + } else if (GAUGES_ACCEPTING_CLIENT_ID.contains(serverGauge)) { + addGaugeWithLabels(serverGauge, CLIENT_ID); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), + ExportedLabels.PARTITION_TABLENAME_TABLETYPE_KAFKATOPIC, EXPORTED_METRIC_PREFIX); + } else if (GAUGES_ACCEPTING_PARTITION.contains(serverGauge)) { + addPartitionGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.PARTITION_TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); + } else if (GAUGES_ACCEPTING_RAW_TABLE_NAME.contains(serverGauge)) { + addGaugeWithLabels(serverGauge, ExportedLabelValues.TABLENAME); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.TABLENAME, EXPORTED_METRIC_PREFIX); + } else { + addGaugeWithLabels(serverGauge, TABLE_NAME_WITH_TYPE); + assertGaugeExportedCorrectly(serverGauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, + EXPORTED_METRIC_PREFIX); + } + } + } + + private void addGaugeWithLabels(ServerGauge serverGauge, String labels) { + _serverMetrics.setValueOfTableGauge(labels, serverGauge, 100L); + } + + private void addPartitionGaugeWithLabels(ServerGauge serverGauge, String labels) { + _serverMetrics.setValueOfPartitionGauge(labels, 3, serverGauge, 100L); + } + + public void addMeterWithLabels(ServerMeter serverMeter, String labels) { + _serverMetrics.addMeteredTableValue(labels, serverMeter, 4L); + } + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new YammerMetricsFactory(); + } + + private boolean meterTrackingRealtimeExceptions(ServerMeter serverMeter) { + return serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS + || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS + || serverMeter == ServerMeter.SCHEDULING_TIMEOUT_EXCEPTIONS || serverMeter == ServerMeter.UNCAUGHT_EXCEPTIONS; + } + + private String getRealtimeExceptionMeterName(ServerMeter serverMeter) { + String meterName = serverMeter.getMeterName(); + return "realtime_exceptions_" + meterName.substring(0, meterName.lastIndexOf("Exceptions")); + } + + private void assertMeterExportedCorrectly(String exportedMeterName) { + assertMeterExportedCorrectly(exportedMeterName, EXPORTED_METRIC_PREFIX); + } + + private void assertMeterExportedCorrectly(String exportedMeterName, List labels) { + assertMeterExportedCorrectly(exportedMeterName, labels, EXPORTED_METRIC_PREFIX); + } +} From 5489639f3a8dd12220ce2afb7031f21c8a5ebed2 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 14:21:56 +0530 Subject: [PATCH 36/45] Addressed comments --- .../metrics/BrokerPrometheusMetricsTest.java | 65 --------------- .../ControllerPrometheusMetricsTest.java | 74 ----------------- .../metrics/MinionPrometheusMetricsTest.java | 53 ------------ .../metrics/ServerPrometheusMetricsTest.java | 81 ------------------- .../BrokerPrometheusMetricsTest.java} | 64 +++++++++++++-- .../ControllerPrometheusMetricsTest.java} | 67 +++++++++++++-- .../MinionPrometheusMetricsTest.java} | 54 ++++++++++--- .../PinotPrometheusMetricsTest.java | 49 +++++------ .../ServerPrometheusMetricsTest.java} | 78 ++++++++++++++++-- ...DropwizardBrokerPrometheusMetricsTest.java | 18 +++++ ...wizardControllerPrometheusMetricsTest.java | 19 +++++ ...DropwizardMinionPrometheusMetricsTest.java | 19 +++++ ...DropwizardServerPrometheusMetricsTest.java | 20 +++++ .../YammerBrokerPrometheusMetricsTest.java | 14 ++++ ...YammerControllerPrometheusMetricsTest.java | 14 ++++ .../YammerMinionPrometheusMetricsTest.java | 14 ++++ .../YammerServerPrometheusMetricsTest.java | 14 ++++ 17 files changed, 384 insertions(+), 333 deletions(-) delete mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java delete mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java delete mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java delete mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{YammerBrokerPrometheusMetricsTest.java => prometheus/BrokerPrometheusMetricsTest.java} (50%) rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{YammerControllerPrometheusMetricsTest.java => prometheus/ControllerPrometheusMetricsTest.java} (70%) rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{YammerMinionPrometheusMetricsTest.java => prometheus/MinionPrometheusMetricsTest.java} (65%) rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{ => prometheus}/PinotPrometheusMetricsTest.java (91%) rename pinot-common/src/test/java/org/apache/pinot/common/metrics/{YammerServerPrometheusMetricsTest.java => prometheus/ServerPrometheusMetricsTest.java} (61%) create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java deleted file mode 100644 index cefa4f754d05..000000000000 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/BrokerPrometheusMetricsTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.pinot.common.metrics; - -import java.util.List; -import org.testng.annotations.DataProvider; - - -public abstract class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { - - protected static final String EXPORTED_METRIC_PREFIX = "pinot_broker_"; - - protected static final String EXPORTED_METRIC_PREFIX_EXCEPTIONS = "exceptions"; - - protected static final List GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX = - List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, - BrokerMeter.QUERY_REJECTED_EXCEPTIONS, BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, - BrokerMeter.RESOURCE_MISSING_EXCEPTIONS); - - protected static final List METERS_ACCEPTING_RAW_TABLENAME = - List.of(BrokerMeter.QUERIES, BrokerMeter.NO_SERVER_FOUND_EXCEPTIONS, BrokerMeter.DOCUMENTS_SCANNED, - BrokerMeter.ENTRIES_SCANNED_IN_FILTER, BrokerMeter.BROKER_RESPONSES_WITH_UNAVAILABLE_SEGMENTS, - BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, - BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, - BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED, BrokerMeter.BROKER_RESPONSES_WITH_TIMEOUTS, - BrokerMeter.ENTRIES_SCANNED_POST_FILTER, BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE, - BrokerMeter.QUERY_QUOTA_EXCEEDED); - - @Override - protected PinotComponent getPinotComponent() { - return PinotComponent.BROKER; - } - - @DataProvider(name = "brokerTimers") - public Object[] brokerTimers() { - return BrokerTimer.values(); - } - - @DataProvider(name = "brokerMeters") - public Object[] brokerMeters() { - return BrokerMeter.values(); - } - - @DataProvider(name = "brokerGauges") - public Object[] brokerGauges() { - return BrokerGauge.values(); - } -} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java deleted file mode 100644 index c1657535bb79..000000000000 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerPrometheusMetricsTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.pinot.common.metrics; - -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.StringUtils; -import org.apache.helix.task.TaskState; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - - -public abstract class ControllerPrometheusMetricsTest extends PinotPrometheusMetricsTest { - //all exported controller metrics have this prefix - protected static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; - protected static final String LABEL_KEY_TASK_TYPE = "taskType"; - - //that accept global gauge with suffix - protected static final List GLOBAL_GAUGES_ACCEPTING_TASKTYPE = - List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, - ControllerGauge.NUM_MINION_SUBTASKS_WAITING, ControllerGauge.NUM_MINION_SUBTASKS_ERROR, - ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); - - //local gauges that accept partition - protected static final List GAUGES_ACCEPTING_PARTITION = - List.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS); - - //these accept task type - protected static final List GAUGES_ACCEPTING_TASKTYPE = - List.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, - ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, - ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR); - - protected static final List GAUGES_ACCEPTING_RAW_TABLENAME = - List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); - - @Override - protected PinotComponent getPinotComponent() { - return PinotComponent.CONTROLLER; - } - - @DataProvider(name = "controllerTimers") - public Object[] controllerTimers() { - return ControllerTimer.values(); - } - - @DataProvider(name = "controllerMeters") - public Object[] controllerMeters() { - return ControllerMeter.values(); - } - - @DataProvider(name = "controllerGauges") - public Object[] controllerGauges() { - return ControllerGauge.values(); - } -} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java deleted file mode 100644 index 998cae8e12ad..000000000000 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/MinionPrometheusMetricsTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.pinot.common.metrics; - -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - - -public abstract class MinionPrometheusMetricsTest extends PinotPrometheusMetricsTest { - //all exported minion metrics have this prefix - protected static final String EXPORTED_METRIC_PREFIX = "pinot_minion_"; - protected static final String METER_PREFIX_NO_TASKS = "numberTasks"; - - @DataProvider(name = "minionTimers") - public Object[] minionTimers() { - return MinionTimer.values(); - } - - @DataProvider(name = "minionMeters") - public Object[] minionMeters() { - return MinionMeter.values(); - } - - @DataProvider(name = "minionGauges") - public Object[] minionGauges() { - return MinionGauge.values(); - } - - @Override - protected PinotComponent getPinotComponent() { - return PinotComponent.MINION; - } -} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java deleted file mode 100644 index cf8778f6d4e3..000000000000 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/ServerPrometheusMetricsTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.pinot.common.metrics; - -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - - -public abstract class ServerPrometheusMetricsTest extends PinotPrometheusMetricsTest { - //all exported server metrics have this prefix - protected static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; - - protected static final List METERS_ACCEPTING_CLIENT_ID = - List.of(ServerMeter.REALTIME_ROWS_CONSUMED, ServerMeter.REALTIME_ROWS_SANITIZED, - ServerMeter.REALTIME_ROWS_FETCHED, ServerMeter.REALTIME_ROWS_FILTERED, - ServerMeter.INVALID_REALTIME_ROWS_DROPPED, ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, - ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, ServerMeter.ROWS_WITH_ERRORS); - - protected static final List METERS_ACCEPTING_RAW_TABLE_NAMES = - List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, - ServerMeter.SEGMENT_UPLOAD_TIMEOUT); - - //gauges that accept clientId - protected static final List GAUGES_ACCEPTING_CLIENT_ID = - List.of(ServerGauge.LLC_PARTITION_CONSUMING, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, - ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, - ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS); - - protected static final List GAUGES_ACCEPTING_PARTITION = - List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, - ServerGauge.REALTIME_INGESTION_OFFSET_LAG, ServerGauge.REALTIME_INGESTION_DELAY_MS, - ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS, - ServerGauge.DEDUP_PRIMARY_KEYS_COUNT); - - protected static final List GAUGES_ACCEPTING_RAW_TABLE_NAME = - List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, - ServerGauge.LUCENE_INDEXING_DELAY_MS, ServerGauge.LUCENE_INDEXING_DELAY_DOCS); - - @DataProvider(name = "serverTimers") - public Object[] serverTimers() { - return ServerTimer.values(); // Provide all values of ServerTimer enum - } - - @DataProvider(name = "serverMeters") - public Object[] serverMeter() { - return ServerMeter.values(); // Provide all values of ServerTimer enum - } - - @DataProvider(name = "serverGauges") - public Object[] serverGauge() { - return ServerGauge.values(); // Provide all values of ServerTimer enum - } - - @Override - protected PinotComponent getPinotComponent() { - return PinotComponent.SERVER; - } -} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java similarity index 50% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerBrokerPrometheusMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java index 555fdb3d588b..37ad6850227e 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerBrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java @@ -1,21 +1,56 @@ -package org.apache.pinot.common.metrics; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.apache.pinot.common.metrics.prometheus; + +import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; -import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.apache.pinot.common.metrics.BrokerGauge; +import org.apache.pinot.common.metrics.BrokerMeter; +import org.apache.pinot.common.metrics.BrokerMetrics; +import org.apache.pinot.common.metrics.BrokerTimer; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class YammerBrokerPrometheusMetricsTest extends BrokerPrometheusMetricsTest { +public abstract class BrokerPrometheusMetricsTest extends PinotPrometheusMetricsTest { + + private static final String EXPORTED_METRIC_PREFIX = "pinot_broker_"; + + private static final String EXPORTED_METRIC_PREFIX_EXCEPTIONS = "exceptions"; + + private static final List GLOBAL_METERS_WITH_EXCEPTIONS_PREFIX = + List.of(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, + BrokerMeter.QUERY_REJECTED_EXCEPTIONS, BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, + BrokerMeter.RESOURCE_MISSING_EXCEPTIONS); + + private static final List METERS_ACCEPTING_RAW_TABLENAME = + List.of(BrokerMeter.QUERIES, BrokerMeter.NO_SERVER_FOUND_EXCEPTIONS, BrokerMeter.DOCUMENTS_SCANNED, + BrokerMeter.ENTRIES_SCANNED_IN_FILTER, BrokerMeter.BROKER_RESPONSES_WITH_UNAVAILABLE_SEGMENTS, + BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, + BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, + BrokerMeter.BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED, BrokerMeter.BROKER_RESPONSES_WITH_TIMEOUTS, + BrokerMeter.ENTRIES_SCANNED_POST_FILTER, BrokerMeter.TOTAL_SERVER_RESPONSE_SIZE, + BrokerMeter.QUERY_QUOTA_EXCEEDED); private BrokerMetrics _brokerMetrics; @BeforeClass public void setup() throws Exception { - _brokerMetrics = new BrokerMetrics(getPinotMetricsFactory().getPinotMetricsRegistry()); + _brokerMetrics = new BrokerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); } @Test(dataProvider = "brokerTimers") @@ -74,7 +109,22 @@ public void meterTest(BrokerMeter meter) { } @Override - protected PinotMetricsFactory getPinotMetricsFactory() { - return new YammerMetricsFactory(); + protected PinotComponent getPinotComponent() { + return PinotComponent.BROKER; + } + + @DataProvider(name = "brokerTimers") + public Object[] brokerTimers() { + return BrokerTimer.values(); + } + + @DataProvider(name = "brokerMeters") + public Object[] brokerMeters() { + return BrokerMeter.values(); + } + + @DataProvider(name = "brokerGauges") + public Object[] brokerGauges() { + return BrokerGauge.values(); } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java similarity index 70% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerControllerPrometheusMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java index d34ab15b78bf..c88e22f673c0 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java @@ -1,23 +1,61 @@ -package org.apache.pinot.common.metrics; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.apache.pinot.common.metrics.prometheus; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.helix.task.TaskState; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; -import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.apache.pinot.common.metrics.ControllerGauge; +import org.apache.pinot.common.metrics.ControllerMeter; +import org.apache.pinot.common.metrics.ControllerMetrics; +import org.apache.pinot.common.metrics.ControllerTimer; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class YammerControllerPrometheusMetricsTest extends ControllerPrometheusMetricsTest { +public abstract class ControllerPrometheusMetricsTest extends PinotPrometheusMetricsTest { + //all exported controller metrics have this prefix + private static final String EXPORTED_METRIC_PREFIX = "pinot_controller_"; + private static final String LABEL_KEY_TASK_TYPE = "taskType"; + + //that accept global gauge with suffix + private static final List GLOBAL_GAUGES_ACCEPTING_TASKTYPE = + List.of(ControllerGauge.NUM_MINION_TASKS_IN_PROGRESS, ControllerGauge.NUM_MINION_SUBTASKS_RUNNING, + ControllerGauge.NUM_MINION_SUBTASKS_WAITING, ControllerGauge.NUM_MINION_SUBTASKS_ERROR, + ControllerGauge.PERCENT_MINION_SUBTASKS_IN_QUEUE, ControllerGauge.PERCENT_MINION_SUBTASKS_IN_ERROR); + + //local gauges that accept partition + private static final List GAUGES_ACCEPTING_PARTITION = + List.of(ControllerGauge.MAX_RECORDS_LAG, ControllerGauge.MAX_RECORD_AVAILABILITY_LAG_MS); + + //these accept task type + private static final List GAUGES_ACCEPTING_TASKTYPE = + List.of(ControllerGauge.TIME_MS_SINCE_LAST_MINION_TASK_METADATA_UPDATE, + ControllerGauge.TIME_MS_SINCE_LAST_SUCCESSFUL_MINION_TASK_GENERATION, + ControllerGauge.LAST_MINION_TASK_GENERATION_ENCOUNTERS_ERROR); + + private static final List GAUGES_ACCEPTING_RAW_TABLENAME = + List.of(ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE); private ControllerMetrics _controllerMetrics; @BeforeClass public void setup() throws Exception { - _controllerMetrics = new ControllerMetrics(getPinotMetricsFactory().getPinotMetricsRegistry()); + _controllerMetrics = new ControllerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); } @Test(dataProvider = "controllerTimers") @@ -138,7 +176,22 @@ private void addMeterWithLabels(ControllerMeter meter, String labels) { } @Override - protected PinotMetricsFactory getPinotMetricsFactory() { - return new YammerMetricsFactory(); + protected PinotComponent getPinotComponent() { + return PinotComponent.CONTROLLER; + } + + @DataProvider(name = "controllerTimers") + public Object[] controllerTimers() { + return ControllerTimer.values(); + } + + @DataProvider(name = "controllerMeters") + public Object[] controllerMeters() { + return ControllerMeter.values(); + } + + @DataProvider(name = "controllerGauges") + public Object[] controllerGauges() { + return ControllerGauge.values(); } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java similarity index 65% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerMinionPrometheusMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java index 0b47984a4764..e3d915cc03b7 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerMinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java @@ -1,25 +1,39 @@ -package org.apache.pinot.common.metrics; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.apache.pinot.common.metrics.prometheus; import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; -import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.apache.pinot.common.metrics.MinionGauge; +import org.apache.pinot.common.metrics.MinionMeter; +import org.apache.pinot.common.metrics.MinionMetrics; +import org.apache.pinot.common.metrics.MinionTimer; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class YammerMinionPrometheusMetricsTest extends MinionPrometheusMetricsTest { +public abstract class MinionPrometheusMetricsTest extends PinotPrometheusMetricsTest { + //all exported minion metrics have this prefix + private static final String EXPORTED_METRIC_PREFIX = "pinot_minion_"; + private static final String METER_PREFIX_NO_TASKS = "numberTasks"; private MinionMetrics _minionMetrics; @BeforeClass public void setup() { - _minionMetrics = new MinionMetrics(getPinotMetricsFactory().getPinotMetricsRegistry()); - } - - @Override - protected PinotMetricsFactory getPinotMetricsFactory() { - return new YammerMetricsFactory(); + _minionMetrics = new MinionMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); } @Test(dataProvider = "minionTimers") @@ -91,4 +105,24 @@ public void gaugeTest(MinionGauge gauge) { assertGaugeExportedCorrectly(gauge.getGaugeName(), ExportedLabels.TABLENAME_TABLETYPE, EXPORTED_METRIC_PREFIX); } } + + @DataProvider(name = "minionTimers") + public Object[] minionTimers() { + return MinionTimer.values(); + } + + @DataProvider(name = "minionMeters") + public Object[] minionMeters() { + return MinionMeter.values(); + } + + @DataProvider(name = "minionGauges") + public Object[] minionGauges() { + return MinionGauge.values(); + } + + @Override + protected PinotComponent getPinotComponent() { + return PinotComponent.MINION; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java similarity index 91% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java index 5fb7f79298c4..a00458fe25a9 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java @@ -1,23 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. */ -package org.apache.pinot.common.metrics; +package org.apache.pinot.common.metrics.prometheus; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Objects; @@ -46,8 +40,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.SimpleHttpResponse; import org.apache.pinot.common.utils.http.HttpClient; -import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; import org.apache.pinot.spi.config.table.TableType; import org.apache.pinot.spi.env.PinotConfiguration; @@ -58,11 +50,11 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; -import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelKeys.*; -import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC; -import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.IN_PROGRESS; -import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME; -import static org.apache.pinot.common.metrics.PinotPrometheusMetricsTest.ExportedLabelValues.TABLETYPE_REALTIME; +import static org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelKeys.*; +import static org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelValues.CONTROLLER_PERIODIC_TASK_CHC; +import static org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelValues.IN_PROGRESS; +import static org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelValues.TABLENAME_WITH_TYPE_REALTIME; +import static org.apache.pinot.common.metrics.prometheus.PinotPrometheusMetricsTest.ExportedLabelValues.TABLETYPE_REALTIME; import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME; @@ -77,9 +69,10 @@ public abstract class PinotPrometheusMetricsTest { protected HttpClient _httpClient; + protected PinotMetricsFactory _pinotMetricsFactory; + //config keys private static final String CONFIG_KEY_JMX_EXPORTER_PARENT_DIR = "jmxExporterConfigsParentDir"; - private static final String CONFIG_KEY_PINOT_METRICS_FACTORY = "pinotMetricsFactory"; private static final String CONFIG_KEY_CONTROLLER_CONFIG_FILE_NAME = "controllerConfigFileName"; private static final String CONFIG_KEY_SERVER_CONFIG_FILE_NAME = "serverConfigFileName"; private static final String CONFIG_KEY_BROKER_CONFIG_FILE_NAME = "brokerConfigFileName"; @@ -121,12 +114,12 @@ public void setupTest() PinotConfiguration pinotConfiguration = new PinotConfiguration(); - PinotMetricsFactory pinotMetricsFactory = getPinotMetricsFactory(); + _pinotMetricsFactory = getPinotMetricsFactory(); pinotConfiguration.setProperty(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, - pinotMetricsFactory.getClass().getCanonicalName()); + _pinotMetricsFactory.getClass().getCanonicalName()); PinotMetricUtils.init(pinotConfiguration); - pinotMetricsFactory.makePinotJmxReporter(pinotMetricsFactory.getPinotMetricsRegistry()).start(); + _pinotMetricsFactory.makePinotJmxReporter(_pinotMetricsFactory.getPinotMetricsRegistry()).start(); _httpClient = new HttpClient(); _httpServer = startExporter(getPinotComponent()); } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java similarity index 61% rename from pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerServerPrometheusMetricsTest.java rename to pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java index 9995e3e8c23e..ec3243235550 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/YammerServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java @@ -1,21 +1,68 @@ -package org.apache.pinot.common.metrics; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.apache.pinot.common.metrics.prometheus; import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; -import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.apache.pinot.common.metrics.ServerGauge; +import org.apache.pinot.common.metrics.ServerMeter; +import org.apache.pinot.common.metrics.ServerMetrics; +import org.apache.pinot.common.metrics.ServerTimer; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class YammerServerPrometheusMetricsTest extends ServerPrometheusMetricsTest { +public abstract class ServerPrometheusMetricsTest extends PinotPrometheusMetricsTest { + //all exported server metrics have this prefix + private static final String EXPORTED_METRIC_PREFIX = "pinot_server_"; - protected ServerMetrics _serverMetrics; + private static final List METERS_ACCEPTING_CLIENT_ID = + List.of(ServerMeter.REALTIME_ROWS_CONSUMED, ServerMeter.REALTIME_ROWS_SANITIZED, + ServerMeter.REALTIME_ROWS_FETCHED, ServerMeter.REALTIME_ROWS_FILTERED, + ServerMeter.INVALID_REALTIME_ROWS_DROPPED, ServerMeter.INCOMPLETE_REALTIME_ROWS_CONSUMED, + ServerMeter.STREAM_CONSUMER_CREATE_EXCEPTIONS, ServerMeter.ROWS_WITH_ERRORS); + + private static final List METERS_ACCEPTING_RAW_TABLE_NAMES = + List.of(ServerMeter.SEGMENT_UPLOAD_FAILURE, ServerMeter.SEGMENT_UPLOAD_SUCCESS, + ServerMeter.SEGMENT_UPLOAD_TIMEOUT); + + //gauges that accept clientId + private static final List GAUGES_ACCEPTING_CLIENT_ID = + List.of(ServerGauge.LLC_PARTITION_CONSUMING, ServerGauge.HIGHEST_STREAM_OFFSET_CONSUMED, + ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_CREATION_WAIT_TIME_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_INITIAL_CONSUMPTION_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_CATCHUP_DURATION_SECONDS, + ServerGauge.LAST_REALTIME_SEGMENT_COMPLETION_DURATION_SECONDS); + + private static final List GAUGES_ACCEPTING_PARTITION = + List.of(ServerGauge.UPSERT_VALID_DOC_ID_SNAPSHOT_COUNT, ServerGauge.UPSERT_PRIMARY_KEYS_IN_SNAPSHOT_COUNT, + ServerGauge.REALTIME_INGESTION_OFFSET_LAG, ServerGauge.REALTIME_INGESTION_DELAY_MS, + ServerGauge.UPSERT_PRIMARY_KEYS_COUNT, ServerGauge.END_TO_END_REALTIME_INGESTION_DELAY_MS, + ServerGauge.DEDUP_PRIMARY_KEYS_COUNT); + + private static final List GAUGES_ACCEPTING_RAW_TABLE_NAME = + List.of(ServerGauge.REALTIME_OFFHEAP_MEMORY_USED, ServerGauge.REALTIME_SEGMENT_NUM_PARTITIONS, + ServerGauge.LUCENE_INDEXING_DELAY_MS, ServerGauge.LUCENE_INDEXING_DELAY_DOCS); + + private ServerMetrics _serverMetrics; @BeforeClass public void setup() throws Exception { - _serverMetrics = new ServerMetrics(getPinotMetricsFactory().getPinotMetricsRegistry()); + _serverMetrics = new ServerMetrics(_pinotMetricsFactory.getPinotMetricsRegistry()); } @Test(dataProvider = "serverTimers") @@ -108,9 +155,24 @@ public void addMeterWithLabels(ServerMeter serverMeter, String labels) { _serverMetrics.addMeteredTableValue(labels, serverMeter, 4L); } + @DataProvider(name = "serverTimers") + public Object[] serverTimers() { + return ServerTimer.values(); // Provide all values of ServerTimer enum + } + + @DataProvider(name = "serverMeters") + public Object[] serverMeter() { + return ServerMeter.values(); // Provide all values of ServerTimer enum + } + + @DataProvider(name = "serverGauges") + public Object[] serverGauge() { + return ServerGauge.values(); // Provide all values of ServerTimer enum + } + @Override - protected PinotMetricsFactory getPinotMetricsFactory() { - return new YammerMetricsFactory(); + protected PinotComponent getPinotComponent() { + return PinotComponent.SERVER; } private boolean meterTrackingRealtimeExceptions(ServerMeter serverMeter) { diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java new file mode 100644 index 000000000000..58681def15d1 --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java @@ -0,0 +1,18 @@ +package org.apache.pinot.common.metrics.prometheus.dropwizard; + +import org.apache.pinot.common.metrics.prometheus.BrokerPrometheusMetricsTest; +import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.testng.annotations.Test; + + +/** + * Disabling tests as Pinot currently uses Yammer and these tests fail for for {@link DropwizardMetricsFactory} + */ +@Test(enabled = false) +public class DropwizardBrokerPrometheusMetricsTest extends BrokerPrometheusMetricsTest { + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new DropwizardMetricsFactory(); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java new file mode 100644 index 000000000000..48673514992a --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java @@ -0,0 +1,19 @@ +package org.apache.pinot.common.metrics.prometheus.dropwizard; + +import org.apache.pinot.common.metrics.prometheus.ControllerPrometheusMetricsTest; +import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.testng.annotations.Test; + + +/** + * Disabling tests as Pinot currently uses Yammer and these tests fail for for {@link DropwizardMetricsFactory} + */ +@Test(enabled = false) +public class DropwizardControllerPrometheusMetricsTest extends ControllerPrometheusMetricsTest { + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new DropwizardMetricsFactory(); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java new file mode 100644 index 000000000000..e03bb1dca147 --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java @@ -0,0 +1,19 @@ +package org.apache.pinot.common.metrics.prometheus.dropwizard; + +import org.apache.pinot.common.metrics.prometheus.MinionPrometheusMetricsTest; +import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.testng.annotations.Test; + + +/** + * Disabling tests as Pinot currently uses Yammer and these tests fail for for {@link DropwizardMetricsFactory} + */ +@Test(enabled = false) +public class DropwizardMinionPrometheusMetricsTest extends MinionPrometheusMetricsTest { + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new DropwizardMetricsFactory(); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java new file mode 100644 index 000000000000..9f704cf3ee8d --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java @@ -0,0 +1,20 @@ +package org.apache.pinot.common.metrics.prometheus.dropwizard; + +import org.apache.pinot.common.metrics.prometheus.ServerPrometheusMetricsTest; +import org.apache.pinot.plugin.metrics.dropwizard.DropwizardMetricsFactory; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; +import org.testng.annotations.Test; + + +/** + * Disabling tests as Pinot currently uses Yammer and these tests fail for for {@link DropwizardMetricsFactory} + */ +@Test(enabled = false) +public class DropwizardServerPrometheusMetricsTest extends ServerPrometheusMetricsTest { + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new YammerMetricsFactory(); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java new file mode 100644 index 000000000000..aab7ef54c523 --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java @@ -0,0 +1,14 @@ +package org.apache.pinot.common.metrics.prometheus.yammer; + +import org.apache.pinot.common.metrics.prometheus.BrokerPrometheusMetricsTest; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; + + +public class YammerBrokerPrometheusMetricsTest extends BrokerPrometheusMetricsTest { + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new YammerMetricsFactory(); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java new file mode 100644 index 000000000000..fd4a8775c56f --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java @@ -0,0 +1,14 @@ +package org.apache.pinot.common.metrics.prometheus.yammer; + +import org.apache.pinot.common.metrics.prometheus.ControllerPrometheusMetricsTest; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; + + +public class YammerControllerPrometheusMetricsTest extends ControllerPrometheusMetricsTest { + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new YammerMetricsFactory(); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java new file mode 100644 index 000000000000..28fa93d89058 --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java @@ -0,0 +1,14 @@ +package org.apache.pinot.common.metrics.prometheus.yammer; + +import org.apache.pinot.common.metrics.prometheus.MinionPrometheusMetricsTest; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; + + +public class YammerMinionPrometheusMetricsTest extends MinionPrometheusMetricsTest { + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new YammerMetricsFactory(); + } +} diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java new file mode 100644 index 000000000000..3bd56b0c62bc --- /dev/null +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java @@ -0,0 +1,14 @@ +package org.apache.pinot.common.metrics.prometheus.yammer; + +import org.apache.pinot.common.metrics.prometheus.ServerPrometheusMetricsTest; +import org.apache.pinot.plugin.metrics.yammer.YammerMetricsFactory; +import org.apache.pinot.spi.annotations.metrics.PinotMetricsFactory; + + +public class YammerServerPrometheusMetricsTest extends ServerPrometheusMetricsTest { + + @Override + protected PinotMetricsFactory getPinotMetricsFactory() { + return new YammerMetricsFactory(); + } +} From 069962cf7e12ac86414a142dc3e8a56076bd2542 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 14:28:28 +0530 Subject: [PATCH 37/45] Added license header --- ...DropwizardBrokerPrometheusMetricsTest.java | 19 +++++++++++++++++++ ...wizardControllerPrometheusMetricsTest.java | 19 +++++++++++++++++++ ...DropwizardMinionPrometheusMetricsTest.java | 19 +++++++++++++++++++ ...DropwizardServerPrometheusMetricsTest.java | 19 +++++++++++++++++++ 4 files changed, 76 insertions(+) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java index 58681def15d1..5d68211967c3 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.pinot.common.metrics.prometheus.dropwizard; import org.apache.pinot.common.metrics.prometheus.BrokerPrometheusMetricsTest; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java index 48673514992a..de73eca67bfe 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.pinot.common.metrics.prometheus.dropwizard; import org.apache.pinot.common.metrics.prometheus.ControllerPrometheusMetricsTest; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java index e03bb1dca147..50447b2e1d5c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.pinot.common.metrics.prometheus.dropwizard; import org.apache.pinot.common.metrics.prometheus.MinionPrometheusMetricsTest; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java index 9f704cf3ee8d..e132eff554d9 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.pinot.common.metrics.prometheus.dropwizard; import org.apache.pinot.common.metrics.prometheus.ServerPrometheusMetricsTest; From 52f23c2c780e64a4c6bb338f0d2acb440f68b06a Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 14:40:29 +0530 Subject: [PATCH 38/45] Added license header --- .../BrokerPrometheusMetricsTest.java | 27 +++++++++++-------- .../ControllerPrometheusMetricsTest.java | 27 +++++++++++-------- .../MinionPrometheusMetricsTest.java | 27 +++++++++++-------- .../PinotPrometheusMetricsTest.java | 27 +++++++++++-------- .../ServerPrometheusMetricsTest.java | 27 +++++++++++-------- .../YammerBrokerPrometheusMetricsTest.java | 19 +++++++++++++ ...YammerControllerPrometheusMetricsTest.java | 19 +++++++++++++ .../YammerMinionPrometheusMetricsTest.java | 19 +++++++++++++ .../YammerServerPrometheusMetricsTest.java | 19 +++++++++++++ 9 files changed, 156 insertions(+), 55 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java index 37ad6850227e..952aae06a03a 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java @@ -1,16 +1,21 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package org.apache.pinot.common.metrics.prometheus; import java.util.List; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java index c88e22f673c0..56766d8902ba 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java @@ -1,16 +1,21 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package org.apache.pinot.common.metrics.prometheus; import java.util.List; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java index e3d915cc03b7..4d249b06d158 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java @@ -1,16 +1,21 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package org.apache.pinot.common.metrics.prometheus; import java.util.List; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java index a00458fe25a9..4b9fef16ff35 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java @@ -1,16 +1,21 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package org.apache.pinot.common.metrics.prometheus; import com.fasterxml.jackson.databind.JsonNode; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java index ec3243235550..e9136d9a3cab 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java @@ -1,16 +1,21 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package org.apache.pinot.common.metrics.prometheus; import java.util.List; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java index aab7ef54c523..44884722336d 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.pinot.common.metrics.prometheus.yammer; import org.apache.pinot.common.metrics.prometheus.BrokerPrometheusMetricsTest; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java index fd4a8775c56f..f7465ac5fca1 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.pinot.common.metrics.prometheus.yammer; import org.apache.pinot.common.metrics.prometheus.ControllerPrometheusMetricsTest; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java index 28fa93d89058..dbb4b17dbb1b 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.pinot.common.metrics.prometheus.yammer; import org.apache.pinot.common.metrics.prometheus.MinionPrometheusMetricsTest; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java index 3bd56b0c62bc..af9c588b5f7c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.pinot.common.metrics.prometheus.yammer; import org.apache.pinot.common.metrics.prometheus.ServerPrometheusMetricsTest; From ee602ccff8b324a1cea9506f64b67a6d9dec11fd Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 15:06:58 +0530 Subject: [PATCH 39/45] Skip dropwizard tests in maven --- pinot-common/pom.xml | 52 +++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index b2abc1dc13bb..bdbf17e8ef70 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -19,7 +19,8 @@ under the License. --> - + 4.0.0 pinot @@ -52,6 +53,13 @@ false + + + **/DropwizardBrokerPrometheusMetricsTest.java + **/DropwizardServerPrometheusMetricsTest.java + **/DropwizardMinionPrometheusMetricsTest.java + **/DropwizardControllerPrometheusMetricsTest.java + @@ -305,33 +313,33 @@ generate-sources - - + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - + - + @@ -394,8 +402,8 @@ + sourceRoot="src/main/codegen/templates" + outputRoot="${project.build.directory}/generated-sources/"/> From f3064f57ab1d51fb3ecd4553276928cc004eb93b Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 15:25:00 +0530 Subject: [PATCH 40/45] Remove `pinotMetricsFactory` from test config --- pinot-common/src/test/resources/metrics/testConfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/pinot-common/src/test/resources/metrics/testConfig.json b/pinot-common/src/test/resources/metrics/testConfig.json index 5bdc0a6d66f6..3f9b6d572b7b 100644 --- a/pinot-common/src/test/resources/metrics/testConfig.json +++ b/pinot-common/src/test/resources/metrics/testConfig.json @@ -1,5 +1,4 @@ { - "pinotMetricsFactory": "YammerMetricsFactory", "jmxExporterConfigsParentDir": "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs", "controllerConfigFileName": "controller.yml", "serverConfigFileName": "server.yml", From 4264af37f3a2b43905689f6af5f3449eed4c9b2e Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 16:03:20 +0530 Subject: [PATCH 41/45] Remove testConfig.json --- .../BrokerPrometheusMetricsTest.java | 5 -- .../ControllerPrometheusMetricsTest.java | 5 -- .../MinionPrometheusMetricsTest.java | 5 -- .../PinotPrometheusMetricsTest.java | 87 ++++++------------- .../ServerPrometheusMetricsTest.java | 5 -- ...DropwizardBrokerPrometheusMetricsTest.java | 6 ++ ...wizardControllerPrometheusMetricsTest.java | 6 ++ ...DropwizardMinionPrometheusMetricsTest.java | 6 ++ ...DropwizardServerPrometheusMetricsTest.java | 6 ++ .../YammerBrokerPrometheusMetricsTest.java | 5 ++ ...YammerControllerPrometheusMetricsTest.java | 5 ++ .../YammerMinionPrometheusMetricsTest.java | 5 ++ .../YammerServerPrometheusMetricsTest.java | 5 ++ 13 files changed, 70 insertions(+), 81 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java index 952aae06a03a..399e5b400b19 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/BrokerPrometheusMetricsTest.java @@ -113,11 +113,6 @@ public void meterTest(BrokerMeter meter) { } } - @Override - protected PinotComponent getPinotComponent() { - return PinotComponent.BROKER; - } - @DataProvider(name = "brokerTimers") public Object[] brokerTimers() { return BrokerTimer.values(); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java index 56766d8902ba..7fcb76eae194 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ControllerPrometheusMetricsTest.java @@ -180,11 +180,6 @@ private void addMeterWithLabels(ControllerMeter meter, String labels) { _controllerMetrics.addMeteredTableValue(labels, meter, 1L); } - @Override - protected PinotComponent getPinotComponent() { - return PinotComponent.CONTROLLER; - } - @DataProvider(name = "controllerTimers") public Object[] controllerTimers() { return ControllerTimer.values(); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java index 4d249b06d158..35a766b0ca67 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/MinionPrometheusMetricsTest.java @@ -125,9 +125,4 @@ public Object[] minionMeters() { public Object[] minionGauges() { return MinionGauge.values(); } - - @Override - protected PinotComponent getPinotComponent() { - return PinotComponent.MINION; - } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java index 4b9fef16ff35..6b8424c738a7 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java @@ -1,24 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. */ package org.apache.pinot.common.metrics.prometheus; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Objects; import com.google.common.io.Resources; import io.prometheus.jmx.JmxCollector; @@ -35,7 +28,6 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -49,7 +41,6 @@ import org.apache.pinot.spi.config.table.TableType; import org.apache.pinot.spi.env.PinotConfiguration; import org.apache.pinot.spi.metrics.PinotMetricUtils; -import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -76,13 +67,6 @@ public abstract class PinotPrometheusMetricsTest { protected PinotMetricsFactory _pinotMetricsFactory; - //config keys - private static final String CONFIG_KEY_JMX_EXPORTER_PARENT_DIR = "jmxExporterConfigsParentDir"; - private static final String CONFIG_KEY_CONTROLLER_CONFIG_FILE_NAME = "controllerConfigFileName"; - private static final String CONFIG_KEY_SERVER_CONFIG_FILE_NAME = "serverConfigFileName"; - private static final String CONFIG_KEY_BROKER_CONFIG_FILE_NAME = "brokerConfigFileName"; - private static final String CONFIG_KEY_MINION_CONFIG_FILE_NAME = "minionConfigFileName"; - //each meter defined in code is exported with these measurements private static final List METER_TYPES = List.of("Count", "FiveMinuteRate", "MeanRate", "OneMinuteRate", "FifteenMinuteRate"); @@ -96,27 +80,10 @@ public abstract class PinotPrometheusMetricsTest { //each gauge defined in code is exported with these measurements private static final List GAUGE_TYPES = List.of("Value"); - private String _exporterConfigsParentDir; - - private final Map _pinotComponentToConfigFileMap = new HashMap<>(); - private HTTPServer _httpServer; @BeforeClass - public void setupTest() - throws Exception { - //read test configuration - JsonNode jsonNode = JsonUtils.DEFAULT_READER.readTree(loadResourceAsString("metrics/testConfig.json")); - _exporterConfigsParentDir = jsonNode.get(CONFIG_KEY_JMX_EXPORTER_PARENT_DIR).textValue(); - _pinotComponentToConfigFileMap.put(PinotComponent.CONTROLLER, - jsonNode.get(CONFIG_KEY_CONTROLLER_CONFIG_FILE_NAME).textValue()); - _pinotComponentToConfigFileMap.put(PinotComponent.SERVER, - jsonNode.get(CONFIG_KEY_SERVER_CONFIG_FILE_NAME).textValue()); - _pinotComponentToConfigFileMap.put(PinotComponent.BROKER, - jsonNode.get(CONFIG_KEY_BROKER_CONFIG_FILE_NAME).textValue()); - _pinotComponentToConfigFileMap.put(PinotComponent.MINION, - jsonNode.get(CONFIG_KEY_MINION_CONFIG_FILE_NAME).textValue()); - + public void setupTest() { PinotConfiguration pinotConfiguration = new PinotConfiguration(); _pinotMetricsFactory = getPinotMetricsFactory(); @@ -126,7 +93,7 @@ public void setupTest() _pinotMetricsFactory.makePinotJmxReporter(_pinotMetricsFactory.getPinotMetricsRegistry()).start(); _httpClient = new HttpClient(); - _httpServer = startExporter(getPinotComponent()); + _httpServer = startExporter(); } @AfterClass @@ -134,19 +101,18 @@ public void cleanup() { _httpServer.close(); } - /** Pinot currently uses the JMX->Prom exporter to export metrics to Prometheus. Normally, this runs as an agent in - * the JVM. In this case however, we've got tests using four different config files (server.yml, broker.yml, - * controller.yml and minion.yml). Loading the same agent in the same JVM multiple times isn't allowed, we are - * copying the agent's code to some degree and starting up the HTTP servers manually. - * For impl, see: + /** + * Pinot currently uses the JMX->Prom exporter to export metrics to Prometheus. Normally, this runs as an agent in the + * JVM. In this case however, we've got tests using four different config files (server.yml, broker.yml, + * controller.yml and minion.yml). Loading the same agent in the same JVM multiple times isn't allowed, we are copying + * the agent's code to some degree and starting up the HTTP servers manually. For impl, see: * ... * /jmx_prometheus_javaagent/src/main/java/io/prometheus/jmx/JavaAgent.java#L48 - * @param pinotComponent the Pinot component to start the server for + * * @return the corresponding HTTP server on a random unoccupied port */ - protected HTTPServer startExporter(PinotComponent pinotComponent) { - String args = - String.format("%s:%s/%s", 0, _exporterConfigsParentDir, _pinotComponentToConfigFileMap.get(pinotComponent)); + protected HTTPServer startExporter() { + String args = String.format("%s:%s", 0, getConfigParentDir()); try { JMXExporterConfig config = parseExporterConfig(args, "0.0.0.0"); CollectorRegistry registry = new CollectorRegistry(); @@ -265,8 +231,6 @@ protected List parseExportedPromMetrics(String response) } } - protected abstract PinotComponent getPinotComponent(); - protected SimpleHttpResponse getExportedPromMetrics() { try { return _httpClient.sendGetRequest(new URI("http://localhost:" + _httpServer.getPort() + "/metrics")); @@ -277,6 +241,8 @@ protected SimpleHttpResponse getExportedPromMetrics() { protected abstract PinotMetricsFactory getPinotMetricsFactory(); + protected abstract String getConfigParentDir(); + /* Implementation copied from: https://github .com/prometheus/jmx_exporter/blob/a3b9443564ff5a78c25fd6566396fda2b7cbf216/jmx_prometheus_javaagent/src/main/java @@ -327,10 +293,6 @@ private String loadResourceAsString(String resourceFileName) { } } - public enum PinotComponent { - SERVER, BROKER, CONTROLLER, MINION - } - public static class ExportedLabels { public static final List TABLENAME = List.of(TABLE, ExportedLabelValues.TABLENAME); @@ -409,6 +371,7 @@ private PromMetric(String metricName, Map labels) { /** * Create an instance of {@link PromMetric} from an exported Prometheus metric + * * @param exportedMetric the exported Prom metric (name + labels) * @return the corresponding {@link PromMetric} */ @@ -429,6 +392,7 @@ public static PromMetric fromExportedMetric(String exportedMetric) { /** * Creates an instance of {@link PromMetric} with a name and an empty label list + * * @param metricName the metric name * @return the corresponding PromMetric */ @@ -438,8 +402,9 @@ public static PromMetric withName(String metricName) { /** * Creates an instance of {@link PromMetric} with a name and an label list + * * @param metricName the metric name - * @param labels the labels + * @param labels the labels * @return the corresponding PromMetric */ public static PromMetric withNameAndLabels(String metricName, List labels) { diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java index e9136d9a3cab..366d538ec1fa 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/ServerPrometheusMetricsTest.java @@ -175,11 +175,6 @@ public Object[] serverGauge() { return ServerGauge.values(); // Provide all values of ServerTimer enum } - @Override - protected PinotComponent getPinotComponent() { - return PinotComponent.SERVER; - } - private boolean meterTrackingRealtimeExceptions(ServerMeter serverMeter) { return serverMeter == ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS || serverMeter == ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java index 5d68211967c3..e2a722cf436f 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java @@ -34,4 +34,10 @@ public class DropwizardBrokerPrometheusMetricsTest extends BrokerPrometheusMetri protected PinotMetricsFactory getPinotMetricsFactory() { return new DropwizardMetricsFactory(); } + + @Override + protected String getConfigParentDir() { + //todo: return the correct dir once this test is enabled + return null; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java index de73eca67bfe..b3e5ac1e458c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java @@ -35,4 +35,10 @@ public class DropwizardControllerPrometheusMetricsTest extends ControllerPrometh protected PinotMetricsFactory getPinotMetricsFactory() { return new DropwizardMetricsFactory(); } + + @Override + protected String getConfigParentDir() { + //todo: return the correct dir once this test is enabled + return null; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java index 50447b2e1d5c..4132da893c2c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java @@ -35,4 +35,10 @@ public class DropwizardMinionPrometheusMetricsTest extends MinionPrometheusMetri protected PinotMetricsFactory getPinotMetricsFactory() { return new DropwizardMetricsFactory(); } + + @Override + protected String getConfigParentDir() { + //todo: return the correct dir once this test is enabled + return null; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java index e132eff554d9..f7d1b28d9444 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java @@ -36,4 +36,10 @@ public class DropwizardServerPrometheusMetricsTest extends ServerPrometheusMetri protected PinotMetricsFactory getPinotMetricsFactory() { return new YammerMetricsFactory(); } + + @Override + protected String getConfigParentDir() { + //todo: return the correct dir once this test is enabled + return null; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java index 44884722336d..90d7b398015f 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java @@ -30,4 +30,9 @@ public class YammerBrokerPrometheusMetricsTest extends BrokerPrometheusMetricsTe protected PinotMetricsFactory getPinotMetricsFactory() { return new YammerMetricsFactory(); } + + @Override + protected String getConfigParentDir() { + return "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml"; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java index f7465ac5fca1..3463808a7a3f 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java @@ -30,4 +30,9 @@ public class YammerControllerPrometheusMetricsTest extends ControllerPrometheusM protected PinotMetricsFactory getPinotMetricsFactory() { return new YammerMetricsFactory(); } + + @Override + protected String getConfigParentDir() { + return "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml"; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java index dbb4b17dbb1b..e40d7d2d6d25 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java @@ -30,4 +30,9 @@ public class YammerMinionPrometheusMetricsTest extends MinionPrometheusMetricsTe protected PinotMetricsFactory getPinotMetricsFactory() { return new YammerMetricsFactory(); } + + @Override + protected String getConfigParentDir() { + return "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml"; + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java index af9c588b5f7c..7953d2452a50 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java @@ -30,4 +30,9 @@ public class YammerServerPrometheusMetricsTest extends ServerPrometheusMetricsTe protected PinotMetricsFactory getPinotMetricsFactory() { return new YammerMetricsFactory(); } + + @Override + protected String getConfigParentDir() { + return "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml"; + } } From a9354ebfe0526ba22648689710b01d3126bd2f80 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 16:05:19 +0530 Subject: [PATCH 42/45] Remove inadvertent changes --- pinot-common/pom.xml | 46 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index bdbf17e8ef70..af2001a9e14c 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -19,8 +19,7 @@ under the License. --> - + 4.0.0 pinot @@ -313,33 +312,33 @@ generate-sources - - + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - + - + @@ -351,6 +350,7 @@ + build-shaded-jar @@ -402,8 +402,8 @@ + sourceRoot="src/main/codegen/templates" + outputRoot="${project.build.directory}/generated-sources/"/> From 46899f3d0bf22944f81eea742a7048c37ebeb400 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 16:06:00 +0530 Subject: [PATCH 43/45] Remove testConfig.json --- pinot-common/src/test/resources/metrics/testConfig.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 pinot-common/src/test/resources/metrics/testConfig.json diff --git a/pinot-common/src/test/resources/metrics/testConfig.json b/pinot-common/src/test/resources/metrics/testConfig.json deleted file mode 100644 index 3f9b6d572b7b..000000000000 --- a/pinot-common/src/test/resources/metrics/testConfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "jmxExporterConfigsParentDir": "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs", - "controllerConfigFileName": "controller.yml", - "serverConfigFileName": "server.yml", - "brokerConfigFileName": "broker.yml", - "minionConfigFileName": "minion.yml" -} \ No newline at end of file From b25b665f9700e2d619ed720ccda041af08d3737f Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 16:07:39 +0530 Subject: [PATCH 44/45] License --- .../PinotPrometheusMetricsTest.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java index 6b8424c738a7..b16de2683a1d 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java @@ -1,14 +1,20 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.pinot.common.metrics.prometheus; From 2fcd38216213e2c0b424782c61954254d6e20c16 Mon Sep 17 00:00:00 2001 From: prashantpandey Date: Tue, 22 Oct 2024 16:10:25 +0530 Subject: [PATCH 45/45] Rename method from getConfigParentDir -> getConfigFile --- .../common/metrics/prometheus/PinotPrometheusMetricsTest.java | 4 ++-- .../dropwizard/DropwizardBrokerPrometheusMetricsTest.java | 2 +- .../dropwizard/DropwizardControllerPrometheusMetricsTest.java | 2 +- .../dropwizard/DropwizardMinionPrometheusMetricsTest.java | 2 +- .../dropwizard/DropwizardServerPrometheusMetricsTest.java | 2 +- .../prometheus/yammer/YammerBrokerPrometheusMetricsTest.java | 2 +- .../yammer/YammerControllerPrometheusMetricsTest.java | 2 +- .../prometheus/yammer/YammerMinionPrometheusMetricsTest.java | 2 +- .../prometheus/yammer/YammerServerPrometheusMetricsTest.java | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java index b16de2683a1d..a3f21ad91d9d 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java @@ -118,7 +118,7 @@ public void cleanup() { * @return the corresponding HTTP server on a random unoccupied port */ protected HTTPServer startExporter() { - String args = String.format("%s:%s", 0, getConfigParentDir()); + String args = String.format("%s:%s", 0, getConfigFile()); try { JMXExporterConfig config = parseExporterConfig(args, "0.0.0.0"); CollectorRegistry registry = new CollectorRegistry(); @@ -247,7 +247,7 @@ protected SimpleHttpResponse getExportedPromMetrics() { protected abstract PinotMetricsFactory getPinotMetricsFactory(); - protected abstract String getConfigParentDir(); + protected abstract String getConfigFile(); /* Implementation copied from: https://github diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java index e2a722cf436f..8d1abd44b286 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardBrokerPrometheusMetricsTest.java @@ -36,7 +36,7 @@ protected PinotMetricsFactory getPinotMetricsFactory() { } @Override - protected String getConfigParentDir() { + protected String getConfigFile() { //todo: return the correct dir once this test is enabled return null; } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java index b3e5ac1e458c..005d83466b91 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardControllerPrometheusMetricsTest.java @@ -37,7 +37,7 @@ protected PinotMetricsFactory getPinotMetricsFactory() { } @Override - protected String getConfigParentDir() { + protected String getConfigFile() { //todo: return the correct dir once this test is enabled return null; } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java index 4132da893c2c..08183428a8eb 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardMinionPrometheusMetricsTest.java @@ -37,7 +37,7 @@ protected PinotMetricsFactory getPinotMetricsFactory() { } @Override - protected String getConfigParentDir() { + protected String getConfigFile() { //todo: return the correct dir once this test is enabled return null; } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java index f7d1b28d9444..9fc5bf4b9690 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/dropwizard/DropwizardServerPrometheusMetricsTest.java @@ -38,7 +38,7 @@ protected PinotMetricsFactory getPinotMetricsFactory() { } @Override - protected String getConfigParentDir() { + protected String getConfigFile() { //todo: return the correct dir once this test is enabled return null; } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java index 90d7b398015f..d58a3fdd3ce0 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerBrokerPrometheusMetricsTest.java @@ -32,7 +32,7 @@ protected PinotMetricsFactory getPinotMetricsFactory() { } @Override - protected String getConfigParentDir() { + protected String getConfigFile() { return "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml"; } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java index 3463808a7a3f..e7f7f11dc857 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerControllerPrometheusMetricsTest.java @@ -32,7 +32,7 @@ protected PinotMetricsFactory getPinotMetricsFactory() { } @Override - protected String getConfigParentDir() { + protected String getConfigFile() { return "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml"; } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java index e40d7d2d6d25..520c809e7ade 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerMinionPrometheusMetricsTest.java @@ -32,7 +32,7 @@ protected PinotMetricsFactory getPinotMetricsFactory() { } @Override - protected String getConfigParentDir() { + protected String getConfigFile() { return "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml"; } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java index 7953d2452a50..e32228716d59 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/yammer/YammerServerPrometheusMetricsTest.java @@ -32,7 +32,7 @@ protected PinotMetricsFactory getPinotMetricsFactory() { } @Override - protected String getConfigParentDir() { + protected String getConfigFile() { return "../docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml"; } }