Skip to content

Commit

Permalink
Merge pull request #817 from commercetools/telemetry-error-capture
Browse files Browse the repository at this point in the history 
[DEVX-468] Capture errors in telemetry middleware
  • Loading branch information
@jenschude
jenschude authored Feb 4, 2025
2 parents ccf877f + 79578ef commit 65e52db
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 19 deletions.
22 changes: 16 additions & 6 deletions ...itoring-datadog/src/main/java/com/commercetools/monitoring/datadog/DatadogMiddleware.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import com.datadog.api.client.ApiException;
import com.datadog.api.client.v2.api.MetricsApi;

import io.vrap.rmf.base.client.ApiHttpException;
import io.vrap.rmf.base.client.ApiHttpRequest;
import io.vrap.rmf.base.client.ApiHttpResponse;
import io.vrap.rmf.base.client.http.TelemetryMiddleware;
Expand Down Expand Up @@ -51,14 +52,23 @@ public DatadogMiddleware(final MetricsApi apiInstance) {
public CompletableFuture<ApiHttpResponse<byte[]>> invoke(ApiHttpRequest request,
Function<ApiHttpRequest, CompletableFuture<ApiHttpResponse<byte[]>>> next) {
final Instant start = Instant.now();
return next.apply(request).thenApply(response -> {
return next.apply(request).handle((response, throwable) -> {
final int statusCode;
if (response != null) {
statusCode = response.getStatusCode();
}
else if (throwable instanceof ApiHttpException && ((ApiHttpException) throwable).getResponse() != null) {
statusCode = ((ApiHttpException) throwable).getResponse().getStatusCode();
}
else {
statusCode = 0;
}
try {
submitClientDurationMetric(request, apiInstance, Duration.between(start, Instant.now()).toMillis(),
response);
submitTotalRequestsMetric(request, apiInstance, response);

if (response.getStatusCode() >= 400) {
submitErrorRequestsMetric(request, apiInstance, response);
statusCode);
submitTotalRequestsMetric(request, apiInstance, statusCode);
if (statusCode >= 400 || throwable != null) {
submitErrorRequestsMetric(request, apiInstance, statusCode);
}
}
catch (ApiException e) {
Expand Down
29 changes: 25 additions & 4 deletions ...s-monitoring-datadog/src/main/java/com/commercetools/monitoring/datadog/DatadogUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,29 +28,50 @@ public class DatadogUtils {

protected static void submitClientDurationMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
final double durationInMillis, final ApiHttpResponse<byte[]> response) throws ApiException {
submitClientDurationMetric(request, apiInstance, durationInMillis, response.getStatusCode());
}

protected static void submitClientDurationMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
final double durationInMillis, final int statusCode) throws ApiException {
final String name = PREFIX + "." + CLIENT_DURATION;
final MetricIntakeType type = MetricIntakeType.UNSPECIFIED;
submitMetricWithHttpTags(name, durationInMillis, type, "ms", request, apiInstance, response);
submitMetricWithHttpTags(name, durationInMillis, type, "ms", request, apiInstance, statusCode);
}

protected static void submitErrorRequestsMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
final ApiHttpResponse<byte[]> response) throws ApiException {
submitErrorRequestsMetric(request, apiInstance, response.getStatusCode());
}

protected static void submitErrorRequestsMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
final int statusCode) throws ApiException {
final String name = PREFIX + "." + CLIENT_REQUEST_ERROR;
final MetricIntakeType count = MetricIntakeType.COUNT;
submitMetricWithHttpTags(name, 1.0, count, "count", request, apiInstance, response);
submitMetricWithHttpTags(name, 1.0, count, "count", request, apiInstance, statusCode);
}

protected static void submitTotalRequestsMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
final ApiHttpResponse<byte[]> response) throws ApiException {
submitTotalRequestsMetric(request, apiInstance, response.getStatusCode());
}

protected static void submitTotalRequestsMetric(final ApiHttpRequest request, final MetricsApi apiInstance,
final int statusCode) throws ApiException {
final String name = PREFIX + "." + CLIENT_REQUEST_TOTAL;
final MetricIntakeType count = MetricIntakeType.COUNT;
submitMetricWithHttpTags(name, 1.0, count, "count", request, apiInstance, response);
submitMetricWithHttpTags(name, 1.0, count, "count", request, apiInstance, statusCode);
}

private static void submitMetricWithHttpTags(final String name, final double value, final MetricIntakeType type,
final String unit, final ApiHttpRequest request, final MetricsApi apiInstance,
final ApiHttpResponse<byte[]> response) throws ApiException {
final List<String> tags = Arrays.asList(format("%s:%s", HTTP_RESPONSE_STATUS_CODE, response.getStatusCode()),
submitMetricWithHttpTags(name, value, type, unit, request, apiInstance, response.getStatusCode());
}

private static void submitMetricWithHttpTags(final String name, final double value, final MetricIntakeType type,
final String unit, final ApiHttpRequest request, final MetricsApi apiInstance, final int statusCode)
throws ApiException {
final List<String> tags = Arrays.asList(format("%s:%s", HTTP_RESPONSE_STATUS_CODE, statusCode),
format("%s:%s", HTTP_REQUEST_METHOD, request.getMethod().name()),
format("%s:%s", SERVER_ADDRESS, request.getUri().getHost()));
if (request.getUri().getPort() > 0) {
Expand Down
17 changes: 14 additions & 3 deletions ...-datadog/src/main/java/com/commercetools/monitoring/datadog/statsd/DatadogMiddleware.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

import com.timgroup.statsd.StatsDClient;

import io.vrap.rmf.base.client.ApiHttpException;
import io.vrap.rmf.base.client.ApiHttpRequest;
import io.vrap.rmf.base.client.ApiHttpResponse;
import io.vrap.rmf.base.client.http.TelemetryMiddleware;
Expand Down Expand Up @@ -49,9 +50,19 @@ public CompletableFuture<ApiHttpResponse<byte[]>> invoke(ApiHttpRequest request,
Function<ApiHttpRequest, CompletableFuture<ApiHttpResponse<byte[]>>> next) {
final Instant start = Instant.now();

return next.apply(request).thenApply(response -> {
return next.apply(request).handle((response, throwable) -> {
final List<String> tags = new ArrayList<>(4);
tags.add(format("%s:%s", HTTP_RESPONSE_STATUS_CODE, response.getStatusCode()));
final int statusCode;
if (response != null) {
statusCode = response.getStatusCode();
}
else if (throwable instanceof ApiHttpException && ((ApiHttpException) throwable).getResponse() != null) {
statusCode = ((ApiHttpException) throwable).getResponse().getStatusCode();
}
else {
statusCode = 0;
}
tags.add(format("%s:%s", HTTP_RESPONSE_STATUS_CODE, statusCode));
tags.add(format("%s:%s", HTTP_REQUEST_METHOD, request.getMethod().name()));
tags.add(format("%s:%s", SERVER_ADDRESS, request.getUri().getHost()));
if (request.getUri().getPort() > 0) {
Expand All @@ -62,7 +73,7 @@ public CompletableFuture<ApiHttpResponse<byte[]>> invoke(ApiHttpRequest request,
Duration.between(start, Instant.now()).toMillis(), tags.toArray(new String[0]));

this.statsDClient.incrementCounter(PREFIX + "." + CLIENT_REQUEST_TOTAL, tags.toArray(new String[0]));
if (response.getStatusCode() >= 400) {
if (statusCode >= 400 || throwable != null) {
this.statsDClient.incrementCounter(PREFIX + "." + CLIENT_REQUEST_ERROR, tags.toArray(new String[0]));
}
return response;
Expand Down
24 changes: 21 additions & 3 deletions ...elic/src/main/java/com/commercetools/monitoring/newrelic/NewRelicTelemetryMiddleware.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import com.newrelic.api.agent.*;

import io.vrap.rmf.base.client.ApiHttpException;
import io.vrap.rmf.base.client.ApiHttpRequest;
import io.vrap.rmf.base.client.ApiHttpResponse;
import io.vrap.rmf.base.client.ContextApiHttpClientImpl;
Expand Down Expand Up @@ -61,21 +62,38 @@ public CompletableFuture<ApiHttpResponse<byte[]>> invoke(ApiHttpRequest request,
Optional<Token> token = context.map(NewRelicContext::getTransaction).map(Transaction::getToken);
Optional<Segment> segment = context.map(c -> c.getTransaction()
.startSegment("commercetools", request.getMethod() + " " + request.getUri().getPath()));
return next.apply(request).thenApply(response -> {
return next.apply(request).handle((response, throwable) -> {
token.ifPresent(Token::linkAndExpire);

final int statusCode;
final String message;
if (response != null) {
statusCode = response.getStatusCode();
message = response.getMessage();
}
else if (throwable instanceof ApiHttpException && ((ApiHttpException) throwable).getResponse() != null) {
ApiHttpResponse<byte[]> errorResponse = ((ApiHttpException) throwable).getResponse();
statusCode = errorResponse.getStatusCode();
message = errorResponse.getMessage();
}
else {
statusCode = 0;
message = throwable.getMessage();
}

segment.ifPresent(s -> s.reportAsExternal(HttpParameters.library("commercetools-sdk-java-v2")
.uri(request.getUri())
.procedure(request.getMethod().name())
.noInboundHeaders()
.status(response.getStatusCode(), response.getMessage())
.status(statusCode, message)
.build()));
segment.ifPresent(Segment::end);

NewRelic.incrementCounter(PREFIX + CLIENT_REQUEST_TOTAL);
NewRelic.recordResponseTimeMetric(PREFIX + CLIENT_DURATION,
Duration.between(start, Instant.now()).toMillis());

if (response.getStatusCode() >= 400) {
if (statusCode >= 400 || throwable != null) {
NewRelic.incrementCounter(PREFIX + CLIENT_REQUEST_ERROR);
}
return response;
Expand Down
17 changes: 14 additions & 3 deletions ...try/src/main/java/com/commercetools/monitoring/opentelemetry/OpenTelemetryMiddleware.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import io.opentelemetry.api.metrics.LongCounter;
import io.opentelemetry.api.metrics.LongHistogram;
import io.opentelemetry.api.metrics.Meter;
import io.vrap.rmf.base.client.ApiHttpException;
import io.vrap.rmf.base.client.ApiHttpRequest;
import io.vrap.rmf.base.client.ApiHttpResponse;
import io.vrap.rmf.base.client.http.TelemetryMiddleware;
Expand Down Expand Up @@ -67,9 +68,19 @@ public OpenTelemetryMiddleware(final OpenTelemetry otel, final boolean enableHis
public CompletableFuture<ApiHttpResponse<byte[]>> invoke(ApiHttpRequest request,
Function<ApiHttpRequest, CompletableFuture<ApiHttpResponse<byte[]>>> next) {
Instant start = Instant.now();
return next.apply(request).thenApply(response -> {
return next.apply(request).handle((response, throwable) -> {
final int statusCode;
if (response != null) {
statusCode = response.getStatusCode();
}
else if (throwable instanceof ApiHttpException && ((ApiHttpException) throwable).getResponse() != null) {
statusCode = ((ApiHttpException) throwable).getResponse().getStatusCode();
}
else {
statusCode = 0;
}
AttributesBuilder builder = Attributes.builder()
.put(OpenTelemetryInfo.HTTP_RESPONSE_STATUS_CODE, response.getStatusCode())
.put(OpenTelemetryInfo.HTTP_RESPONSE_STATUS_CODE, statusCode)
.put(OpenTelemetryInfo.HTTP_REQUEST_METHOD, request.getMethod().name())
.put(OpenTelemetryInfo.SERVER_ADDRESS, request.getUri().getHost());
if (request.getUri().getPort() > 0) {
Expand All @@ -79,7 +90,7 @@ public CompletableFuture<ApiHttpResponse<byte[]>> invoke(ApiHttpRequest request,
Optional.ofNullable(histogram)
.ifPresent(h -> h.record(Duration.between(start, Instant.now()).toMillis(), attributes));
requestCounter.add(1, attributes);
if (response.getStatusCode() >= 400) {
if (statusCode >= 400 || throwable != null) {
errorCounter.add(1, attributes);
}
return response;
Expand Down

0 comments on commit 65e52db

Please sign in to comment.