diff --git a/baseapp/pom.xml b/baseapp/pom.xml index c6be2d49..3662cb1f 100644 --- a/baseapp/pom.xml +++ b/baseapp/pom.xml @@ -17,7 +17,7 @@ 1.8 UTF-8 UTF-8 - 1.3.29 + 1.3.7 4.1.0 0.9.10 diff --git a/gateway-ha/gateway-ha-config.yml b/gateway-ha/gateway-ha-config.yml index 70f1ee92..e8cd62e0 100644 --- a/gateway-ha/gateway-ha-config.yml +++ b/gateway-ha/gateway-ha-config.yml @@ -6,6 +6,10 @@ requestRouter: port: 8080 name: prestoRouter historySize: 1000 + requestHeaderSize: 2048000 + responseHeaderSize: 2048000 + requestBufferSize: 2048000 + responseBufferSize: 2048000 dataStore: jdbcUrl: jdbc:mysql://127.0.0.1:3306/prestogateway diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java index c16dabd0..19395718 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java @@ -1,6 +1,10 @@ package com.lyft.data.gateway.ha.handler; +import static com.codahale.metrics.MetricRegistry.name; + +import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.io.CharStreams; @@ -28,6 +32,7 @@ import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.util.Callback; + @Slf4j public class QueryIdCachingProxyHandler extends ProxyHandler { public static final String PROXY_TARGET_HEADER = "proxytarget"; @@ -57,13 +62,19 @@ public class QueryIdCachingProxyHandler extends ProxyHandler { private final Meter requestMeter; private final int serverApplicationPort; + private final Counter errorCounter4xx; + private final Counter errorCounter5xx; + public QueryIdCachingProxyHandler( QueryHistoryManager queryHistoryManager, RoutingManager routingManager, RoutingGroupSelector routingGroupSelector, int serverApplicationPort, - Meter requestMeter) { + Meter requestMeter, + MetricRegistry metrics) { this.requestMeter = requestMeter; + this.errorCounter4xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "4xx")); + this.errorCounter5xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "5xx")); this.routingManager = routingManager; this.routingGroupSelector = routingGroupSelector; this.queryHistoryManager = queryHistoryManager; @@ -270,6 +281,11 @@ protected void postConnectionHook( } catch (Exception e) { log.error("Error in proxying falling back to super call", e); } + if (response.getStatus() >= 500) { + errorCounter5xx.inc(); + } else if (response.getStatus() >= 400) { + errorCounter4xx.inc(); + } super.postConnectionHook(request, response, buffer, offset, length, callback); } diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java index edadc249..b2b37c0f 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java @@ -1,6 +1,7 @@ package com.lyft.data.gateway.ha.module; import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; import com.google.inject.Provides; import com.google.inject.Singleton; import com.lyft.data.baseapp.AppModule; @@ -23,6 +24,7 @@ import com.lyft.data.proxyserver.ProxyServerConfiguration; import io.dropwizard.setup.Environment; + public class HaGatewayProviderModule extends AppModule { private final ResourceGroupsManager resourceGroupsManager; @@ -42,9 +44,9 @@ public HaGatewayProviderModule(HaGatewayConfiguration configuration, Environment } protected ProxyHandler getProxyHandler() { + MetricRegistry metrics = getEnvironment().metrics(); Meter requestMeter = - getEnvironment() - .metrics() + metrics .meter(getConfiguration().getRequestRouter().getName() + ".requests"); // By default, use routing group header to route @@ -61,7 +63,8 @@ protected ProxyHandler getProxyHandler() { getRoutingManager(), routingGroupSelector, getApplicationPort(), - requestMeter); + requestMeter, + metrics); } @Provides @@ -81,6 +84,7 @@ public ProxyServer provideGateway() { routerProxyConfig.setKeystorePass(routerConfiguration.getKeystorePass()); routerProxyConfig.setForwardKeystore(routerConfiguration.isForwardKeystore()); routerProxyConfig.setPreserveHost("false"); + ProxyHandler proxyHandler = getProxyHandler(); gateway = new ProxyServer(routerProxyConfig, proxyHandler); } diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index d14b6999..b679c39b 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -18,10 +18,16 @@ 1.8 UTF-8 UTF-8 + 1.3.7 + + io.dropwizard + dropwizard-core + ${dropwizard.version} + org.eclipse.jetty jetty-server diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index c82e8a4e..b34153e3 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -35,6 +35,15 @@ protected String rewriteTarget(HttpServletRequest request) { */ public void preConnectionHook(HttpServletRequest request, Request proxyRequest) { // you may override it. + + // [sev-16337] with a 10% probably, log the request headers for debugging + if (Math.random() < 0.10) { + log.debug("(preConnectionHook) Request URL: {} , request URI {} , servlet path {} ," + + "toString {}, getContentLength {}, getRequestHeaderSize {}, requestHeaders {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength(), getRequestHeaderSize(request), + errorLogHeaders(request)); + } } /** @@ -60,16 +69,74 @@ protected void postConnectionHook( request.getRequestURL()); } response.getOutputStream().write(buffer, offset, length); + // [sev-16337] with a 10% probably, log the request and response headers + // and size for debugging + if (Math.random() < 0.10) { + log.debug("(postConnectionHook) Request URL: {} , request URI {} , servlet path {} , " + + "toString {}, getContentLength {}, getRequestHeaderSize {}, " + + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength(), getRequestHeaderSize(request), + getResponseHeaderSize(response), errorLogHeaders(request), + errorLogHeaders(response)); + } callback.succeeded(); } catch (Throwable var9) { - log.error("Exception occurred while processing request URL: {} , request URI {} ," - + " servlet path {} , toString {}", request.getRequestURL(), - request.getRequestURI(), request.getServletPath(), request.toString(), var9); + log.error("(postConnectionHook) Exception occurred while processing request URL: {} , " + + "request URI {} , servlet path {} , toString {}, getContentLength {}, " + + "getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, " + + "responseHeaders {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength(), getRequestHeaderSize(request), + getResponseHeaderSize(response), errorLogHeaders(request), + errorLogHeaders(response), var9); + callback.failed(var9); } } + protected int getRequestHeaderSize(HttpServletRequest request) { + int headerSize = 0; + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + String headerValue = request.getHeader(headerName); + headerSize += headerName.length() + headerValue.length(); + } + return headerSize; + } + + private int getResponseHeaderSize(HttpServletResponse response) { + int headerSize = 0; + for (String headerName : response.getHeaderNames()) { + String headerValue = response.getHeader(headerName); + headerSize += headerName.length() + headerValue.length(); + } + return headerSize; + } + + protected String errorLogHeaders(HttpServletRequest request) { + StringBuilder sb = new StringBuilder("------- error HttpServletRequest headers---------"); + Enumeration headers = request.getHeaderNames(); + while (headers.hasMoreElements()) { + String header = headers.nextElement(); + sb.append(header + "->" + request.getHeader(header) + "\n"); + } + + return sb.toString(); + } + + protected String errorLogHeaders(HttpServletResponse response) { + StringBuilder sb = new StringBuilder("------- error HttpServletResponse headers---------"); + Collection headers = response.getHeaderNames(); + for (String header : headers) { + sb.append(header + "->" + response.getHeader(header) + "\n"); + } + + return sb.toString(); + } + protected void debugLogHeaders(HttpServletRequest request) { if (log.isDebugEnabled()) { log.debug("-------HttpServletRequest headers---------"); diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 89e8bd1b..e2bfc96d 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -3,6 +3,8 @@ import java.io.Closeable; import java.io.File; import java.util.EnumSet; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.servlet.DispatcherType; @@ -10,16 +12,23 @@ import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.ResponseHandler; import org.apache.http.util.TextUtils; +import org.eclipse.jetty.http.HttpParser; import org.eclipse.jetty.http.HttpScheme; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.proxy.ConnectHandler; +import org.eclipse.jetty.server.CustomRequestLog; +import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.handler.RequestLogHandler; +import org.eclipse.jetty.server.handler.StatisticsHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -29,6 +38,9 @@ public class ProxyServer implements Closeable { private final Server server; private final ProxyServletImpl proxy; private final ProxyHandler proxyHandler; + private final ScheduledExecutorService scheduler = + Executors.newScheduledThreadPool(1); + private ServletContextHandler context; public ProxyServer(ProxyServerConfiguration config, ProxyHandler proxyHandler) { @@ -59,6 +71,7 @@ private void setupContext(ProxyServerConfiguration config) { sslContextFactory.setStopTimeout(TimeUnit.SECONDS.toMillis(15)); sslContextFactory.setSslSessionTimeout((int) TimeUnit.SECONDS.toMillis(15)); + if (!TextUtils.isBlank(keystorePath)) { sslContextFactory.setKeyStorePath(keystoreFile.getAbsolutePath()); sslContextFactory.setKeyStorePassword(keystorePass); @@ -68,12 +81,16 @@ private void setupContext(ProxyServerConfiguration config) { HttpConfiguration httpsConfig = new HttpConfiguration(); httpsConfig.setSecureScheme(HttpScheme.HTTPS.asString()); httpsConfig.setSecurePort(config.getLocalPort()); + httpsConfig.setIdleTimeout(150000); httpsConfig.setOutputBufferSize(32768); + httpsConfig.setRequestHeaderSize(2048000); + httpsConfig.setResponseHeaderSize(2048000); SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); httpsConfig.addCustomizer(src); + httpsConfig.addCustomizer(new org.eclipse.jetty.server.ForwardedRequestCustomizer()); connector = new ServerConnector( server, @@ -86,26 +103,41 @@ private void setupContext(ProxyServerConfiguration config) { connector.setPort(config.getLocalPort()); connector.setName(config.getName()); connector.setAccepting(true); + connector.setIdleTimeout(150000L); + connector.setAcceptQueueSize(1024); this.server.addConnector(connector); // Setup proxy handler to handle CONNECT methods ConnectHandler proxyConnectHandler = new ConnectHandler(); - this.server.setHandler(proxyConnectHandler); + + HandlerCollection handlers = new HandlerCollection(); + + RequestLogHandler requestLogHandler = new RequestLogHandler(); + StatisticsHandler statsHandler = new StatisticsHandler(); + statsHandler.setHandler(proxyConnectHandler); + + //possible not needed + //requestLogHandler.setRequestLog(customRequestLog); + handlers.setHandlers(new Handler[] { requestLogHandler, statsHandler, proxyConnectHandler }); + + this.server.setHandler(handlers); if (proxyHandler != null) { proxy.setProxyHandler(proxyHandler); + } ServletHolder proxyServlet = new ServletHolder(config.getName(), proxy); - proxyServlet.setInitParameter("proxyTo", config.getProxyTo()); proxyServlet.setInitParameter("prefix", config.getPrefix()); proxyServlet.setInitParameter("trustAll", config.getTrustAll()); proxyServlet.setInitParameter("preserveHost", config.getPreserveHost()); + proxyServlet.setInitParameter("timeout", "120000"); + // Setup proxy servlet this.context = - new ServletContextHandler(proxyConnectHandler, "/", ServletContextHandler.SESSIONS); + new ServletContextHandler(handlers, "/", ServletContextHandler.SESSIONS); this.context.addServlet(proxyServlet, "/*"); this.context.addFilter(RequestFilter.class, "/*", EnumSet.allOf(DispatcherType.class)); } @@ -118,6 +150,16 @@ public void start() { try { this.server.start(); + + // Schedule a task to log metrics at a fixed rate + StatisticsHandler stats = this.server.getChildHandlerByClass(StatisticsHandler.class); + this.scheduler.scheduleAtFixedRate(() -> { + log.debug("(jetty) Num requests: " + stats.getRequests()); + log.debug("(jetty) Num active requests: " + stats.getRequestsActive()); + log.debug("(jetty) Responses with 4xx status: " + stats.getResponses4xx()); + log.debug("(jetty) Responses with 5xx status: " + stats.getResponses5xx()); + // Log other metrics as needed + }, 0, 5, TimeUnit.MINUTES); } catch (Exception e) { log.error("Error starting proxy server", e); throw new IllegalStateException(e); @@ -128,6 +170,7 @@ public void start() { public void close() { try { this.server.stop(); + this.scheduler.shutdown(); } catch (Exception e) { log.error("Could not close the proxy server", e); } diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java index 385dd56e..8a1c36c2 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java @@ -45,7 +45,11 @@ protected HttpClient newHttpClient() { HttpClient httpClient = new HttpClient(sslFactory); httpClient.setMaxConnectionsPerDestination(10000); - httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(60)); + httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(65)); + httpClient.setIdleTimeout(TimeUnit.SECONDS.toMillis(65)); + httpClient.setRequestBufferSize(2048000); + httpClient.setResponseBufferSize(2048000); + return httpClient; }