From 6718de08c261c037e4c711e4edaefc9da6479aec Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Mon, 1 Jul 2024 15:14:41 -0700 Subject: [PATCH 01/38] add log debug headers --- .../src/main/java/com/lyft/data/proxyserver/ProxyHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index c82e8a4e..b8a61829 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -66,6 +66,7 @@ protected void postConnectionHook( log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), var9); + debugLogHeaders(request); callback.failed(var9); } } From 28bde1d22322954caa99ea024e40385c1ae3f073 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Mon, 1 Jul 2024 15:21:35 -0700 Subject: [PATCH 02/38] use error log level --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index b8a61829..b2b0ee62 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -66,11 +66,20 @@ protected void postConnectionHook( log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), var9); - debugLogHeaders(request); + errorLogHeaders(request); callback.failed(var9); } } + protected void errorLogHeaders(HttpServletRequest request) { + log.error("-------HttpServletRequest headers---------"); + Enumeration headers = request.getHeaderNames(); + while (headers.hasMoreElements()) { + String header = headers.nextElement(); + log.error(header + "->" + request.getHeader(header)); + } + } + protected void debugLogHeaders(HttpServletRequest request) { if (log.isDebugEnabled()) { log.debug("-------HttpServletRequest headers---------"); From 2773cf3efee01ce6ededb5de4c5e274ebe3c8be4 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Mon, 1 Jul 2024 15:36:24 -0700 Subject: [PATCH 03/38] add logging for error response header --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index b2b0ee62..ec20f0d8 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -67,12 +67,13 @@ protected void postConnectionHook( + " servlet path {} , toString {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), var9); errorLogHeaders(request); + errorLogHeaders(response); callback.failed(var9); } } protected void errorLogHeaders(HttpServletRequest request) { - log.error("-------HttpServletRequest headers---------"); + log.error("------- error HttpServletRequest headers---------"); Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); @@ -80,6 +81,14 @@ protected void errorLogHeaders(HttpServletRequest request) { } } + protected void errorLogHeaders(HttpServletResponse response) { + log.error("------- error HttpServletResponse headers---------"); + Collection headers = response.getHeaderNames(); + for (String header : headers) { + log.error(header + "->" + response.getHeader(header)); + } + } + protected void debugLogHeaders(HttpServletRequest request) { if (log.isDebugEnabled()) { log.debug("-------HttpServletRequest headers---------"); From 8bc1af92e60ca5e2ffefd7ca0d4a433c79ad9d52 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Mon, 1 Jul 2024 18:32:55 -0700 Subject: [PATCH 04/38] other config changes --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 89e8bd1b..4087e0b8 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -86,10 +86,13 @@ private void setupContext(ProxyServerConfiguration config) { connector.setPort(config.getLocalPort()); connector.setName(config.getName()); connector.setAccepting(true); + connector.setIdleTimeout(150000L); + connector.setAcceptQueueSize(1024); this.server.addConnector(connector); // Setup proxy handler to handle CONNECT methods ConnectHandler proxyConnectHandler = new ConnectHandler(); + proxyConnectHandler.getIdleTimeout() this.server.setHandler(proxyConnectHandler); if (proxyHandler != null) { @@ -102,6 +105,7 @@ private void setupContext(ProxyServerConfiguration config) { proxyServlet.setInitParameter("prefix", config.getPrefix()); proxyServlet.setInitParameter("trustAll", config.getTrustAll()); proxyServlet.setInitParameter("preserveHost", config.getPreserveHost()); + proxyServlet.setInitParameter("timeout", "120000"); // Setup proxy servlet this.context = From c4e320064dd6d97758b8850227671cca9cd83e4b Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Mon, 1 Jul 2024 18:34:08 -0700 Subject: [PATCH 05/38] config testing --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 4087e0b8..cbddcdca 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -92,7 +92,6 @@ private void setupContext(ProxyServerConfiguration config) { // Setup proxy handler to handle CONNECT methods ConnectHandler proxyConnectHandler = new ConnectHandler(); - proxyConnectHandler.getIdleTimeout() this.server.setHandler(proxyConnectHandler); if (proxyHandler != null) { From d1ad10aa1ca6a768dcae1fea16058ffa3cda406a Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Mon, 1 Jul 2024 18:56:05 -0700 Subject: [PATCH 06/38] test --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index cbddcdca..0d22efb7 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -74,6 +74,7 @@ private void setupContext(ProxyServerConfiguration config) { src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); httpsConfig.addCustomizer(src); + httpsConfig.addCustomizer( new org.eclipse.jetty.server.ForwardedRequestCustomizer() ); connector = new ServerConnector( server, From 365e481445513b778e2425095bf32a8b153eeed2 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:01:43 -0700 Subject: [PATCH 07/38] add more changes --- gateway-ha/gateway-ha-config.yml | 4 ++++ .../data/gateway/ha/module/HaGatewayProviderModule.java | 1 + .../main/java/com/lyft/data/proxyserver/ProxyServer.java | 8 +++++++- .../java/com/lyft/data/proxyserver/ProxyServletImpl.java | 4 +++- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/gateway-ha/gateway-ha-config.yml b/gateway-ha/gateway-ha-config.yml index 70f1ee92..e8cd62e0 100644 --- a/gateway-ha/gateway-ha-config.yml +++ b/gateway-ha/gateway-ha-config.yml @@ -6,6 +6,10 @@ requestRouter: port: 8080 name: prestoRouter historySize: 1000 + requestHeaderSize: 2048000 + responseHeaderSize: 2048000 + requestBufferSize: 2048000 + responseBufferSize: 2048000 dataStore: jdbcUrl: jdbc:mysql://127.0.0.1:3306/prestogateway diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java index edadc249..d0184393 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java @@ -81,6 +81,7 @@ public ProxyServer provideGateway() { routerProxyConfig.setKeystorePass(routerConfiguration.getKeystorePass()); routerProxyConfig.setForwardKeystore(routerConfiguration.isForwardKeystore()); routerProxyConfig.setPreserveHost("false"); + ProxyHandler proxyHandler = getProxyHandler(); gateway = new ProxyServer(routerProxyConfig, proxyHandler); } diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 0d22efb7..bcc690bf 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -59,6 +59,7 @@ private void setupContext(ProxyServerConfiguration config) { sslContextFactory.setStopTimeout(TimeUnit.SECONDS.toMillis(15)); sslContextFactory.setSslSessionTimeout((int) TimeUnit.SECONDS.toMillis(15)); + if (!TextUtils.isBlank(keystorePath)) { sslContextFactory.setKeyStorePath(keystoreFile.getAbsolutePath()); sslContextFactory.setKeyStorePassword(keystorePass); @@ -68,13 +69,18 @@ private void setupContext(ProxyServerConfiguration config) { HttpConfiguration httpsConfig = new HttpConfiguration(); httpsConfig.setSecureScheme(HttpScheme.HTTPS.asString()); httpsConfig.setSecurePort(config.getLocalPort()); + httpsConfig.setIdleTimeout(150000); httpsConfig.setOutputBufferSize(32768); + httpsConfig.setRequestHeaderSize(2048000); + httpsConfig.setResponseHeaderSize(2048000); + + SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); httpsConfig.addCustomizer(src); - httpsConfig.addCustomizer( new org.eclipse.jetty.server.ForwardedRequestCustomizer() ); + httpsConfig.addCustomizer(new org.eclipse.jetty.server.ForwardedRequestCustomizer()); connector = new ServerConnector( server, diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java index 385dd56e..b541f589 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java @@ -45,7 +45,9 @@ protected HttpClient newHttpClient() { HttpClient httpClient = new HttpClient(sslFactory); httpClient.setMaxConnectionsPerDestination(10000); - httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(60)); + httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(65)); + httpClient.setIdleTimeout(TimeUnit.SECONDS.toMillis(65)); + return httpClient; } From 57e0995935596fd95a3da72389fef5cd5e41ac7c Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:02:57 -0700 Subject: [PATCH 08/38] more config changes --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index bcc690bf..b3ac5a42 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -74,8 +74,6 @@ private void setupContext(ProxyServerConfiguration config) { httpsConfig.setRequestHeaderSize(2048000); httpsConfig.setResponseHeaderSize(2048000); - - SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); From 3eb9da8b9f80cc2afd5848f691399c556cb570ae Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:06:57 -0700 Subject: [PATCH 09/38] add logging for error request/response header size --- .../lyft/data/proxyserver/ProxyHandler.java | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index ec20f0d8..44acf8e6 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -64,29 +64,38 @@ protected void postConnectionHook( callback.succeeded(); } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," - + " servlet path {} , toString {}", request.getRequestURL(), - request.getRequestURI(), request.getServletPath(), request.toString(), var9); - errorLogHeaders(request); - errorLogHeaders(response); + + " servlet path {} , toString {}, size {}", request.getRequestURL(), + request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), var9); + + requestHeaderLog = getErrorLogHeaders(request); + log.error(requestHeaderLog); + + responseHeaderLog = getErrorLogHeaders(response); + log.error(responseHeaderLog); callback.failed(var9); } } - protected void errorLogHeaders(HttpServletRequest request) { - log.error("------- error HttpServletRequest headers---------"); + protected String getErrorLogHeaders(HttpServletRequest request) { + StringBuilder sb = new StringBuilder(); + sb.append("------- error HttpServletRequest headers---------\n"); Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); - log.error(header + "->" + request.getHeader(header)); + sb.append(header).append("->").append(request.getHeader(header)).append("\n"); } + return sb.toString(); } - protected void errorLogHeaders(HttpServletResponse response) { - log.error("------- error HttpServletResponse headers---------"); - Collection headers = response.getHeaderNames(); - for (String header : headers) { - log.error(header + "->" + response.getHeader(header)); + protected void getErrorLogHeaders(HttpServletResponse response) { + StringBuilder sb = new StringBuilder(); + sb.append("------- error HttpServletResponse headers---------\n"); + Enumeration headers = response.getHeaderNames(); + while (headers.hasMoreElements()) { + String header = headers.nextElement(); + sb.append(header).append("->").append(response.getHeader(header)).append("\n"); } + return sb.toString(); } protected void debugLogHeaders(HttpServletRequest request) { From 7bd21d9139c34941ca85bb982f34daf58d5515cc Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:08:01 -0700 Subject: [PATCH 10/38] add logging for success at 1% --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 44acf8e6..3bc91147 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -61,6 +61,15 @@ protected void postConnectionHook( } response.getOutputStream().write(buffer, offset, length); + // [sev-16337] with a 1% probably, log the request and response headers and size for debugging + if (Math.random() < 0.01) { + debugLogHeaders(request); + debugLogHeaders(response); + log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength()); + } + callback.succeeded(); } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," From e781b3fb5848798a5afbd9647b5f80c04368e02b Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:10:00 -0700 Subject: [PATCH 11/38] remove full header debug from success --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 3bc91147..bf09b71b 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -61,10 +61,8 @@ protected void postConnectionHook( } response.getOutputStream().write(buffer, offset, length); - // [sev-16337] with a 1% probably, log the request and response headers and size for debugging - if (Math.random() < 0.01) { - debugLogHeaders(request); - debugLogHeaders(response); + // [sev-16337] with a 10% probably, log the request and response headers and size for debugging + if (Math.random() < 0.10) { log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength()); @@ -76,10 +74,10 @@ protected void postConnectionHook( + " servlet path {} , toString {}, size {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), var9); - requestHeaderLog = getErrorLogHeaders(request); + String requestHeaderLog = getErrorLogHeaders(request); log.error(requestHeaderLog); - responseHeaderLog = getErrorLogHeaders(response); + String responseHeaderLog = getErrorLogHeaders(response); log.error(responseHeaderLog); callback.failed(var9); } @@ -96,7 +94,7 @@ protected String getErrorLogHeaders(HttpServletRequest request) { return sb.toString(); } - protected void getErrorLogHeaders(HttpServletResponse response) { + protected String getErrorLogHeaders(HttpServletResponse response) { StringBuilder sb = new StringBuilder(); sb.append("------- error HttpServletResponse headers---------\n"); Enumeration headers = response.getHeaderNames(); From fba8a403c036473d51fde75280f61874769aad51 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:16:28 -0700 Subject: [PATCH 12/38] more updates --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 +- .../main/java/com/lyft/data/proxyserver/ProxyServletImpl.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index b3ac5a42..3fd2424f 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -73,7 +73,7 @@ private void setupContext(ProxyServerConfiguration config) { httpsConfig.setOutputBufferSize(32768); httpsConfig.setRequestHeaderSize(2048000); httpsConfig.setResponseHeaderSize(2048000); - + SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java index b541f589..8a1c36c2 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java @@ -47,6 +47,8 @@ protected HttpClient newHttpClient() { httpClient.setMaxConnectionsPerDestination(10000); httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(65)); httpClient.setIdleTimeout(TimeUnit.SECONDS.toMillis(65)); + httpClient.setRequestBufferSize(2048000); + httpClient.setResponseBufferSize(2048000); return httpClient; } From 603ff4d0ce324985bb39965d5b4262b2a010e203 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:15:58 -0700 Subject: [PATCH 13/38] fix --- .../lyft/data/proxyserver/ProxyHandler.java | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index bf09b71b..d5ff024d 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -60,7 +60,6 @@ protected void postConnectionHook( request.getRequestURL()); } response.getOutputStream().write(buffer, offset, length); - // [sev-16337] with a 10% probably, log the request and response headers and size for debugging if (Math.random() < 0.10) { log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", @@ -73,36 +72,27 @@ protected void postConnectionHook( log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}, size {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), var9); - - String requestHeaderLog = getErrorLogHeaders(request); - log.error(requestHeaderLog); - - String responseHeaderLog = getErrorLogHeaders(response); - log.error(responseHeaderLog); + errorLogHeaders(request); + errorLogHeaders(response); callback.failed(var9); } } - protected String getErrorLogHeaders(HttpServletRequest request) { - StringBuilder sb = new StringBuilder(); - sb.append("------- error HttpServletRequest headers---------\n"); + protected void errorLogHeaders(HttpServletRequest request) { + log.error("------- error HttpServletRequest headers---------"); Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); - sb.append(header).append("->").append(request.getHeader(header)).append("\n"); + log.error(header + "->" + request.getHeader(header)); } - return sb.toString(); } - protected String getErrorLogHeaders(HttpServletResponse response) { - StringBuilder sb = new StringBuilder(); - sb.append("------- error HttpServletResponse headers---------\n"); - Enumeration headers = response.getHeaderNames(); - while (headers.hasMoreElements()) { - String header = headers.nextElement(); - sb.append(header).append("->").append(response.getHeader(header)).append("\n"); + protected void errorLogHeaders(HttpServletResponse response) { + log.error("------- error HttpServletResponse headers---------"); + Collection headers = response.getHeaderNames(); + for (String header : headers) { + log.error(header + "->" + response.getHeader(header)); } - return sb.toString(); } protected void debugLogHeaders(HttpServletRequest request) { @@ -172,4 +162,4 @@ protected boolean isCompressed(final byte[] compressed) { return (compressed[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (compressed[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8)); } -} +} \ No newline at end of file From c17fbf7cab167cb8068bd3e16799ae4ad35549f1 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:29:17 -0700 Subject: [PATCH 14/38] fix style --- .../main/java/com/lyft/data/proxyserver/ProxyHandler.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index d5ff024d..e52e9e8f 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -60,8 +60,9 @@ protected void postConnectionHook( request.getRequestURL()); } response.getOutputStream().write(buffer, offset, length); - // [sev-16337] with a 10% probably, log the request and response headers and size for debugging - if (Math.random() < 0.10) { + // [sev-16337] with a 1% probably, log the request and response headers + // and size for debugging + if (Math.random() < 0.01) { log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength()); @@ -71,7 +72,8 @@ protected void postConnectionHook( } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}, size {}", request.getRequestURL(), - request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), var9); + request.getRequestURI(), request.getServletPath(), request.toString(), + request.getContentLength(), var9); errorLogHeaders(request); errorLogHeaders(response); callback.failed(var9); From ccde25ea6376c29bfb442e09863046fe34ad1601 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:40:23 -0700 Subject: [PATCH 15/38] add check for header not just content size --- .../lyft/data/proxyserver/ProxyHandler.java | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index e52e9e8f..67ddb3da 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -63,23 +63,46 @@ protected void postConnectionHook( // [sev-16337] with a 1% probably, log the request and response headers // and size for debugging if (Math.random() < 0.01) { - log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", + log.debug("Request URL: {} , request URI {} , servlet path {} ," + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), - request.toString(), request.getContentLength()); + request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response)); } callback.succeeded(); } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," - + " servlet path {} , toString {}, size {}", request.getRequestURL(), - request.getRequestURI(), request.getServletPath(), request.toString(), - request.getContentLength(), var9); + + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," + + "getResponseHeaderSize {}", request.getRequestURL(), request.getRequestURI(), + request.getServletPath(), request.toString(), + request.getContentLength(), getRequestHeaderSize(request), getRequestHeaderSize(response) var9); errorLogHeaders(request); errorLogHeaders(response); callback.failed(var9); } } + protected int getRequestHeaderSize(HttpServletRequest request) { + int headerSize = 0; + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + String headerValue = request.getHeader(headerName); + headerSize += headerName.length() + headerValue.length(); + } + return headerSize; +} + +private int getResponseHeaderSize(HttpServletResponse response) { + int headerSize = 0; + for (String headerName : response.getHeaderNames()) { + String headerValue = response.getHeader(headerName); + headerSize += headerName.length() + headerValue.length(); + } + return headerSize; +} + + protected void errorLogHeaders(HttpServletRequest request) { log.error("------- error HttpServletRequest headers---------"); Enumeration headers = request.getHeaderNames(); From a3eb31957107668a9b25fffdb56adb802a92b7c7 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 22:13:33 +0530 Subject: [PATCH 16/38] Update ProxyHandler.java --- .../lyft/data/proxyserver/ProxyHandler.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 67ddb3da..da9ac050 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -73,11 +73,11 @@ protected void postConnectionHook( } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," - + "getResponseHeaderSize {}", request.getRequestURL(), request.getRequestURI(), - request.getServletPath(), request.toString(), - request.getContentLength(), getRequestHeaderSize(request), getRequestHeaderSize(response) var9); - errorLogHeaders(request); - errorLogHeaders(response); + + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), + request.getContentLength(), getRequestHeaderSize(request), getRequestHeaderSize(response), + errorLogHeaders(request), errorLogHeaders(response), var9); + callback.failed(var9); } } @@ -102,22 +102,24 @@ private int getResponseHeaderSize(HttpServletResponse response) { return headerSize; } - - protected void errorLogHeaders(HttpServletRequest request) { - log.error("------- error HttpServletRequest headers---------"); + protected String errorLogHeaders(HttpServletRequest request) { + StringBuilder sb = "------- error HttpServletRequest headers---------"; Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); - log.error(header + "->" + request.getHeader(header)); + sb.append(header); } + return sb.toString(); } - protected void errorLogHeaders(HttpServletResponse response) { - log.error("------- error HttpServletResponse headers---------"); - Collection headers = response.getHeaderNames(); - for (String header : headers) { - log.error(header + "->" + response.getHeader(header)); + protected String errorLogHeaders(HttpServletResponse response) { + StringBuilder sb = "------- error HttpServletResponse headers---------"; + Enumeration headers = request.getHeaderNames(); + while (headers.hasMoreElements()) { + String header = headers.nextElement(); + sb.append(header); } + return sb.toString(); } protected void debugLogHeaders(HttpServletRequest request) { @@ -187,4 +189,4 @@ protected boolean isCompressed(final byte[] compressed) { return (compressed[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (compressed[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8)); } -} \ No newline at end of file +} From 8f72c712e6c9601964cabf546c6be74bf3dab268 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:45:09 -0700 Subject: [PATCH 17/38] dropwizard update --- baseapp/pom.xml | 2 +- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/baseapp/pom.xml b/baseapp/pom.xml index c6be2d49..3662cb1f 100644 --- a/baseapp/pom.xml +++ b/baseapp/pom.xml @@ -17,7 +17,7 @@ 1.8 UTF-8 UTF-8 - 1.3.29 + 1.3.7 4.1.0 0.9.10 diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 3fd2424f..b3ac5a42 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -73,7 +73,7 @@ private void setupContext(ProxyServerConfiguration config) { httpsConfig.setOutputBufferSize(32768); httpsConfig.setRequestHeaderSize(2048000); httpsConfig.setResponseHeaderSize(2048000); - + SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); From f488a042f00849c71483a85679653cbdd89bf8e8 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:51:10 -0700 Subject: [PATCH 18/38] compile fix --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index da9ac050..9b52daa9 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -103,22 +103,23 @@ private int getResponseHeaderSize(HttpServletResponse response) { } protected String errorLogHeaders(HttpServletRequest request) { - StringBuilder sb = "------- error HttpServletRequest headers---------"; + StringBuilder sb = new StringBuilder("------- error HttpServletRequest headers---------"); Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); sb.append(header); } + return sb.toString(); } protected String errorLogHeaders(HttpServletResponse response) { - StringBuilder sb = "------- error HttpServletResponse headers---------"; - Enumeration headers = request.getHeaderNames(); - while (headers.hasMoreElements()) { - String header = headers.nextElement(); + StringBuilder sb = new StringBuilder("------- error HttpServletResponse headers---------"); + Collection headers = response.getHeaderNames(); + for (String header : headers) { sb.append(header); } + return sb.toString(); } From 60db65e652676e22c520d399f058baaf47cb34fe Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:53:13 -0700 Subject: [PATCH 19/38] more fixes --- .../src/main/java/com/lyft/data/proxyserver/ProxyHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 9b52daa9..1c582a27 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -75,7 +75,7 @@ protected void postConnectionHook( + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), - request.getContentLength(), getRequestHeaderSize(request), getRequestHeaderSize(response), + request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), errorLogHeaders(response), var9); callback.failed(var9); @@ -109,7 +109,7 @@ protected String errorLogHeaders(HttpServletRequest request) { String header = headers.nextElement(); sb.append(header); } - + return sb.toString(); } From 7092cdb998a6b9289c08322c8d1bfea46a3bd352 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 10:05:07 -0700 Subject: [PATCH 20/38] lint fixes --- .../lyft/data/proxyserver/ProxyHandler.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 1c582a27..08d3f628 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -64,9 +64,10 @@ protected void postConnectionHook( // and size for debugging if (Math.random() < 0.01) { log.debug("Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}", - request.getRequestURL(), request.getRequestURI(), request.getServletPath(), - request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response)); + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength(), getRequestHeaderSize(request), + getResponseHeaderSize(response)); } callback.succeeded(); @@ -74,9 +75,10 @@ protected void postConnectionHook( log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", - request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), - request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), - errorLogHeaders(request), errorLogHeaders(response), var9); + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength(), getRequestHeaderSize(request), + getResponseHeaderSize(response), errorLogHeaders(request), + errorLogHeaders(response), var9); callback.failed(var9); } @@ -86,21 +88,21 @@ protected int getRequestHeaderSize(HttpServletRequest request) { int headerSize = 0; Enumeration headerNames = request.getHeaderNames(); while (headerNames.hasMoreElements()) { - String headerName = headerNames.nextElement(); - String headerValue = request.getHeader(headerName); - headerSize += headerName.length() + headerValue.length(); + String headerName = headerNames.nextElement(); + String headerValue = request.getHeader(headerName); + headerSize += headerName.length() + headerValue.length(); } return headerSize; -} + } -private int getResponseHeaderSize(HttpServletResponse response) { - int headerSize = 0; - for (String headerName : response.getHeaderNames()) { + private int getResponseHeaderSize(HttpServletResponse response) { + int headerSize = 0; + for (String headerName : response.getHeaderNames()) { String headerValue = response.getHeader(headerName); headerSize += headerName.length() + headerValue.length(); + } + return headerSize; } - return headerSize; -} protected String errorLogHeaders(HttpServletRequest request) { StringBuilder sb = new StringBuilder("------- error HttpServletRequest headers---------"); From fd6b9f63af374b6c009102c14d7f1e0c13fb1e5a Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 10:45:52 -0700 Subject: [PATCH 21/38] fix header logging --- .../src/main/java/com/lyft/data/proxyserver/ProxyHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 08d3f628..4b04c29f 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -109,7 +109,7 @@ protected String errorLogHeaders(HttpServletRequest request) { Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); - sb.append(header); + sb.append(header + "->" + request.getHeader(header) + "\n"); } return sb.toString(); @@ -119,7 +119,7 @@ protected String errorLogHeaders(HttpServletResponse response) { StringBuilder sb = new StringBuilder("------- error HttpServletResponse headers---------"); Collection headers = response.getHeaderNames(); for (String header : headers) { - sb.append(header); + sb.append(header + "->" + response.getHeader(header) + "\n"); } return sb.toString(); From 765f61d627e0e4b959028a9608a6bfefa811cc8c Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 23:40:33 +0530 Subject: [PATCH 22/38] Update ProxyHandler.java --- .../main/java/com/lyft/data/proxyserver/ProxyHandler.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 4b04c29f..1406d904 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -64,10 +64,11 @@ protected void postConnectionHook( // and size for debugging if (Math.random() < 0.01) { log.debug("Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}", + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), - getResponseHeaderSize(response)); + getResponseHeaderSize(response), errorLogHeaders(request), + errorLogHeaders(response)); } callback.succeeded(); From b145513798489803b0f8d01dc19a641115bed9ee Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 23:42:41 +0530 Subject: [PATCH 23/38] Update ProxyHandler.java --- .../src/main/java/com/lyft/data/proxyserver/ProxyHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 1406d904..0afd1c91 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -64,7 +64,8 @@ protected void postConnectionHook( // and size for debugging if (Math.random() < 0.01) { log.debug("Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, + requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), From 1ce21041649fb9e1a5055e30517bc1a6c53616be Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 14:29:56 -0700 Subject: [PATCH 24/38] add logging in preconnection hook --- .../lyft/data/proxyserver/ProxyHandler.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 0afd1c91..cb790383 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -35,6 +35,15 @@ protected String rewriteTarget(HttpServletRequest request) { */ public void preConnectionHook(HttpServletRequest request, Request proxyRequest) { // you may override it. + + // [sev-16337] with a 10% probably, log the request headers for debugging + if (Math.random() < 0.10) { + log.debug("(preConnectionHook) Request URL: {} , request URI {} , servlet path {} ," + + "toString {}, getContentLength {}, getRequestHeaderSize {}, requestHeaders {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength(), getRequestHeaderSize(request), + errorLogHeaders(request)); + } } /** @@ -60,12 +69,12 @@ protected void postConnectionHook( request.getRequestURL()); } response.getOutputStream().write(buffer, offset, length); - // [sev-16337] with a 1% probably, log the request and response headers + // [sev-16337] with a 10% probably, log the request and response headers // and size for debugging - if (Math.random() < 0.01) { - log.debug("Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, - requestHeaders {}, responseHeaders {}", + if (Math.random() < 0.10) { + log.debug("(postConnectionHook - success) Request URL: {} , request URI {} , servlet path {} ," + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, " + + "requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), @@ -74,9 +83,9 @@ protected void postConnectionHook( callback.succeeded(); } catch (Throwable var9) { - log.error("Exception occurred while processing request URL: {} , request URI {} ," - + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," - + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", + log.error("(postConnectionHook - failed) Exception occurred while processing request URL: {} , " + + "request URI {} , servlet path {} , toString {}, getContentLength {}, " + + "getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), From 80ac96c5b220e3b102708ed886272cc5ae72c313 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 14:43:06 -0700 Subject: [PATCH 25/38] checkstyle fixes --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index cb790383..b34153e3 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -72,9 +72,9 @@ protected void postConnectionHook( // [sev-16337] with a 10% probably, log the request and response headers // and size for debugging if (Math.random() < 0.10) { - log.debug("(postConnectionHook - success) Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, " - + "requestHeaders {}, responseHeaders {}", + log.debug("(postConnectionHook) Request URL: {} , request URI {} , servlet path {} , " + + "toString {}, getContentLength {}, getRequestHeaderSize {}, " + + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), @@ -83,9 +83,10 @@ protected void postConnectionHook( callback.succeeded(); } catch (Throwable var9) { - log.error("(postConnectionHook - failed) Exception occurred while processing request URL: {} , " + log.error("(postConnectionHook) Exception occurred while processing request URL: {} , " + "request URI {} , servlet path {} , toString {}, getContentLength {}, " - + "getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", + + "getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, " + + "responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), From 5911d0b229894b2fc7a17884c3a3e7cd39ab185e Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 17:47:39 -0400 Subject: [PATCH 26/38] adding error counters --- .../handler/QueryIdCachingProxyHandler.java | 19 ++++++++++++++++++- .../ha/module/HaGatewayProviderModule.java | 11 ++++++++--- proxyserver/pom.xml | 7 +++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java index c16dabd0..e272811b 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java @@ -1,6 +1,8 @@ package com.lyft.data.gateway.ha.handler; +import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.io.CharStreams; @@ -28,6 +30,8 @@ import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.util.Callback; +import static com.codahale.metrics.MetricRegistry.name; + @Slf4j public class QueryIdCachingProxyHandler extends ProxyHandler { public static final String PROXY_TARGET_HEADER = "proxytarget"; @@ -57,13 +61,20 @@ public class QueryIdCachingProxyHandler extends ProxyHandler { private final Meter requestMeter; private final int serverApplicationPort; + private final Counter errorCounter4xx; + private final Counter errorCounter5xx; + private final MetricRegistry metrics = new MetricRegistry(); + public QueryIdCachingProxyHandler( QueryHistoryManager queryHistoryManager, RoutingManager routingManager, RoutingGroupSelector routingGroupSelector, int serverApplicationPort, - Meter requestMeter) { + Meter requestMeter, + MetricRegistry metrics) { this.requestMeter = requestMeter; + this.errorCounter4xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "4xx")); + this.errorCounter5xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "5xx"));; this.routingManager = routingManager; this.routingGroupSelector = routingGroupSelector; this.queryHistoryManager = queryHistoryManager; @@ -270,6 +281,12 @@ protected void postConnectionHook( } catch (Exception e) { log.error("Error in proxying falling back to super call", e); } +// if (response.getStatus() >= 500){ +// errorCounter5xx.inc(); +// } else if (response.getStatus() >= 400){ +// errorCounter4xx.inc(); +// } + super.postConnectionHook(request, response, buffer, offset, length, callback); } diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java index d0184393..6d4fbf99 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java @@ -1,6 +1,8 @@ package com.lyft.data.gateway.ha.module; +import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; import com.google.inject.Provides; import com.google.inject.Singleton; import com.lyft.data.baseapp.AppModule; @@ -23,6 +25,8 @@ import com.lyft.data.proxyserver.ProxyServerConfiguration; import io.dropwizard.setup.Environment; +import static com.codahale.metrics.MetricRegistry.name; + public class HaGatewayProviderModule extends AppModule { private final ResourceGroupsManager resourceGroupsManager; @@ -42,9 +46,9 @@ public HaGatewayProviderModule(HaGatewayConfiguration configuration, Environment } protected ProxyHandler getProxyHandler() { + MetricRegistry metrics = getEnvironment().metrics(); Meter requestMeter = - getEnvironment() - .metrics() + metrics .meter(getConfiguration().getRequestRouter().getName() + ".requests"); // By default, use routing group header to route @@ -61,7 +65,8 @@ protected ProxyHandler getProxyHandler() { getRoutingManager(), routingGroupSelector, getApplicationPort(), - requestMeter); + requestMeter, + metrics); } @Provides diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index d14b6999..9ad19af2 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -18,6 +18,7 @@ 1.8 UTF-8 UTF-8 + 1.3.29 @@ -62,6 +63,12 @@ mockwebserver test + + io.dropwizard.metrics + metrics-core + 4.2.7 + compile + From 4d1c079da98566d921fdf9eec8232f4cb9feb144 Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 17:49:53 -0400 Subject: [PATCH 27/38] forgot to uncomment --- .../gateway/ha/handler/QueryIdCachingProxyHandler.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java index e272811b..ea1b1873 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java @@ -281,11 +281,11 @@ protected void postConnectionHook( } catch (Exception e) { log.error("Error in proxying falling back to super call", e); } -// if (response.getStatus() >= 500){ -// errorCounter5xx.inc(); -// } else if (response.getStatus() >= 400){ -// errorCounter4xx.inc(); -// } + if (response.getStatus() >= 500){ + errorCounter5xx.inc(); + } else if (response.getStatus() >= 400){ + errorCounter4xx.inc(); + } super.postConnectionHook(request, response, buffer, offset, length, callback); } From 29613c4cb4f288b6e17a07882098b5c12205ac13 Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 17:56:50 -0400 Subject: [PATCH 28/38] fix checkstyle --- .../gateway/ha/handler/QueryIdCachingProxyHandler.java | 8 ++++---- .../data/gateway/ha/module/HaGatewayProviderModule.java | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java index ea1b1873..270fb2df 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java @@ -1,5 +1,7 @@ package com.lyft.data.gateway.ha.handler; +import static com.codahale.metrics.MetricRegistry.name; + import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; @@ -30,7 +32,6 @@ import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.util.Callback; -import static com.codahale.metrics.MetricRegistry.name; @Slf4j public class QueryIdCachingProxyHandler extends ProxyHandler { @@ -281,12 +282,11 @@ protected void postConnectionHook( } catch (Exception e) { log.error("Error in proxying falling back to super call", e); } - if (response.getStatus() >= 500){ + if (response.getStatus() >= 500) { errorCounter5xx.inc(); - } else if (response.getStatus() >= 400){ + } else if (response.getStatus() >= 400) { errorCounter4xx.inc(); } - super.postConnectionHook(request, response, buffer, offset, length, callback); } diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java index 6d4fbf99..b2b37c0f 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java @@ -1,6 +1,5 @@ package com.lyft.data.gateway.ha.module; -import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.google.inject.Provides; @@ -25,7 +24,6 @@ import com.lyft.data.proxyserver.ProxyServerConfiguration; import io.dropwizard.setup.Environment; -import static com.codahale.metrics.MetricRegistry.name; public class HaGatewayProviderModule extends AppModule { From d9cfe8ddbb61986a9320abe8f254512911ab929a Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 18:29:02 -0400 Subject: [PATCH 29/38] update dropwizard ver --- .../data/gateway/ha/handler/QueryIdCachingProxyHandler.java | 3 +-- proxyserver/pom.xml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java index 270fb2df..19395718 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java @@ -64,7 +64,6 @@ public class QueryIdCachingProxyHandler extends ProxyHandler { private final Counter errorCounter4xx; private final Counter errorCounter5xx; - private final MetricRegistry metrics = new MetricRegistry(); public QueryIdCachingProxyHandler( QueryHistoryManager queryHistoryManager, @@ -75,7 +74,7 @@ public QueryIdCachingProxyHandler( MetricRegistry metrics) { this.requestMeter = requestMeter; this.errorCounter4xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "4xx")); - this.errorCounter5xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "5xx"));; + this.errorCounter5xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "5xx")); this.routingManager = routingManager; this.routingGroupSelector = routingGroupSelector; this.queryHistoryManager = queryHistoryManager; diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index 9ad19af2..5b318e41 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -18,7 +18,7 @@ 1.8 UTF-8 UTF-8 - 1.3.29 + 1.3.7 From 0fee702cf9ea44120a72373da4583b43b08e32f8 Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 18:33:50 -0400 Subject: [PATCH 30/38] updating dropwizard dependency --- proxyserver/pom.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index 5b318e41..b679c39b 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -23,6 +23,11 @@ + + io.dropwizard + dropwizard-core + ${dropwizard.version} + org.eclipse.jetty jetty-server @@ -63,12 +68,6 @@ mockwebserver test - - io.dropwizard.metrics - metrics-core - 4.2.7 - compile - From 94ed9d8115b2f1519d0a57b282da25b1a9eafab8 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 20:21:21 -0700 Subject: [PATCH 31/38] update --- gateway-ha/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 8b321801..0c790da8 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -20,7 +20,7 @@ UTF-8 UTF-8 9.4.48.v20220622 - 1.3.29 + 1.3.7 4.1.0 23.0 4.1.0 From 2cb6d6695b1577f0c79b1d3f255546f3ca8ed938 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 3 Jul 2024 13:44:52 -0700 Subject: [PATCH 32/38] pom changes --- gateway-ha/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 0c790da8..8b321801 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -20,7 +20,7 @@ UTF-8 UTF-8 9.4.48.v20220622 - 1.3.7 + 1.3.29 4.1.0 23.0 4.1.0 From 1c25d758140a75e1b93a3a5264b3e269ca7aae1d Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 3 Jul 2024 15:07:12 -0700 Subject: [PATCH 33/38] try request logging --- .../com/lyft/data/proxyserver/ProxyServer.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index b3ac5a42..1fe5d647 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -10,16 +10,22 @@ import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.ResponseHandler; import org.apache.http.util.TextUtils; +import org.eclipse.jetty.http.HttpParser; import org.eclipse.jetty.http.HttpScheme; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.proxy.ConnectHandler; +import org.eclipse.jetty.server.CustomRequestLog; +import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.handler.RequestLogHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -97,7 +103,16 @@ private void setupContext(ProxyServerConfiguration config) { // Setup proxy handler to handle CONNECT methods ConnectHandler proxyConnectHandler = new ConnectHandler(); - this.server.setHandler(proxyConnectHandler); + + HandlerCollection handlers = new HandlerCollection(); + + RequestLogHandler requestLogHandler = new RequestLogHandler(); + //possible not needed + //requestLogHandler.setRequestLog(customRequestLog); + handlers.setHandlers(new Handler[] { requestLogHandler, proxyConnectHandler }); + + + this.server.setHandler(handlers); if (proxyHandler != null) { proxy.setProxyHandler(proxyHandler); From 76f038b89587acde27a79df7b4d362b34bfd08a8 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 3 Jul 2024 15:51:29 -0700 Subject: [PATCH 34/38] add handler --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 1fe5d647..7e1d65cd 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -116,19 +116,20 @@ private void setupContext(ProxyServerConfiguration config) { if (proxyHandler != null) { proxy.setProxyHandler(proxyHandler); + } ServletHolder proxyServlet = new ServletHolder(config.getName(), proxy); - proxyServlet.setInitParameter("proxyTo", config.getProxyTo()); proxyServlet.setInitParameter("prefix", config.getPrefix()); proxyServlet.setInitParameter("trustAll", config.getTrustAll()); proxyServlet.setInitParameter("preserveHost", config.getPreserveHost()); proxyServlet.setInitParameter("timeout", "120000"); + // Setup proxy servlet this.context = - new ServletContextHandler(proxyConnectHandler, "/", ServletContextHandler.SESSIONS); + new ServletContextHandler(handlers, "/", ServletContextHandler.SESSIONS); this.context.addServlet(proxyServlet, "/*"); this.context.addFilter(RequestFilter.class, "/*", EnumSet.allOf(DispatcherType.class)); } From 7d60671b6fdd8028dc8c807d4cef96432d0b46ef Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Wed, 3 Jul 2024 14:14:46 -0700 Subject: [PATCH 35/38] add stats handler --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 7e1d65cd..6bdc4b31 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -26,6 +26,7 @@ import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.server.handler.RequestLogHandler; +import org.eclipse.jetty.server.handler.StatisticsHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -107,9 +108,10 @@ private void setupContext(ProxyServerConfiguration config) { HandlerCollection handlers = new HandlerCollection(); RequestLogHandler requestLogHandler = new RequestLogHandler(); + StatisticsHandler statsHandler = new StatisticsHandler(); //possible not needed //requestLogHandler.setRequestLog(customRequestLog); - handlers.setHandlers(new Handler[] { requestLogHandler, proxyConnectHandler }); + handlers.setHandlers(new Handler[] { requestLogHandler, statsHandler, proxyConnectHandler }); this.server.setHandler(handlers); From 2cd95fe8fb6933e0748d13837157d308e3ac4ba3 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Wed, 3 Jul 2024 16:05:40 -0700 Subject: [PATCH 36/38] add StatsHandler jetty logs --- .../com/lyft/data/proxyserver/ProxyServer.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 6bdc4b31..24e387c5 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -3,6 +3,8 @@ import java.io.Closeable; import java.io.File; import java.util.EnumSet; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.servlet.DispatcherType; @@ -36,6 +38,9 @@ public class ProxyServer implements Closeable { private final Server server; private final ProxyServletImpl proxy; private final ProxyHandler proxyHandler; + private final ScheduledExecutorService scheduler = + Executors.newScheduledThreadPool(1); + private ServletContextHandler context; public ProxyServer(ProxyServerConfiguration config, ProxyHandler proxyHandler) { @@ -109,11 +114,12 @@ private void setupContext(ProxyServerConfiguration config) { RequestLogHandler requestLogHandler = new RequestLogHandler(); StatisticsHandler statsHandler = new StatisticsHandler(); + statsHandler.setHandler(proxyConnectHandler); + //possible not needed //requestLogHandler.setRequestLog(customRequestLog); handlers.setHandlers(new Handler[] { requestLogHandler, statsHandler, proxyConnectHandler }); - this.server.setHandler(handlers); if (proxyHandler != null) { @@ -144,6 +150,16 @@ public void start() { try { this.server.start(); + + // Schedule a task to log metrics at a fixed rate + StatisticsHandler stats = this.server.getChildHandlerByClass(StatisticsHandler.class); + this.scheduler.scheduleAtFixedRate(() -> { + log.debug("(jetty) Num requests: " + stats.getRequests()); + log.debug("(jetty) Num requests: " + stats.getRequestsActive()); + log.debug("(jetty) Responses with 4xx status: " + stats.getResponses4xx()); + log.debug("(jetty) Responses with 5xx status: " + stats.getResponses5xx()); + // Log other metrics as needed + }, 0, 5, TimeUnit.MINUTES); } catch (Exception e) { log.error("Error starting proxy server", e); throw new IllegalStateException(e); From a2303ac3cc0d3101ac52bb6d1df9b47bffdd8a00 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Wed, 3 Jul 2024 16:18:07 -0700 Subject: [PATCH 37/38] fix debug log --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 24e387c5..8706b019 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -155,7 +155,7 @@ public void start() { StatisticsHandler stats = this.server.getChildHandlerByClass(StatisticsHandler.class); this.scheduler.scheduleAtFixedRate(() -> { log.debug("(jetty) Num requests: " + stats.getRequests()); - log.debug("(jetty) Num requests: " + stats.getRequestsActive()); + log.debug("(jetty) Num active requests: " + stats.getRequestsActive()); log.debug("(jetty) Responses with 4xx status: " + stats.getResponses4xx()); log.debug("(jetty) Responses with 5xx status: " + stats.getResponses5xx()); // Log other metrics as needed From c772828b5e76ad2c5188346d95e3ec0f68f29125 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Wed, 3 Jul 2024 16:22:25 -0700 Subject: [PATCH 38/38] shutdown scheduler thread on close --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 8706b019..e2bfc96d 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -170,6 +170,7 @@ public void start() { public void close() { try { this.server.stop(); + this.scheduler.shutdown(); } catch (Exception e) { log.error("Could not close the proxy server", e); }