From 1232e506fe0ed071b2d95f369b0e04c0cffe931b Mon Sep 17 00:00:00 2001 From: Frederick Michielssen Date: Wed, 3 Aug 2016 17:31:07 +0200 Subject: [PATCH] Added heartbeat mechanism to detect and free up idle/closed sessions --- .../controllers/AppController.java | 2 + .../controllers/HeartbeatController.java | 35 ++++++++++ .../openanalytics/services/DockerService.java | 3 + .../services/HeartbeatService.java | 66 +++++++++++++++++++ src/main/resources/application.yml | 2 + src/main/resources/templates/app.html | 11 +++- 6 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 src/main/java/eu/openanalytics/controllers/HeartbeatController.java create mode 100644 src/main/java/eu/openanalytics/services/HeartbeatService.java diff --git a/src/main/java/eu/openanalytics/controllers/AppController.java b/src/main/java/eu/openanalytics/controllers/AppController.java index 542106d7..b4838590 100644 --- a/src/main/java/eu/openanalytics/controllers/AppController.java +++ b/src/main/java/eu/openanalytics/controllers/AppController.java @@ -52,6 +52,8 @@ String app(ModelMap map, Principal principal, HttpServletRequest request) { map.put("title", environment.getProperty("shiny.proxy.title")); map.put("logo", environment.getProperty("shiny.proxy.logo-url")); map.put("container", "/" + mapping + environment.getProperty("shiny.proxy.landing-page")); + map.put("heartbeatRate", environment.getProperty("shiny.proxy.heartbeat-rate", "10000")); + return "app"; } } diff --git a/src/main/java/eu/openanalytics/controllers/HeartbeatController.java b/src/main/java/eu/openanalytics/controllers/HeartbeatController.java new file mode 100644 index 00000000..85a3d7a7 --- /dev/null +++ b/src/main/java/eu/openanalytics/controllers/HeartbeatController.java @@ -0,0 +1,35 @@ +package eu.openanalytics.controllers; + +import java.io.IOException; +import java.security.Principal; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; + +import eu.openanalytics.services.HeartbeatService; + +@Controller +public class HeartbeatController { + + @Inject + HeartbeatService heartbeatService; + + @RequestMapping("/heartbeat/**") + void heartbeat(Principal principal, HttpServletRequest request, HttpServletResponse response) { + String userName = (principal == null) ? request.getSession().getId() : principal.getName(); + Matcher matcher = Pattern.compile(".*/app/(.*)").matcher(request.getRequestURI()); + String appName = matcher.matches() ? matcher.group(1) : null; + heartbeatService.heartbeatReceived(userName, appName); + try { + response.setStatus(200); + response.getWriter().write("Ok"); + response.getWriter().flush(); + } catch (IOException e) {} + } +} diff --git a/src/main/java/eu/openanalytics/services/DockerService.java b/src/main/java/eu/openanalytics/services/DockerService.java index 08f34d5a..f4ed321d 100644 --- a/src/main/java/eu/openanalytics/services/DockerService.java +++ b/src/main/java/eu/openanalytics/services/DockerService.java @@ -79,6 +79,7 @@ public static class Proxy { public String containerId; public String userName; public String appName; + public long startupTimestamp; } @Bean @@ -119,6 +120,7 @@ public List listProxies() { copy.containerId = proxy.containerId; copy.userName = proxy.userName; copy.appName = proxy.appName; + copy.startupTimestamp = proxy.startupTimestamp; proxies.add(copy); } } @@ -218,6 +220,7 @@ private Proxy startProxy(String userName, String appName) { ContainerInfo info = dockerClient.inspectContainer(container.id()); proxy.name = info.name().substring(1); proxy.containerId = container.id(); + proxy.startupTimestamp = System.currentTimeMillis(); } catch (Exception e) { releasePort(proxy.port); throw new ShinyProxyException("Failed to start container: " + e.getMessage(), e); diff --git a/src/main/java/eu/openanalytics/services/HeartbeatService.java b/src/main/java/eu/openanalytics/services/HeartbeatService.java new file mode 100644 index 00000000..caeaca09 --- /dev/null +++ b/src/main/java/eu/openanalytics/services/HeartbeatService.java @@ -0,0 +1,66 @@ +package eu.openanalytics.services; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import org.apache.log4j.Logger; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import eu.openanalytics.services.DockerService.Proxy; + +@Service +public class HeartbeatService { + + @Inject + Environment environment; + + @Inject + DockerService dockerService; + + private Logger log = Logger.getLogger(HeartbeatService.class); + + private Map heartbeatTimestamps; + + @PostConstruct + public void init() { + heartbeatTimestamps = new ConcurrentHashMap<>(); + new Thread(new AppCleaner(), "HeartbeatThread").start(); + } + + public void heartbeatReceived(String user, String app) { + heartbeatTimestamps.put(user, System.currentTimeMillis()); + } + + private class AppCleaner implements Runnable { + @Override + public void run() { + long cleanupInterval = 2 * Long.parseLong(environment.getProperty("shiny.proxy.heartbeat-rate", "10000")); + long heartbeatTimeout = Long.parseLong(environment.getProperty("shiny.proxy.heartbeat-timeout", "60000")); + + while (true) { + try { + long currentTimestamp = System.currentTimeMillis(); + for (Proxy proxy: dockerService.listProxies()) { + Long lastHeartbeat = heartbeatTimestamps.get(proxy.userName); + if (lastHeartbeat == null) lastHeartbeat = proxy.startupTimestamp; + long proxySilence = currentTimestamp - lastHeartbeat; + if (proxySilence > heartbeatTimeout) { + log.info(String.format("Releasing inactive proxy [user: %s] [app: %s] [silence: %dms]", proxy.userName, proxy.appName, proxySilence)); + dockerService.releaseProxy(proxy.userName); + heartbeatTimestamps.remove(proxy.userName); + } + } + } catch (Throwable t) { + log.error("Error in HeartbeatThread", t); + } + try { + Thread.sleep(cleanupInterval); + } catch (InterruptedException e) {} + } + } + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b3efd3d3..42537f62 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -3,6 +3,8 @@ shiny: title: Open Analytics Shiny Proxy logo-url: http://www.openanalytics.eu/sites/www.openanalytics.eu/themes/oa/logo.png landing-page: / + heartbeat-rate: 10000 + heartbeat-timeout: 60000 port: 8080 authentication: ldap # LDAP configuration diff --git a/src/main/resources/templates/app.html b/src/main/resources/templates/app.html index a0fe1899..3c8f9496 100644 --- a/src/main/resources/templates/app.html +++ b/src/main/resources/templates/app.html @@ -17,8 +17,17 @@ - \ No newline at end of file