Skip inactive cluster nodes

java/pom.xml

@@ -3,16 +3,37 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
+    <name>ScyllaDB Alternator Load Balancing Client</name>
     <groupId>com.scylladb.alternator</groupId>
     <artifactId>LoadBalancing</artifactId>
-    <version>1.0</version>
+    <version>1.1.0-SNAPSHOT</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
     </properties>
 
+    <distributionManagement>
+        <repository>
+            <id>yieldmo-releases</id>
+            <name>Yieldmo Releases</name>
+            <url>https://nexus.yieldmo.com/repository/releases/</url>
+        </repository>
+        <snapshotRepository>
+            <id>yieldmo-snapshots</id>
+            <name>Yieldmo Snapshots</name>
+            <url>https://nexus.yieldmo.com/repository/snapshots/</url>
+        </snapshotRepository>
+    </distributionManagement>
+
+    <repositories>
+        <repository>
+            <id>yieldmo-releases</id>
+            <url>https://nexus.yieldmo.com/repository/releases/</url>
+        </repository>
+    </repositories>
+
     <dependencyManagement>
         <dependencies>
             <!-- AWS SDK for Java version 1 -->

java/src/main/java/com/scylladb/alternator/AlternatorLiveNodes.java

@@ -1,17 +1,18 @@
 package com.scylladb.alternator;
 
+import java.util.HashSet;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Scanner;
 import java.util.Arrays;
-import java.util.regex.Pattern;
-import java.util.regex.Matcher;
+import java.util.Set;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
 import java.net.URISyntaxException;
 import java.net.MalformedURLException;
 import java.net.HttpURLConnection;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 import java.util.logging.Level;
 
@@ -25,11 +26,14 @@
  * to one of these nodes.
  */
 public class AlternatorLiveNodes extends Thread {
+    private static final long BAD_NODES_RESET_INTERVAL_MILLIS = TimeUnit.HOURS.toMillis(1);
     private String alternatorScheme;
     private int alternatorPort;
 
     private List<String> liveNodes;
     private int nextLiveNodeIndex;
+    private Set<String> badNodes = new HashSet<>();
+    private long badNodesListLastResetMillis = System.currentTimeMillis();
 
     private static Logger logger = Logger.getLogger(AlternatorLiveNodes.class.getName());
 
@@ -86,9 +90,9 @@ public URI nextAsURI() {
         }
     }
 
-    public URL nextAsURL(String file) {
+    public URL nextAsURL(String file, String nextNode) {
         try {
-            return new URL(alternatorScheme, nextNode(), alternatorPort, file);
+            return new URL(alternatorScheme, nextNode, alternatorPort, file);
         } catch (MalformedURLException e) {
             // Can only happen if alternatorScheme is an unknown one.
             logger.log(Level.WARNING, "nextAsURL", e);
@@ -104,8 +108,10 @@ private static String streamToString(java.io.InputStream is) {
     }
 
     private void updateLiveNodes() {
-        List<String> newHosts = new ArrayList<String>();
-        URL url = nextAsURL("/localnodes");
+        clearBadNodes();
+        List<String> newHosts = new ArrayList<>();
+        String nextNode = nextNode();
+        URL url = nextAsURL("/localnodes", nextNode);
         try {
             // Note that despite this being called HttpURLConnection, it actually
             // supports HTTPS as well.
@@ -126,13 +132,26 @@ private void updateLiveNodes() {
             }
         } catch (IOException e) {
             logger.log(Level.FINE, "Request failed: " + url, e);
+            badNodes.add(nextNode);
+            logger.log(Level.WARNING, "Marked node " + nextNode + " as bad");
         }
+        newHosts.removeAll(badNodes);
         if (!newHosts.isEmpty()) {
             synchronized(this) {
                 this.liveNodes = newHosts;
                 this.nextLiveNodeIndex = 0;
             }
-            logger.log(Level.FINE, "Updated hosts to " + this.liveNodes.toString());
+            logger.log(Level.FINE, "Updated hosts to " + this.liveNodes);
+            if (!badNodes.isEmpty()) {
+                logger.log(Level.FINE, "Bad nodes " + badNodes);
+            }
+        }
+    }
+
+    private void clearBadNodes() {
+        if (System.currentTimeMillis() - badNodesListLastResetMillis > BAD_NODES_RESET_INTERVAL_MILLIS) {
+            badNodes.clear();
+            badNodesListLastResetMillis = System.currentTimeMillis();
         }
     }
 }