Consolidated all snappydata collections in snappy-util

- consolidated all kolokobe generated collections and OpenHashSet into separate project snappydata-util which can be used elsewhere too (e.g. snappy-spark) - replaced all uses of trove TIntObjectHashMap and TLongObjectHashMap by LongObjectHashMap - minor change to OpenHashSet to use Consumer instead of Function
ashetkar · Jan 8, 2018 · 1adfe8c · 1adfe8c
1 parent 29b4453
commit 1adfe8c
Show file tree

Hide file tree

Showing 32 changed files with 219 additions and 218 deletions.
diff --git a/build.gradle b/build.gradle
@@ -688,8 +688,8 @@ if (!rootProject.hasProperty('store')) {
 
 task generateSources {
   dependsOn subprojectBase + 'gemfire-jgroups:jgMagic'
-  dependsOn subprojectBase + 'gemfire-util:compileJava'
   dependsOn subprojectBase + 'gemfire-core:createVersionPropertiesFile'
+  dependsOn subprojectBase + 'snappydata-util:compileJava'
   dependsOn subprojectBase + 'snappydata-store-core:compileJavacc'
   dependsOn subprojectBase + 'snappydata-store-core:generatePropertiesFiles'
   dependsOn subprojectBase + 'gemfire-core:createVersionPropertiesFile'

diff --git a/gemfire-core/build.gradle b/gemfire-core/build.gradle
@@ -127,6 +127,7 @@ dependencies {
   compile project(subprojectBase + 'gemfire-util')
   compile project(subprojectBase + 'gemfire-shared')
   compile project(subprojectBase + 'gemfire-trove')
+  compile project(subprojectBase + 'snappydata-util')
 
   jcaCompile sourceSets.main.output
 }

diff --git a/...ire-core/src/main/java/com/gemstone/gemfire/distributed/internal/DistributionAdvisor.java b/...ire-core/src/main/java/com/gemstone/gemfire/distributed/internal/DistributionAdvisor.java
@@ -45,7 +45,7 @@
 import com.gemstone.gemfire.internal.cache.persistence.PersistentMemberID;
 import com.gemstone.gemfire.internal.cache.versions.VersionSource;
 import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
-import com.gemstone.gemfire.internal.shared.OpenHashSet;
+import io.snappydata.collection.OpenHashSet;
 import com.gemstone.gemfire.internal.shared.Version;
 import com.gemstone.gemfire.internal.util.ArrayUtils;
 

diff --git a/.../com/gemstone/gemfire/distributed/internal/membership/jgroup/JGroupMembershipManager.java b/.../com/gemstone/gemfire/distributed/internal/membership/jgroup/JGroupMembershipManager.java
@@ -84,7 +84,6 @@
 import com.gemstone.gemfire.internal.tcp.Stub;
 import com.gemstone.gemfire.internal.tcp.TCPConduit;
 import com.gemstone.gemfire.internal.util.Breadcrumbs;
-import com.gemstone.gnu.trove.TIntObjectHashMap;
 import com.gemstone.org.jgroups.Address;
 import com.gemstone.org.jgroups.Channel;
 import com.gemstone.org.jgroups.ChannelClosedException;
@@ -111,6 +110,7 @@
 import com.gemstone.org.jgroups.stack.ProtocolStack;
 import com.gemstone.org.jgroups.util.GemFireTracer;
 import com.gemstone.gnu.trove.TObjectProcedure;
+import io.snappydata.collection.LongObjectHashMap;
 
 public final class JGroupMembershipManager implements MembershipManager {
 
@@ -3293,13 +3293,13 @@ public Set send(InternalDistributedMember[] destinations,
             calculatedMembers.add((JGroupMember)destinations[i].getNetMember());
           }
         } // send to explicit list
-        TIntObjectHashMap messages = new TIntObjectHashMap();
+        LongObjectHashMap<Message> messages = LongObjectHashMap.withExpectedSize(2);
         long startSer = theStats.startMsgSerialization();
         boolean firstMessage = true;
         for (Iterator it=calculatedMembers.iterator(); it.hasNext(); ) {
           JGroupMember mbr = (JGroupMember)it.next();
           short version = mbr.getAddress().getVersionOrdinal();
-          if ( !messages.containsKey(version) ) {
+          if (!messages.contains(version)) {
             Message jmsg = createJGMessage(msg, local, version);
             messages.put(version, jmsg);
             if (firstMessage) {
@@ -3316,7 +3316,7 @@ public Set send(InternalDistributedMember[] destinations,
           JGroupMember mbr = it.next();
           IpAddress to = mbr.getAddress();
           short version = to.getVersionOrdinal();
-          Message jmsg = (Message)messages.get(version);
+          Message jmsg = messages.get(version);
           if (DistributionManager.VERBOSE || logger.fineEnabled())
             logger.info(
                 LocalizedStrings.DEBUG,

diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/CopyOnWriteHashSet.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/CopyOnWriteHashSet.java
@@ -25,7 +25,7 @@
 import java.util.Iterator;
 import java.util.Set;
 
-import com.gemstone.gemfire.internal.shared.OpenHashSet;
+import io.snappydata.collection.OpenHashSet;
 
 /**
  * A Hash set where every modification makes an internal copy 

diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/DSFIDFactory.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/DSFIDFactory.java
@@ -125,11 +125,11 @@
 import com.gemstone.gemfire.pdx.internal.CheckTypeRegistryState;
 import com.gemstone.gemfire.pdx.internal.EnumId;
 import com.gemstone.gemfire.pdx.internal.EnumInfo;
-import com.gemstone.gnu.trove.TIntObjectHashMap;
 import com.gemstone.org.jgroups.View;
 import com.gemstone.org.jgroups.protocols.pbcast.JoinRsp;
 import com.gemstone.org.jgroups.stack.IpAddress;
 import com.gemstone.org.jgroups.util.StreamableFixedID;
+import io.snappydata.collection.LongObjectHashMap;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -169,7 +169,8 @@ public Version[] getSerializationVersions() {
 
   private static volatile boolean typesRegistered;
   private static final Supplier[] dsfidMap = new Supplier[256];
-  private static final TIntObjectHashMap dsfidMap2 = new TIntObjectHashMap(800);
+  private static final LongObjectHashMap<Supplier<?>> dsfidMap2 =
+      LongObjectHashMap.withExpectedSize(512);
 
   static {
     if (!InternalDistributedSystem.isHadoopGfxdLonerMode()) {
@@ -863,14 +864,14 @@ public static Object create(int dsfid, DataInput in)
         if (dsfid >= Byte.MIN_VALUE && dsfid <= Byte.MAX_VALUE) {
           creator = dsfidMap[dsfid + Byte.MAX_VALUE + 1];
         } else {
-          creator = (Supplier<?>)dsfidMap2.get(dsfid);
+          creator = dsfidMap2.get(dsfid);
         }
         if (creator == null && !typesRegistered) {
           registerTypes();
           if (dsfid >= Byte.MIN_VALUE && dsfid <= Byte.MAX_VALUE) {
             creator = dsfidMap[dsfid + Byte.MAX_VALUE + 1];
           } else {
-            creator = (Supplier<?>)dsfidMap2.get(dsfid);
+            creator = dsfidMap2.get(dsfid);
           }
         }
         if (creator != null) {
@@ -1130,7 +1131,7 @@ static Supplier<?>[] getDsfidmap() {
     return dsfidMap;
   }
 
-  static TIntObjectHashMap getDsfidmap2() {
+  static LongObjectHashMap<Supplier<?>> getDsfidmap2() {
     return dsfidMap2;
   }
 }
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/CacheServerLauncher.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/CacheServerLauncher.java
@@ -51,7 +51,7 @@
 import com.gemstone.gemfire.internal.process.StartupStatusListener;
 import com.gemstone.gemfire.internal.shared.LauncherBase;
 import com.gemstone.gemfire.internal.shared.NativeCalls;
-import com.gemstone.gemfire.internal.shared.OpenHashSet;
+import io.snappydata.collection.OpenHashSet;
 import com.gemstone.gemfire.internal.util.JavaCommandBuilder;
 
 import static com.gemstone.gemfire.internal.cache.Status.*;

diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskInitFile.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskInitFile.java
@@ -75,7 +75,7 @@
 import com.gemstone.gnu.trove.TIntHashSet;
 import com.gemstone.gnu.trove.TLongHashSet;
 import com.gemstone.gnu.trove.TLongIterator;
-import io.snappydata.collection.IntObjectHashMap;
+import io.snappydata.collection.LongObjectHashMap;
 
 /**
  * Does all the IF file work for a DiskStoreImpl.
@@ -1837,14 +1837,15 @@ private void savePRConfigs() {
       this.ifTotalRecordCount++;
     }
   }
-  
+
   private void saveCanonicalIds() {
-    IntObjectHashMap<Object> mappings = canonicalIdHolder.getAllMappings();
-    for (Map.Entry<Integer, Object> e : mappings.entrySet()) {
-      writeCanonicalId(e.getKey(), e.getValue());
-    }
+    LongObjectHashMap<Object> mappings = canonicalIdHolder.getAllMappings();
+    mappings.forEachWhile((id, v) -> {
+      writeCanonicalId((int)id, v);
+      return true;
+    });
   }
-  
+
   private void saveRevokedMembers() {
     for(PersistentMemberPattern revoked : revokedMembers) {
       writeRevokedMember(revoked);

diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java
@@ -148,7 +148,7 @@
 import com.gemstone.gemfire.internal.shared.ClientSharedUtils;
 import com.gemstone.gemfire.internal.shared.HeapBufferAllocator;
 import com.gemstone.gemfire.internal.shared.NativeCalls;
-import com.gemstone.gemfire.internal.shared.OpenHashSet;
+import io.snappydata.collection.OpenHashSet;
 import com.gemstone.gemfire.internal.shared.SystemProperties;
 import com.gemstone.gemfire.internal.shared.Version;
 import com.gemstone.gemfire.internal.shared.unsafe.DirectBufferAllocator;
@@ -1552,30 +1552,34 @@ public Map getSnapshotRVV() {
     try {
       // Wait for all the regions to get initialized before taking snapshot.
       lockForSnapshotRvv.readLock().lock();
-      Set<LocalRegion> regions = getApplicationRegions();
-      ObjectObjectHashMap<String, Map> snapshot =
-          ObjectObjectHashMap.withExpectedSize(regions.size());
-      for (LocalRegion region : regions) {
-        if (region.getPartitionAttributes() != null && ((PartitionedRegion)region).isDataStore()
-            && ((PartitionedRegion)region).concurrencyChecksEnabled) {
-          region.waitForData();
-          for (BucketRegion br : ((PartitionedRegion)region).getDataStore().getAllLocalBucketRegions()) {
-            // if null then create the rvv for that bucket.!
+      final ObjectObjectHashMap<String, Map> snapshot =
+          ObjectObjectHashMap.withExpectedSize(this.pathToRegion.size());
+      this.pathToRegion.values().forEach(region -> {
+        RegionVersionVector rvv;
+        if (region instanceof BucketRegion) {
+          BucketRegion br = (BucketRegion)region;
+          PartitionedRegion pr = br.getPartitionedRegion();
+          if (pr.concurrencyChecksEnabled) {
+            pr.waitForData();
+            // if null then create the rvv for that bucket!
             // For Initialization case, so that we have all the data before snapshot.
             br.waitForData();
-            snapshot.put(br.getFullPath(), br.getVersionVector().getSnapShotOfMemberVersion());
+            snapshot.put(br.getFullPath(),
+                br.getVersionVector().getSnapShotOfMemberVersion());
           }
-        } else if (region.getVersionVector() != null) {
-          // if null then create the rvv for that region.!
+        } else if ((rvv = region.getVersionVector()) != null &&
+            !region.isSecret() && !region.isUsedForMetaRegion() &&
+            !region.isInternalRegion() && !(region instanceof HARegion) &&
+            !region.isUsedForPartitionedRegionAdmin()) {
+          // if null then create the rvv for that region!
           // For Initialization case, so that we have all the data before snapshot.
           region.waitForData();
-          snapshot.put(region.getFullPath(), region.getVersionVector().getSnapShotOfMemberVersion());
+          snapshot.put(region.getFullPath(), rvv.getSnapShotOfMemberVersion());
         }
-      }
+      });
       return snapshot;
     } finally {
       lockForSnapshotRvv.readLock().unlock();
-
     }
   }
 

diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java
@@ -210,7 +210,7 @@
 import com.gemstone.gemfire.internal.offheap.annotations.Retained;
 import com.gemstone.gemfire.internal.offheap.annotations.Unretained;
 import com.gemstone.gemfire.internal.sequencelog.EntryLogger;
-import com.gemstone.gemfire.internal.shared.OpenHashSet;
+import io.snappydata.collection.OpenHashSet;
 import com.gemstone.gemfire.internal.shared.Version;
 import com.gemstone.gemfire.internal.size.ReflectionObjectSizer;
 import com.gemstone.gemfire.internal.size.ReflectionSingleObjectSizer;

diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/Oplog.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/Oplog.java
@@ -118,7 +118,7 @@
 import com.gemstone.gemfire.internal.sequencelog.EntryLogger;
 import com.gemstone.gemfire.internal.shared.ClientSharedUtils;
 import com.gemstone.gemfire.internal.shared.NativeCalls;
-import com.gemstone.gemfire.internal.shared.OpenHashSet;
+import io.snappydata.collection.OpenHashSet;
 import com.gemstone.gemfire.internal.shared.UnsupportedGFXDVersionException;
 import com.gemstone.gemfire.internal.shared.Version;
 import com.gemstone.gemfire.internal.shared.unsafe.ChannelBufferUnsafeDataInputStream;

diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PRHARedundancyProvider.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PRHARedundancyProvider.java
@@ -60,7 +60,7 @@
 import com.gemstone.gemfire.internal.cache.persistence.PersistentMemberID;
 import com.gemstone.gemfire.internal.cache.persistence.PersistentStateListener;
 import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
-import com.gemstone.gemfire.internal.shared.OpenHashSet;
+import io.snappydata.collection.OpenHashSet;
 import com.gemstone.org.jgroups.util.StringId;
 
 /**

diff --git a/...core/src/main/java/com/gemstone/gemfire/internal/cache/persistence/CanonicalIdHolder.java b/...core/src/main/java/com/gemstone/gemfire/internal/cache/persistence/CanonicalIdHolder.java
@@ -16,7 +16,7 @@
  */
 package com.gemstone.gemfire.internal.cache.persistence;
 
-import io.snappydata.collection.IntObjectHashMap;
+import io.snappydata.collection.LongObjectHashMap;
 import io.snappydata.collection.ObjectLongHashMap;
 
 /**
@@ -33,8 +33,8 @@ public class CanonicalIdHolder {
   /**
    * Map of integer representation to canonicalized member ids.
    */
-  private final IntObjectHashMap<Object> idToObject =
-      IntObjectHashMap.withExpectedSize(10);
+  private final LongObjectHashMap<Object> idToObject =
+      LongObjectHashMap.withExpectedSize(10);
 
   /**
    * Map of canonicalized member ids to integer representation.
@@ -87,7 +87,7 @@ public int createId(Object object) {
    * @return a map of id to object for all objects
    * held by this canonical id holder.
    */
-  public IntObjectHashMap<Object> getAllMappings() {
+  public LongObjectHashMap<Object> getAllMappings() {
     return idToObject;
   }
 }
diff --git a/...-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java b/...-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java
@@ -49,7 +49,7 @@
 import com.gemstone.gemfire.internal.cache.TXStateInterface;
 import com.gemstone.gemfire.internal.cache.persistence.DiskStoreID;
 import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
-import com.gemstone.gemfire.internal.shared.OpenHashSet;
+import io.snappydata.collection.OpenHashSet;
 import com.gemstone.gemfire.internal.shared.Version;
 import com.gemstone.gemfire.internal.util.concurrent.CopyOnWriteHashMap;
 import io.snappydata.collection.ObjectObjectHashMap;

diff --git a/...core/src/main/java/com/gemstone/gemfire/internal/i18n/AbstractStringIdResourceBundle.java b/...core/src/main/java/com/gemstone/gemfire/internal/i18n/AbstractStringIdResourceBundle.java
@@ -21,13 +21,13 @@
 import com.gemstone.gemfire.i18n.StringIdImpl;
 import com.gemstone.gemfire.internal.ClassPathLoader;
 import com.gemstone.org.jgroups.util.StringId;
+import io.snappydata.collection.LongObjectHashMap;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.Locale;
-import com.gemstone.gnu.trove.TIntObjectHashMap;
 
 
 /**
@@ -38,7 +38,7 @@
  * @since 5.7 
  */
 public class AbstractStringIdResourceBundle {
-  private TIntObjectHashMap data;
+  private LongObjectHashMap<String> data;
 
   /** 
    * The {@link java.util.ResourceBundle} that implements the message lookup
@@ -47,8 +47,8 @@ public class AbstractStringIdResourceBundle {
   private static AbstractStringIdResourceBundle messageBundle;
 
   /**
-   * Init method to populate the TIntObjectHashMap for Non-english locales
-   * <code>data = new TIntObjectHashMap();</code>
+   * Init method to populate the LongObjectHashMap for Non-english locales
+   * <code>data = LongObjectHashMap.withExpectedSize(...);</code>
    *
    * The default bundle, English, will be
    * <code>data = null</code>
@@ -76,8 +76,8 @@ private void initData(String baseName, Locale l) {
     }
   }
 
-  private TIntObjectHashMap readDataFile(InputStream is) {
-    TIntObjectHashMap map = new TIntObjectHashMap();
+  private LongObjectHashMap<String> readDataFile(InputStream is) {
+    LongObjectHashMap<String> map = LongObjectHashMap.withExpectedSize(128);
     boolean complete = false;
     BufferedReader input = null;
     try {
@@ -129,7 +129,7 @@ private AbstractStringIdResourceBundle() {
   public String getString(StringId key) {
     if (usingRawMode())
       return key.getRawText();
-    String txt = (String) data.get(((StringIdImpl)key).id);
+    String txt = data.get(((StringIdImpl)key).id);
     if( txt != null ) {
       return txt;
     } else {

diff --git a/.../main/java/com/gemstone/gemfire/internal/BackwardCompatibilitySerializationJUnitTest.java b/.../main/java/com/gemstone/gemfire/internal/BackwardCompatibilitySerializationJUnitTest.java
@@ -191,13 +191,13 @@ public void testAllMessages() throws Exception {
 
     // some msgs require distributed system
     Cache c = new CacheFactory().create();
-    for (Object o : DSFIDFactory.getDsfidmap2().getValues()) {
-      Supplier<?> cons = (Supplier<?>)o;
+    DSFIDFactory.getDsfidmap2().forEachWhile((id, cons) -> {
       if (cons != null) {
         DataSerializableFixedID ds = (DataSerializableFixedID)cons.get();
         checkSupportForRollingUpgrade(ds);
       }
-    }
+      return true;
+    });
     c.close();
   }