addressing review comments

Signed-off-by: Santhosh Gandhe <[email protected]>
san81 · Oct 24, 2024 · cf35364 · cf35364
1 parent 5a4e966
commit cf35364
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 15 deletions.
diff --git a/...er/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/Crawler.java b/...er/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/Crawler.java
@@ -46,9 +46,9 @@ public long crawl(long lastPollTime,
                     continue;
                 }
                 itemInfoList.add(nextItem);
-                Map<String, String> metadata = nextItem.getMetadata();
-                long niCreated = Long.parseLong(metadata.get(CREATED)!=null? metadata.get(CREATED):"0");
-                long niUpdated = Long.parseLong(metadata.get(UPDATED)!=null? metadata.get(UPDATED):"0");
+                Map<String, Object> metadata = nextItem.getMetadata();
+                long niCreated = Long.parseLong(metadata.get(CREATED)!=null? (String)metadata.get(CREATED):"0");
+                long niUpdated = Long.parseLong(metadata.get(UPDATED)!=null? (String)metadata.get(UPDATED):"0");
                 updatedPollTime = Math.max(updatedPollTime, niCreated);
                 updatedPollTime = Math.max(updatedPollTime, niUpdated);
             }

diff --git a/...src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasClient.java b/...src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasClient.java
@@ -25,7 +25,19 @@ public interface SaasClient {
     Iterator<ItemInfo> listItems();
 
 
+    /**
+     * Method to set the last time we polled the service to check for any changes.
+     *
+     * @param lastPollTime time in milliseconds
+     */
     void setLastPollTime(long lastPollTime);
 
+    /**
+     * Method for executing a particular partition or a chunk of work
+     *
+     * @param state worker node state holds the details of this particular chunk of work
+     * @param buffer pipeline buffer to write the results into
+     * @param sourceConfig pipeline configuration from the yaml
+     */
     void executePartition(SaasWorkerProgressState state, Buffer<Record<Event>> buffer, SaasSourceConfig sourceConfig);
 }
diff --git a/...in/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasSourcePlugin.java b/...in/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasSourcePlugin.java
@@ -4,7 +4,6 @@
 import lombok.Getter;
 import org.opensearch.dataprepper.metrics.PluginMetrics;
 import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager;
-import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor;
 import org.opensearch.dataprepper.model.buffer.Buffer;
 import org.opensearch.dataprepper.model.codec.ByteDecoder;
 import org.opensearch.dataprepper.model.event.Event;
@@ -52,7 +51,6 @@ public abstract class SaasSourcePlugin implements Source<Record<Event>>, UsesEnh
   private final Crawler crawler;
 
 
-  @DataPrepperPluginConstructor
   public SaasSourcePlugin(final PluginMetrics pluginMetrics,
                           final SaasSourceConfig sourceConfig,
                           final PluginFactory pluginFactory,
@@ -95,11 +93,6 @@ public void stop() {
     this.executorService.shutdownNow();
   }
 
-  @Override
-  public boolean areAcknowledgementsEnabled() {
-    return Source.super.areAcknowledgementsEnabled();
-  }
-
 
   @Override
   public void setEnhancedSourceCoordinator(EnhancedSourceCoordinator sourceCoordinator) {

diff --git a/.../src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfo.java b/.../src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfo.java
@@ -22,7 +22,7 @@ public abstract class ItemInfo {
      * contents itself which can be used to apply regex filtering, change data capture etc. general
      * assumption here is that fetching metadata should be faster than fetching entire Item
      */
-    Map<String, String> metadata;
+    Map<String, Object> metadata;
 
     /**
      * Process your change log events serially (preferably in a single thread) and ensure that you are
@@ -37,7 +37,7 @@ public ItemInfo(String itemId) {
         this.itemId = itemId;
     }
 
-    public ItemInfo(@NonNull String itemId, Map<String, String> metadata, @NonNull Long eventTime) {
+    public ItemInfo(@NonNull String itemId, Map<String, Object> metadata, @NonNull Long eventTime) {
         this.itemId = itemId;
         this.metadata = metadata;
         this.eventTime = eventTime;

diff --git a/...rc/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/CrawlerTest.java b/...rc/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/CrawlerTest.java
@@ -157,7 +157,7 @@ private int getMaxItemsPerPage() throws NoSuchFieldException, IllegalAccessExcep
     }
 
     private static class TestItemInfo extends ItemInfo {
-        public TestItemInfo(String itemId, Map<String, String> metadata, Long eventTime) {
+        public TestItemInfo(String itemId, Map<String, Object> metadata, Long eventTime) {
             super(itemId, metadata, eventTime);
 
         }
@@ -178,7 +178,7 @@ public Map<String, String> getKeyAttributes() {
     }
 
     private ItemInfo createTestItemInfo(String id, String created, String updated) {
-        Map<String, String> metadata = new HashMap<>();
+        Map<String, Object> metadata = new HashMap<>();
         if (created != null) metadata.put(Crawler.CREATED, created);
         if (updated != null) metadata.put(Crawler.UPDATED, updated);
 

diff --git a/.../test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfoTest.java b/.../test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfoTest.java
@@ -12,7 +12,7 @@ public class ItemInfoTest {
 
     static class TestItemInfo extends ItemInfo {
 
-        public TestItemInfo(@NonNull String itemId, Map<String, String> metadata, @NonNull Long eventTime) {
+        public TestItemInfo(@NonNull String itemId, Map<String, Object> metadata, @NonNull Long eventTime) {
             super(itemId, metadata, eventTime);
         }