From cf3536421720b54ca29b8cca08e83aa6f7de20a3 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:24:02 -0700 Subject: [PATCH] addressing review comments Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../plugins/source/saas_crawler/base/Crawler.java | 6 +++--- .../plugins/source/saas_crawler/base/SaasClient.java | 12 ++++++++++++ .../source/saas_crawler/base/SaasSourcePlugin.java | 7 ------- .../plugins/source/saas_crawler/model/ItemInfo.java | 4 ++-- .../source/saas_crawler/base/CrawlerTest.java | 4 ++-- .../source/saas_crawler/model/ItemInfoTest.java | 2 +- 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/Crawler.java b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/Crawler.java index 3e44d44eb5..76397cf909 100644 --- a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/Crawler.java +++ b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/Crawler.java @@ -46,9 +46,9 @@ public long crawl(long lastPollTime, continue; } itemInfoList.add(nextItem); - Map metadata = nextItem.getMetadata(); - long niCreated = Long.parseLong(metadata.get(CREATED)!=null? metadata.get(CREATED):"0"); - long niUpdated = Long.parseLong(metadata.get(UPDATED)!=null? metadata.get(UPDATED):"0"); + Map metadata = nextItem.getMetadata(); + long niCreated = Long.parseLong(metadata.get(CREATED)!=null? (String)metadata.get(CREATED):"0"); + long niUpdated = Long.parseLong(metadata.get(UPDATED)!=null? (String)metadata.get(UPDATED):"0"); updatedPollTime = Math.max(updatedPollTime, niCreated); updatedPollTime = Math.max(updatedPollTime, niUpdated); } diff --git a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasClient.java b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasClient.java index e304f5120c..c78784f298 100644 --- a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasClient.java +++ b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasClient.java @@ -25,7 +25,19 @@ public interface SaasClient { Iterator listItems(); + /** + * Method to set the last time we polled the service to check for any changes. + * + * @param lastPollTime time in milliseconds + */ void setLastPollTime(long lastPollTime); + /** + * Method for executing a particular partition or a chunk of work + * + * @param state worker node state holds the details of this particular chunk of work + * @param buffer pipeline buffer to write the results into + * @param sourceConfig pipeline configuration from the yaml + */ void executePartition(SaasWorkerProgressState state, Buffer> buffer, SaasSourceConfig sourceConfig); } diff --git a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasSourcePlugin.java b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasSourcePlugin.java index c93e2d8e7a..87c92fb103 100644 --- a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasSourcePlugin.java +++ b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/SaasSourcePlugin.java @@ -4,7 +4,6 @@ import lombok.Getter; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; -import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.model.event.Event; @@ -52,7 +51,6 @@ public abstract class SaasSourcePlugin implements Source>, UsesEnh private final Crawler crawler; - @DataPrepperPluginConstructor public SaasSourcePlugin(final PluginMetrics pluginMetrics, final SaasSourceConfig sourceConfig, final PluginFactory pluginFactory, @@ -95,11 +93,6 @@ public void stop() { this.executorService.shutdownNow(); } - @Override - public boolean areAcknowledgementsEnabled() { - return Source.super.areAcknowledgementsEnabled(); - } - @Override public void setEnhancedSourceCoordinator(EnhancedSourceCoordinator sourceCoordinator) { diff --git a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfo.java b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfo.java index 58ddfdc172..5dc2f86ef5 100644 --- a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfo.java @@ -22,7 +22,7 @@ public abstract class ItemInfo { * contents itself which can be used to apply regex filtering, change data capture etc. general * assumption here is that fetching metadata should be faster than fetching entire Item */ - Map metadata; + Map metadata; /** * Process your change log events serially (preferably in a single thread) and ensure that you are @@ -37,7 +37,7 @@ public ItemInfo(String itemId) { this.itemId = itemId; } - public ItemInfo(@NonNull String itemId, Map metadata, @NonNull Long eventTime) { + public ItemInfo(@NonNull String itemId, Map metadata, @NonNull Long eventTime) { this.itemId = itemId; this.metadata = metadata; this.eventTime = eventTime; diff --git a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/CrawlerTest.java b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/CrawlerTest.java index 2de0f1f98e..d4f5a63f30 100644 --- a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/CrawlerTest.java +++ b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/base/CrawlerTest.java @@ -157,7 +157,7 @@ private int getMaxItemsPerPage() throws NoSuchFieldException, IllegalAccessExcep } private static class TestItemInfo extends ItemInfo { - public TestItemInfo(String itemId, Map metadata, Long eventTime) { + public TestItemInfo(String itemId, Map metadata, Long eventTime) { super(itemId, metadata, eventTime); } @@ -178,7 +178,7 @@ public Map getKeyAttributes() { } private ItemInfo createTestItemInfo(String id, String created, String updated) { - Map metadata = new HashMap<>(); + Map metadata = new HashMap<>(); if (created != null) metadata.put(Crawler.CREATED, created); if (updated != null) metadata.put(Crawler.UPDATED, updated); diff --git a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfoTest.java index 8561bbc661..c3093472aa 100644 --- a/data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfoTest.java +++ b/data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas_crawler/model/ItemInfoTest.java @@ -12,7 +12,7 @@ public class ItemInfoTest { static class TestItemInfo extends ItemInfo { - public TestItemInfo(@NonNull String itemId, Map metadata, @NonNull Long eventTime) { + public TestItemInfo(@NonNull String itemId, Map metadata, @NonNull Long eventTime) { super(itemId, metadata, eventTime); }