Skip to content

Commit

Permalink
module name renamed to source-crawler
Browse files Browse the repository at this point in the history
Signed-off-by: Santhosh Gandhe <[email protected]>
  • Loading branch information
san81 committed Oct 24, 2024
1 parent cf35364 commit 6894d24
Show file tree
Hide file tree
Showing 28 changed files with 69 additions and 69 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ plugins {

dependencies {

implementation project(path: ':data-prepper-plugins:saas-source-plugins:saas-crawler')
implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler')
implementation project(path: ':data-prepper-api')
implementation project(path: ':data-prepper-plugins:aws-plugin-api')
implementation project(path: ':data-prepper-plugins:buffer-common')
Expand All @@ -15,7 +15,7 @@ dependencies {
implementation 'com.fasterxml.jackson.core:jackson-core'
implementation 'com.fasterxml.jackson.core:jackson-databind'
implementation 'javax.inject:javax.inject:1'
implementation 'org.springframework:spring-web:5.3.39'
implementation("org.springframework:spring-web:${libs.versions.spring.get()}")
implementation 'org.springframework.retry:spring-retry:1.3.4'

implementation 'org.projectlombok:lombok:1.18.30'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
import org.opensearch.dataprepper.model.buffer.Buffer;
import org.opensearch.dataprepper.model.event.Event;
import org.opensearch.dataprepper.model.record.Record;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.SaasClient;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.SaasSourceConfig;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.saas_crawler.model.ItemInfo;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.SaasClient;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.SaasSourceConfig;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@
import org.opensearch.dataprepper.model.plugin.PluginFactory;
import org.opensearch.dataprepper.model.record.Record;
import org.opensearch.dataprepper.model.source.Source;
import org.opensearch.dataprepper.plugins.source.saas_crawler.SaasCrawlerApplicationContextMarker;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.SaasPluginExecutorServiceProvider;
import org.opensearch.dataprepper.plugins.source.source_crawler.SaasCrawlerApplicationContextMarker;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.SaasPluginExecutorServiceProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler;
package org.opensearch.dataprepper.plugins.source.source_crawler;

/**
* Market interface to indicate the base package to scan for dependency injection
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.base;
package org.opensearch.dataprepper.plugins.source.source_crawler.base;

import org.opensearch.dataprepper.model.buffer.Buffer;
import org.opensearch.dataprepper.model.event.Event;
import org.opensearch.dataprepper.model.record.Record;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.saas_crawler.model.ItemInfo;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.base;
package org.opensearch.dataprepper.plugins.source.source_crawler.base;

import org.opensearch.dataprepper.model.buffer.Buffer;
import org.opensearch.dataprepper.model.event.Event;
import org.opensearch.dataprepper.model.record.Record;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.saas_crawler.model.ItemInfo;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo;

import java.util.Iterator;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.base;
package org.opensearch.dataprepper.plugins.source.source_crawler.base;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.base;
package org.opensearch.dataprepper.plugins.source.source_crawler.base;

/**
* Marker interface to all the SAAS connectors configuration
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.base;
package org.opensearch.dataprepper.plugins.source.source_crawler.base;


import lombok.Getter;
Expand All @@ -14,10 +14,10 @@
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.UsesEnhancedSourceCoordination;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.PartitionFactory;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.LeaderPartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.scheduler.LeaderScheduler;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.scheduler.WorkerScheduler;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.PartitionFactory;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.LeaderPartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.scheduler.LeaderScheduler;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.scheduler.WorkerScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination;

import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.LeaderPartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.LeaderPartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.SaasSourcePartition;

import java.util.function.Function;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition;

import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.LeaderProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.LeaderProgressState;

import java.util.Optional;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition;

import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState;

import java.util.Optional;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.scheduler;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.scheduler;

import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.SaasSourcePlugin;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.LeaderPartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.LeaderProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.SaasSourcePlugin;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.LeaderPartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.LeaderProgressState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.scheduler;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.scheduler;

import org.opensearch.dataprepper.model.buffer.Buffer;
import org.opensearch.dataprepper.model.event.Event;
import org.opensearch.dataprepper.model.record.Record;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.SaasSourceConfig;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.SaasSourceConfig;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state;

import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state;

import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.model;
package org.opensearch.dataprepper.plugins.source.source_crawler.model;


import lombok.Getter;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.base;
package org.opensearch.dataprepper.plugins.source.source_crawler.base;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand All @@ -9,9 +9,9 @@
import org.opensearch.dataprepper.model.event.Event;
import org.opensearch.dataprepper.model.record.Record;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.saas_crawler.model.ItemInfo;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo;

import java.lang.reflect.Field;
import java.util.ArrayList;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.base;
package org.opensearch.dataprepper.plugins.source.source_crawler.base;

import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.base;
package org.opensearch.dataprepper.plugins.source.source_crawler.base;


import org.junit.jupiter.api.BeforeEach;
Expand All @@ -17,7 +17,7 @@
import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.scheduler.LeaderScheduler;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.scheduler.LeaderScheduler;

import java.util.concurrent.ExecutorService;
import java.util.function.Function;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination;


import org.junit.jupiter.api.Test;
Expand All @@ -7,9 +7,9 @@
import org.mockito.junit.jupiter.MockitoExtension;
import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.LeaderPartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.LeaderProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.LeaderPartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.LeaderProgressState;

import java.util.Optional;
import java.util.UUID;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.scheduler;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.scheduler;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
Expand All @@ -7,9 +7,9 @@
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.SaasSourcePlugin;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.LeaderPartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.SaasSourcePlugin;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.LeaderPartition;

import java.time.Duration;
import java.util.Optional;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.scheduler;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.scheduler;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
Expand All @@ -10,11 +10,11 @@
import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.saas_crawler.base.SaasSourceConfig;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.PartitionFactory;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state.SaasWorkerProgressState;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler;
import org.opensearch.dataprepper.plugins.source.source_crawler.base.SaasSourceConfig;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.PartitionFactory;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.partition.SaasSourcePartition;
import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState;

import java.util.Optional;
import java.util.UUID;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.coordination.state;
package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.opensearch.dataprepper.plugins.source.saas_crawler.model;
package org.opensearch.dataprepper.plugins.source.source_crawler.model;

import lombok.NonNull;
import org.junit.jupiter.api.Test;
Expand Down
2 changes: 1 addition & 1 deletion settings.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,6 @@ include 'data-prepper-plugin-schema'
include 'data-prepper-plugins:kinesis-source'
include 'data-prepper-plugins:opensearch-api-source'
include 'data-prepper-plugins:saas-source-plugins'
include 'data-prepper-plugins:saas-source-plugins:saas-crawler'
include 'data-prepper-plugins:saas-source-plugins:source-crawler'
include 'data-prepper-plugins:saas-source-plugins:jira-source'

0 comments on commit 6894d24

Please sign in to comment.