Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

full test coverage for base folder, spotless fixes #6

Merged
merged 1 commit into from
Oct 21, 2024
Merged

full test coverage for base folder, spotless fixes #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion data-prepper-plugins/saas-source-plugins/jira-source/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,3 @@

### Timer
- `requestProcessDuration`: measures latency of requests processed by the jira source plugin.

14 changes: 11 additions & 3 deletions ...earch/dataprepper/plugins/source/saas/crawler/base/SaasPluginExecutorServiceProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.PreDestroy;
import javax.inject.Named;
import javax.annotation.PreDestroy;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
Expand All @@ -13,8 +13,16 @@
public class SaasPluginExecutorServiceProvider {
Logger log = LoggerFactory.getLogger(SaasPluginExecutorServiceProvider.class);
public static final int DEFAULT_THREAD_COUNT = 50;
private final ExecutorService executorService;

private final ExecutorService executorService = Executors.newFixedThreadPool(DEFAULT_THREAD_COUNT);
public SaasPluginExecutorServiceProvider() {
executorService = Executors.newFixedThreadPool(DEFAULT_THREAD_COUNT);
}

//Constructor for testing
public SaasPluginExecutorServiceProvider(ExecutorService testExecutorService) {
executorService = testExecutorService;
}

public ExecutorService get() {
return executorService;
Expand All @@ -35,4 +43,4 @@ public void terminateExecutor() {
executorService.shutdownNow();
}
}
}
}
90 changes: 83 additions & 7 deletions ...rc/test/java/org/opensearch/dataprepper/plugins/source/saas/crawler/base/CrawlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
Expand Down Expand Up @@ -45,10 +47,8 @@ public class CrawlerTest {
@Mock
private ItemInfo item;


private Crawler crawler;


@BeforeEach
public void setup() {
crawler = new Crawler(client);
Expand Down Expand Up @@ -101,10 +101,86 @@ void testCrawlWithMultiplePartitions() throws NoSuchFieldException, IllegalAcce

}

@Test
void testCrawlWithNullItemsInList() throws NoSuchFieldException, IllegalAccessException {
long lastPollTime = 0;
List<ItemInfo> itemInfoList = new ArrayList<>();
int maxItemsPerPage = getMaxItemsPerPage();
itemInfoList.add(null);
for (int i = 0; i < maxItemsPerPage-1; i++) {
itemInfoList.add(item);
}
when(client.listItems()).thenReturn(itemInfoList.iterator());
crawler.crawl(lastPollTime, coordinator);
verify(coordinator, times(1)).createPartition(any(SaasSourcePartition.class));
}

@Test
void testUpdatingPollTimeNullMetaData() {
long lastPollTime = 0;
List<ItemInfo> itemInfoList = new ArrayList<>();
ItemInfo testItem = createTestItemInfo("1",null, null);
itemInfoList.add(testItem);
when(client.listItems()).thenReturn(itemInfoList.iterator());
long updatedPollTime = crawler.crawl(lastPollTime, coordinator);
assert(updatedPollTime != 0);
}

@Test
void testUpdatedPollTimeNiCreatedLarger() {
long lastPollTime = 0;
List<ItemInfo> itemInfoList = new ArrayList<>();
ItemInfo testItem = createTestItemInfo("1","10" , "5");
itemInfoList.add(testItem);
when(client.listItems()).thenReturn(itemInfoList.iterator());
long updatedPollTime = crawler.crawl(lastPollTime, coordinator);
assert(updatedPollTime == 11);
}
@Test
void testUpdatedPollTimeNiUpdatedLarger() {
long lastPollTime = 0;
List<ItemInfo> itemInfoList = new ArrayList<>();
ItemInfo testItem = createTestItemInfo("1","5" , "10");
itemInfoList.add(testItem);
when(client.listItems()).thenReturn(itemInfoList.iterator());
long updatedPollTime = crawler.crawl(lastPollTime, coordinator);
assert(updatedPollTime == 11);
}


private int getMaxItemsPerPage() throws NoSuchFieldException, IllegalAccessException {
Field maxItemsPerPageField = Crawler.class.getDeclaredField("maxItemsPerPage");
maxItemsPerPageField.setAccessible(true);
return (int) maxItemsPerPageField.get(null);
}
Field maxItemsPerPageField = Crawler.class.getDeclaredField("maxItemsPerPage");
maxItemsPerPageField.setAccessible(true);
return (int) maxItemsPerPageField.get(null);
}

private static class TestItemInfo extends ItemInfo {
public TestItemInfo(String itemId, Map<String, String> metadata, Long eventTime) {
super(itemId, metadata, eventTime);

}
@Override
public String getPartitionKey() {
return getItemId();
}

@Override
public String getId() {
return getItemId();
}

@Override
public Map<String, String> getKeyAttributes() {
return new HashMap<>();
}
}

private ItemInfo createTestItemInfo(String id, String created, String updated) {
Map<String, String> metadata = new HashMap<>();
if (created != null) metadata.put(Crawler.CREATED, created);
if (updated != null) metadata.put(Crawler.UPDATED, updated);

return new TestItemInfo(id, metadata, System.currentTimeMillis());
}

}
}
29 changes: 27 additions & 2 deletions ...h/dataprepper/plugins/source/saas/crawler/base/SaasPluginExecutorServiceProviderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,29 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
public class SaasPluginExecutorServiceProviderTest {

private SaasPluginExecutorServiceProvider provider;
private ExecutorService executorService;

private SaasPluginExecutorServiceProvider provider2;
@Mock
private ExecutorService mockExecutorService;

@BeforeEach
void setUp() {
provider = new SaasPluginExecutorServiceProvider();
Expand All @@ -34,11 +44,26 @@ void testConstruction() {
assertNotNull(provider);
}


@Test
void testTerminateExecutor() {
provider.terminateExecutor();
assertTrue(executorService.isShutdown());
assertTrue(executorService.isTerminated());
}
}

@Test
void terminateExecutorInterruptionTest() throws InterruptedException {
provider2 = new SaasPluginExecutorServiceProvider(mockExecutorService);
when(mockExecutorService.awaitTermination(anyLong(),any(TimeUnit.class))).thenThrow(new InterruptedException());
AtomicBoolean wasInterrupted = new AtomicBoolean(false);

Thread testThread = new Thread(() -> {
provider2.terminateExecutor();
wasInterrupted.set(Thread.interrupted());
});
testThread.start();
testThread.join();

assertTrue(wasInterrupted.get());
}
}
36 changes: 30 additions & 6 deletions ...ava/org/opensearch/dataprepper/plugins/source/saas/crawler/base/SaasSourcePluginTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@


import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
Expand Down Expand Up @@ -55,21 +56,33 @@ public class SaasSourcePluginTest {
@Mock
private AcknowledgementSetManager acknowledgementSetManager;

@Mock
SourcePartitionStoreItem mockItem;

@Mock
EnhancedSourcePartition mockPartition;

@Mock
private EnhancedSourceCoordinator sourceCoordinator;

private SaasSourcePlugin saasSourcePlugin;
private testSaasSourcePlugin saasSourcePlugin;

static class TestSaasSourcePlugin extends SaasSourcePlugin {
public TestSaasSourcePlugin(PluginMetrics pluginMetrics, SaasSourceConfig sourceConfig, PluginFactory pluginFactory, AcknowledgementSetManager acknowledgementSetManager, Crawler crawler, SaasPluginExecutorServiceProvider executorServiceProvider) {
@Nested
public class testSaasSourcePlugin extends SaasSourcePlugin {
public testSaasSourcePlugin(final PluginMetrics pluginMetrics,
final SaasSourceConfig sourceConfig,
final PluginFactory pluginFactory,
final AcknowledgementSetManager acknowledgementSetManager,
final Crawler crawler,
final SaasPluginExecutorServiceProvider executorServiceProvider) {
super(pluginMetrics, sourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider);
}
}

@BeforeEach
void setUp() {
when(executorServiceProvider.get()).thenReturn(executorService);
saasSourcePlugin = new TestSaasSourcePlugin(pluginMetrics, sourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider);
saasSourcePlugin = new testSaasSourcePlugin(pluginMetrics, sourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider);
}

@Test
Expand All @@ -84,6 +97,16 @@ void testSetEnhancedSourceCoordinator() {
verify(sourceCoordinator).initialize();
}

@Test
void areAcknowledgementsEnabledTest() {
assertFalse(saasSourcePlugin.areAcknowledgementsEnabled());
}

@Test
void saasSourceConfigGetterTest() {
assertNotNull(saasSourcePlugin.getSourceConfig());
}

@Test
void startTest() {
saasSourcePlugin.setEnhancedSourceCoordinator(sourceCoordinator);
Expand All @@ -93,7 +116,7 @@ void startTest() {
}

@Test
void testExecutorServiceSchedulersSubmitted(){
void testExecutorServiceSchedulersSubmitted() {
saasSourcePlugin.setEnhancedSourceCoordinator(sourceCoordinator);
saasSourcePlugin.start(buffer);
verify(executorService, times(1)).submit(any(LeaderScheduler.class));
Expand Down Expand Up @@ -121,4 +144,5 @@ void testGetDecoder() {

}

}

}
Loading