diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java index 9c12ccf0e0..9c77c43527 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java @@ -2,12 +2,31 @@ import lombok.Getter; import lombok.Setter; +import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean; import org.opensearch.dataprepper.plugins.source.jira.utils.Constants; +import org.opensearch.dataprepper.plugins.source.jira.utils.JiraContentType; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.UUID; +import java.util.regex.Pattern; + +import static org.opensearch.dataprepper.plugins.source.jira.JiraService.CONTENT_TYPE; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE; @Setter @Getter @@ -105,6 +124,52 @@ public JiraItemInfoBuilder withIssueType(String issueType) { this.issueType = issueType; return this; } + + public JiraItemInfoBuilder withIssueBean(IssueBean issue) { + Map issueMetadata = new HashMap<>(); + if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) { + issueMetadata.put(PROJECT_KEY, ((Map) issue.getFields().get(PROJECT)).get(KEY).toString()); + this.project = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString(); + } + + if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(NAME))) { + issueMetadata.put(PROJECT_NAME, ((Map) issue.getFields().get(PROJECT)).get(NAME).toString()); + } + + long created = 0; + Pattern JiraDateTimePattern = Pattern.compile( + "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[-+]\\d{4}$"); + DateTimeFormatter offsetDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + if (Objects.nonNull(issue.getFields().get(CREATED)) && JiraDateTimePattern.matcher(issue.getFields().get(CREATED) + .toString()).matches()) { + String charSequence = issue.getFields().get(CREATED).toString(); + OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter); + new Date(offsetDateTime.toInstant().toEpochMilli()); + created = offsetDateTime.toEpochSecond() * 1000; + } + issueMetadata.put(CREATED, String.valueOf(created)); + + long updated = 0; + if (JiraDateTimePattern.matcher(issue.getFields().get(UPDATED).toString()).matches()) { + String charSequence = issue.getFields().get(UPDATED).toString(); + OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter); + new Date(offsetDateTime.toInstant().toEpochMilli()); + updated = offsetDateTime.toEpochSecond() * 1000; + } + issueMetadata.put(UPDATED, String.valueOf(updated)); + + issueMetadata.put(ISSUE_KEY, issue.getKey()); + this.id = issue.getKey(); + + issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType()); + this.issueType = JiraContentType.ISSUE.getType(); + + this.itemId = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey(); + + this.metadata = issueMetadata; + + return this; + } } -} +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java index 8a3b030d68..156f3fe0cf 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java @@ -14,9 +14,7 @@ import javax.inject.Named; import java.time.Instant; -import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,16 +25,12 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.LIVE; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._PROJECT; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.CLOSING_ROUND_BRACKET; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.DELIMITER; @@ -129,46 +123,14 @@ private void searchForNewTicketsAndAddToQueue(JiraSourceConfig configuration, In */ private void addItemsToQueue(List issueList, Queue itemInfoQueue) { issueList.forEach(issue -> { - Map issueMetadata = new HashMap<>(); - if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) { - issueMetadata.put(PROJECT_KEY, - ((Map) issue.getFields().get(PROJECT)).get(KEY).toString()); - } - if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(NAME))) { - issueMetadata.put(PROJECT_NAME, - ((Map) issue.getFields().get(PROJECT)).get(NAME).toString()); - } - - long created = 0; - if (Objects.nonNull(issue.getFields()) && issue.getFields().get(CREATED) - .toString().length() >= 23) { - String charSequence = issue.getFields().get(CREATED).toString().substring(0, 23) + "Z"; - OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence); - new Date(offsetDateTime.toInstant().toEpochMilli()); - created = offsetDateTime.toEpochSecond() * 1000; - } - issueMetadata.put(CREATED, String.valueOf(created)); - - long updated = 0; - if (issue.getFields().get(UPDATED).toString().length() >= 23) { - String charSequence = issue.getFields().get(UPDATED).toString().substring(0, 23) + "Z"; - OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence); - new Date(offsetDateTime.toInstant().toEpochMilli()); - updated = offsetDateTime.toEpochSecond() * 1000; - } - issueMetadata.put(UPDATED, String.valueOf(updated)); - - issueMetadata.put(ISSUE_KEY, issue.getKey()); - issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType()); - String id = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey(); - - itemInfoQueue.add(createItemInfo(id, issueMetadata)); + itemInfoQueue.add(JiraItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build()); - if (Objects.nonNull(issueMetadata.get(PROJECT_KEY)) && !jiraProjectCache - .containsKey(issueMetadata.get(PROJECT_KEY))) { - jiraProjectCache.put((String) issueMetadata.get(PROJECT_KEY), LIVE); + if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) { + String projectKey = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString(); + if (!jiraProjectCache.containsKey(projectKey)) { + jiraProjectCache.put(projectKey, LIVE); + } } - }); } @@ -247,4 +209,4 @@ private ItemInfo createItemInfo(String key, Map metadata) { .build(); } -} +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java index 5a65129e16..49dc6873bd 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java @@ -5,13 +5,13 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.jira.utils.Constants; import java.time.Instant; import java.util.Map; import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; @@ -53,6 +53,11 @@ void testGetters() { assertEquals(jiraItemInfo.getEventTime(), eventTime); } + @Test + void testGetKeyAttributes() { + assertInstanceOf(Map.class, jiraItemInfo.getKeyAttributes()); + } + @Test void testSetter() { jiraItemInfo.setEventTime(Instant.now()); @@ -89,4 +94,4 @@ void testGetLastModifiedAt() { assertEquals(Instant.ofEpochMilli(7), jiraItemInfo.getLastModifiedAt()); } -} +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java index 149396d1dc..1e791988cb 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java @@ -34,6 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -112,6 +113,8 @@ void testJiraServiceInitialization() throws JsonProcessingException { JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient); assertNotNull(jiraService); + when(jiraRestClient.getIssue(anyString())).thenReturn("test String"); + assertNotNull(jiraService.getIssue("test Key")); } @Test @@ -125,10 +128,12 @@ public void testGetJiraEntities() throws JsonProcessingException { JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); List mockIssues = new ArrayList<>(); - IssueBean issue1 = createIssueBean(false); + IssueBean issue1 = createIssueBean(false, false); mockIssues.add(issue1); - IssueBean issue2 = createIssueBean(true); + IssueBean issue2 = createIssueBean(true, false); mockIssues.add(issue2); + IssueBean issue3 = createIssueBean(false, true); + mockIssues.add(issue3); SearchResults mockSearchResults = mock(SearchResults.class); when(mockSearchResults.getIssues()).thenReturn(mockIssues); @@ -139,7 +144,7 @@ public void testGetJiraEntities() throws JsonProcessingException { Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); jiraService.getJiraEntities(jiraSourceConfig, timestamp, itemInfoQueue); - //one additional item is added for the project + assertEquals(mockIssues.size() + 1, itemInfoQueue.size()); } @@ -153,7 +158,7 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); List mockIssues = new ArrayList<>(); for (int i = 0; i < 50; i++) { - IssueBean issue1 = createIssueBean(false); + IssueBean issue1 = createIssueBean(false, false); mockIssues.add(issue1); } @@ -177,8 +182,9 @@ public void testBadProjectKeys() throws JsonProcessingException { issueType.add("Task"); issueStatus.add("Done"); projectKey.add("Bad Project Key"); - projectKey.add(""); + projectKey.add("A"); projectKey.add("!@#$"); + projectKey.add("AAAAAAAAAAAAAA"); JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient); @@ -204,7 +210,8 @@ public void testGetJiraEntitiesException() throws JsonProcessingException { assertThrows(RuntimeException.class, () -> jiraService.getJiraEntities(jiraSourceConfig, timestamp, itemInfoQueue)); } - private IssueBean createIssueBean(boolean nullFields) { + + private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { IssueBean issue1 = new IssueBean(); issue1.setId(UUID.randomUUID().toString()); issue1.setKey("issue_1_key"); @@ -213,12 +220,15 @@ private IssueBean createIssueBean(boolean nullFields) { Map fieldMap = new HashMap<>(); if (!nullFields) { - fieldMap.put(CREATED, Instant.now()); - fieldMap.put(UPDATED, Instant.now()); + fieldMap.put(CREATED, "2024-07-06T21:12:23.437-0700"); + fieldMap.put(UPDATED, "2024-07-06T21:12:23.106-0700"); } else { fieldMap.put(CREATED, 0); fieldMap.put(UPDATED, 0); } + if (createdNull) { + fieldMap.put(CREATED, null); + } Map issueTypeMap = new HashMap<>(); issueTypeMap.put("name", "Task"); @@ -246,4 +256,4 @@ private IssueBean createIssueBean(boolean nullFields) { return issue1; } -} +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java index d95235e3ee..46cf58b7a9 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java @@ -6,12 +6,25 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import java.util.concurrent.ExecutorService; + import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraOauthConfig.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; @ExtendWith(MockitoExtension.class) public class JiraSourceTest { @@ -31,15 +44,65 @@ public class JiraSourceTest { @Mock private AcknowledgementSetManager acknowledgementSetManager; - @Mock private Crawler crawler; - private PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + @Mock + private EnhancedSourceCoordinator sourceCooridinator; + + @Mock + Buffer> buffer; + + @Mock + private PluginExecutorServiceProvider executorServiceProvider; + + @Mock + private ExecutorService executorService; +// = new PluginExecutorServiceProvider(); @Test void initialization() { + when(executorServiceProvider.get()).thenReturn(executorService); JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); assertNotNull(source); } -} + + @Test + void testStart() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(jiraSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); + when(jiraSourceConfig.getJiraId()).thenReturn("Test Id"); + when(jiraSourceConfig.getJiraCredential()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + verify(executorService, atLeast(1)).submit(any(Runnable.class)); + } + + @Test + void testStop() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(jiraSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); + when(jiraSourceConfig.getJiraId()).thenReturn("Test Id"); + when(jiraSourceConfig.getJiraCredential()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + source.stop(); + verify(executorService).shutdownNow(); + } + + @Test + void testStop_WhenNotStarted() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + + source.stop(); + + verify(executorService, never()).shutdown(); + } +} \ No newline at end of file