From 4270988d55d1fdc48c54eb4f390ff6f35ff5029a Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 1 Nov 2024 10:33:15 -0700 Subject: [PATCH 1/6] JiraSourceTests Signed-off-by: Maxwell Brown --- .../plugins/source/jira/JiraItemInfoTest.java | 1 - .../plugins/source/jira/JiraSourceTest.java | 67 ++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java index 5a65129e16..608072b1e7 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java @@ -5,7 +5,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.jira.utils.Constants; import java.time.Instant; import java.util.Map; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java index d95235e3ee..560ad8b0d5 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java @@ -6,12 +6,25 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import java.util.concurrent.ExecutorService; + import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; @ExtendWith(MockitoExtension.class) public class JiraSourceTest { @@ -31,15 +44,65 @@ public class JiraSourceTest { @Mock private AcknowledgementSetManager acknowledgementSetManager; - @Mock private Crawler crawler; - private PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + @Mock + private EnhancedSourceCoordinator sourceCooridinator; + + @Mock + Buffer> buffer; + + @Mock + private PluginExecutorServiceProvider executorServiceProvider; + + @Mock + private ExecutorService executorService; +// = new PluginExecutorServiceProvider(); @Test void initialization() { + when(executorServiceProvider.get()).thenReturn(executorService); JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); assertNotNull(source); } + + @Test + void testStart() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(jiraSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); + when(jiraSourceConfig.getJiraId()).thenReturn("Test Id"); + when(jiraSourceConfig.getJiraCredential()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + verify(executorService, atLeast(1)).submit(any(Runnable.class)); + } + + @Test + void testStop() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(jiraSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); + when(jiraSourceConfig.getJiraId()).thenReturn("Test Id"); + when(jiraSourceConfig.getJiraCredential()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + source.stop(); + verify(executorService).shutdownNow(); + } + + @Test + void testStop_WhenNotStarted() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + + source.stop(); + + verify(executorService, never()).shutdown(); + } } From 6d4e3cf086d6fd33ca915e4801ad656d671c0627 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 1 Nov 2024 10:35:49 -0700 Subject: [PATCH 2/6] JiraItemInfo coverage Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/source/jira/JiraItemInfoTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java index 608072b1e7..80aa7a2e1e 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java @@ -11,6 +11,7 @@ import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; @@ -52,6 +53,11 @@ void testGetters() { assertEquals(jiraItemInfo.getEventTime(), eventTime); } + @Test + void testGetKeyAttributes() { + assertInstanceOf(Map.class, jiraItemInfo.getKeyAttributes()); + } + @Test void testSetter() { jiraItemInfo.setEventTime(Instant.now()); From 1600e9c22664d223a60c6305a81dfbd5fae418a0 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 1 Nov 2024 10:59:47 -0700 Subject: [PATCH 3/6] jira service branch coverage Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/source/jira/JiraServiceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java index 4ecb6cb054..3768dadf8f 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java @@ -184,8 +184,9 @@ public void testBadProjectKeys() throws JsonProcessingException { issueType.add("Task"); issueStatus.add("Done"); projectKey.add("Bad Project Key"); - projectKey.add(""); + projectKey.add("A"); projectKey.add("!@#$"); + projectKey.add("AAAAAAAAAAAAAA"); JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); JiraService jiraService = new JiraService(restTemplate, jiraSourceConfig, authConfig); From 3e5d83aa4b0401d6a24d1ec73de440052cad7907 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 1 Nov 2024 11:12:22 -0700 Subject: [PATCH 4/6] branch coverage jira service Signed-off-by: Maxwell Brown --- .../plugins/source/jira/JiraService.java | 2 +- .../plugins/source/jira/JiraServiceTest.java | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java index 967cff44d4..1446591894 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java @@ -177,7 +177,7 @@ private void addItemsToQueue(List issueList, Queue itemInfo } long created = 0; - if (Objects.nonNull(issue.getFields()) && issue.getFields().get(CREATED) + if (Objects.nonNull(issue.getFields().get(CREATED)) && issue.getFields().get(CREATED) .toString().length() >= 23) { String charSequence = issue.getFields().get(CREATED).toString().substring(0, 23) + "Z"; OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java index 3768dadf8f..fa21503cad 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java @@ -132,10 +132,12 @@ public void testGetJiraEntities() throws JsonProcessingException { JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); JiraService jiraService = spy(new JiraService(restTemplate, jiraSourceConfig, authConfig)); List mockIssues = new ArrayList<>(); - IssueBean issue1 = createIssueBean(false); + IssueBean issue1 = createIssueBean(false, false); mockIssues.add(issue1); - IssueBean issue2 = createIssueBean(true); + IssueBean issue2 = createIssueBean(true, false); mockIssues.add(issue2); + IssueBean issue3 = createIssueBean(false, true); + mockIssues.add(issue3); SearchResults mockSearchResults = mock(SearchResults.class); when(mockSearchResults.getIssues()).thenReturn(mockIssues); @@ -146,7 +148,7 @@ public void testGetJiraEntities() throws JsonProcessingException { Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); jiraService.getJiraEntities(jiraSourceConfig, timestamp, itemInfoQueue); - //one additional item is added for the project + assertEquals(mockIssues.size() + 1, itemInfoQueue.size()); } @@ -160,7 +162,7 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep JiraService jiraService = spy(new JiraService(restTemplate, jiraSourceConfig, authConfig)); List mockIssues = new ArrayList<>(); for (int i = 0; i < 50; i++) { - IssueBean issue1 = createIssueBean(false); + IssueBean issue1 = createIssueBean(false, false); mockIssues.add(issue1); } @@ -297,7 +299,7 @@ void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptio assertThrows(expectedExceptionType, () -> jiraService.getIssue("key")); } - private IssueBean createIssueBean(boolean nullFields) { + private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { IssueBean issue1 = new IssueBean(); issue1.setId(UUID.randomUUID().toString()); issue1.setKey("issue_1_key"); @@ -312,6 +314,9 @@ private IssueBean createIssueBean(boolean nullFields) { fieldMap.put(CREATED, 0); fieldMap.put(UPDATED, 0); } + if (createdNull) { + fieldMap.put(CREATED, null); + } Map issueTypeMap = new HashMap<>(); issueTypeMap.put("name", "Task"); From d9c555df869d921a789e3e9d3f63acf342c2273d Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 1 Nov 2024 12:09:27 -0700 Subject: [PATCH 5/6] move add Items to queue logic into JiraItemInfo Signed-off-by: Maxwell Brown --- .../plugins/source/jira/JiraItemInfo.java | 62 +++++++++++++++++++ .../plugins/source/jira/JiraService.java | 50 ++------------- 2 files changed, 68 insertions(+), 44 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java index 9c12ccf0e0..dfcc36867e 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java @@ -2,12 +2,30 @@ import lombok.Getter; import lombok.Setter; +import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean; import org.opensearch.dataprepper.plugins.source.jira.utils.Constants; +import org.opensearch.dataprepper.plugins.source.jira.utils.JiraContentType; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.Date; +import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.UUID; +import java.util.regex.Pattern; + +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CONTENT_TYPE; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE; @Setter @Getter @@ -105,6 +123,50 @@ public JiraItemInfoBuilder withIssueType(String issueType) { this.issueType = issueType; return this; } + + public JiraItemInfoBuilder withIssueBean(IssueBean issue) { + Map issueMetadata = new HashMap<>(); + if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) { + issueMetadata.put(PROJECT_KEY, ((Map) issue.getFields().get(PROJECT)).get(KEY).toString()); + this.project = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString(); + } + + if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(NAME))) { + issueMetadata.put(PROJECT_NAME, ((Map) issue.getFields().get(PROJECT)).get(NAME).toString()); + } + + long created = 0; + Pattern GreaterThanOrEqualTo23 = Pattern.compile("^.{23,}$"); + if (Objects.nonNull(issue.getFields().get(CREATED)) && GreaterThanOrEqualTo23.matcher(issue.getFields().get(CREATED) + .toString()).matches()) { + String charSequence = issue.getFields().get(CREATED).toString().substring(0, 23) + "Z"; + OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence); + new Date(offsetDateTime.toInstant().toEpochMilli()); + created = offsetDateTime.toEpochSecond() * 1000; + } + issueMetadata.put(CREATED, String.valueOf(created)); + + long updated = 0; + if (GreaterThanOrEqualTo23.matcher(issue.getFields().get(UPDATED).toString()).matches()) { + String charSequence = issue.getFields().get(UPDATED).toString().substring(0, 23) + "Z"; + OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence); + new Date(offsetDateTime.toInstant().toEpochMilli()); + updated = offsetDateTime.toEpochSecond() * 1000; + } + issueMetadata.put(UPDATED, String.valueOf(updated)); + + issueMetadata.put(ISSUE_KEY, issue.getKey()); + this.id = issue.getKey(); + + issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType()); + this.issueType = JiraContentType.ISSUE.getType(); + + this.itemId = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey(); + + this.metadata = issueMetadata; + + return this; + } } } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java index 1446591894..896547ec11 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java @@ -23,9 +23,7 @@ import javax.inject.Named; import java.net.URI; import java.time.Instant; -import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,7 +39,6 @@ import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BAD_REQUEST_EXCEPTION; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CLOSING_ROUND_BRACKET; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CONTENT_TYPE; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.DELIMITER; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.EXPAND_FIELD; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.EXPAND_VALUE; @@ -53,13 +50,11 @@ import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.LIVE; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.MAX_RESULT; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PREFIX; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_IN; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.RATE_LIMIT; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.REST_API_FETCH_ISSUE; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.REST_API_SEARCH; @@ -70,7 +65,6 @@ import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.SUFFIX; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.TOKEN_EXPIRED; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._PROJECT; @@ -166,46 +160,14 @@ private void searchForNewTicketsAndAddToQueue(JiraSourceConfig configuration, In */ private void addItemsToQueue(List issueList, Queue itemInfoQueue) { issueList.forEach(issue -> { - Map issueMetadata = new HashMap<>(); - if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) { - issueMetadata.put(PROJECT_KEY, - ((Map) issue.getFields().get(PROJECT)).get(KEY).toString()); - } - if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(NAME))) { - issueMetadata.put(PROJECT_NAME, - ((Map) issue.getFields().get(PROJECT)).get(NAME).toString()); - } - - long created = 0; - if (Objects.nonNull(issue.getFields().get(CREATED)) && issue.getFields().get(CREATED) - .toString().length() >= 23) { - String charSequence = issue.getFields().get(CREATED).toString().substring(0, 23) + "Z"; - OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence); - new Date(offsetDateTime.toInstant().toEpochMilli()); - created = offsetDateTime.toEpochSecond() * 1000; - } - issueMetadata.put(CREATED, String.valueOf(created)); - - long updated = 0; - if (issue.getFields().get(UPDATED).toString().length() >= 23) { - String charSequence = issue.getFields().get(UPDATED).toString().substring(0, 23) + "Z"; - OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence); - new Date(offsetDateTime.toInstant().toEpochMilli()); - updated = offsetDateTime.toEpochSecond() * 1000; - } - issueMetadata.put(UPDATED, String.valueOf(updated)); - - issueMetadata.put(ISSUE_KEY, issue.getKey()); - issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType()); - String id = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey(); - - itemInfoQueue.add(createItemInfo(id, issueMetadata)); + itemInfoQueue.add(JiraItemInfo.builder().withIssueBean(issue).build()); - if (Objects.nonNull(issueMetadata.get(PROJECT_KEY)) && !jiraProjectCache - .containsKey(issueMetadata.get(PROJECT_KEY))) { - jiraProjectCache.put((String) issueMetadata.get(PROJECT_KEY), LIVE); + if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) { + String projectKey = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString(); + if (!jiraProjectCache.containsKey(projectKey)) { + jiraProjectCache.put(projectKey, LIVE); + } } - }); } From bea45c85d8f3543e8be59fc7d560153684db127d Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 1 Nov 2024 13:45:50 -0700 Subject: [PATCH 6/6] fixing regex and adding date time formatter Signed-off-by: Maxwell Brown --- .../plugins/source/jira/JiraItemInfo.java | 17 ++++++++++------- .../plugins/source/jira/JiraService.java | 2 +- .../plugins/source/jira/JiraServiceTest.java | 5 +++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java index dfcc36867e..c601b5ec99 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java @@ -9,6 +9,7 @@ import java.time.Instant; import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -136,20 +137,22 @@ public JiraItemInfoBuilder withIssueBean(IssueBean issue) { } long created = 0; - Pattern GreaterThanOrEqualTo23 = Pattern.compile("^.{23,}$"); - if (Objects.nonNull(issue.getFields().get(CREATED)) && GreaterThanOrEqualTo23.matcher(issue.getFields().get(CREATED) + Pattern JiraDateTimePattern = Pattern.compile( + "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[-+]\\d{4}$"); + DateTimeFormatter offsetDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + if (Objects.nonNull(issue.getFields().get(CREATED)) && JiraDateTimePattern.matcher(issue.getFields().get(CREATED) .toString()).matches()) { - String charSequence = issue.getFields().get(CREATED).toString().substring(0, 23) + "Z"; - OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence); + String charSequence = issue.getFields().get(CREATED).toString(); + OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter); new Date(offsetDateTime.toInstant().toEpochMilli()); created = offsetDateTime.toEpochSecond() * 1000; } issueMetadata.put(CREATED, String.valueOf(created)); long updated = 0; - if (GreaterThanOrEqualTo23.matcher(issue.getFields().get(UPDATED).toString()).matches()) { - String charSequence = issue.getFields().get(UPDATED).toString().substring(0, 23) + "Z"; - OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence); + if (JiraDateTimePattern.matcher(issue.getFields().get(UPDATED).toString()).matches()) { + String charSequence = issue.getFields().get(UPDATED).toString(); + OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter); new Date(offsetDateTime.toInstant().toEpochMilli()); updated = offsetDateTime.toEpochSecond() * 1000; } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java index 896547ec11..ee298240bf 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java @@ -160,7 +160,7 @@ private void searchForNewTicketsAndAddToQueue(JiraSourceConfig configuration, In */ private void addItemsToQueue(List issueList, Queue itemInfoQueue) { issueList.forEach(issue -> { - itemInfoQueue.add(JiraItemInfo.builder().withIssueBean(issue).build()); + itemInfoQueue.add(JiraItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build()); if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) { String projectKey = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString(); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java index fa21503cad..c1d2c67e39 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java @@ -308,8 +308,9 @@ private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { Map fieldMap = new HashMap<>(); if (!nullFields) { - fieldMap.put(CREATED, Instant.now()); - fieldMap.put(UPDATED, Instant.now()); + fieldMap.put(CREATED, "2024-07-06T21:12:23.437-0700"); + fieldMap.put(UPDATED, "2024-07-06T21:12:23.106-0700"); +// fieldMap.put(UPDATED, Instant.now()); } else { fieldMap.put(CREATED, 0); fieldMap.put(UPDATED, 0);