Skip to content

Commit

Permalink
re add changes and fix issues
Browse files Browse the repository at this point in the history 
Signed-off-by: Maxwell Brown <[email protected]>
  • Loading branch information
@Galactus22625
Galactus22625 committed Nov 1, 2024
1 parent a8c3218 commit 3c15223
Show file tree
Hide file tree
Showing 5 changed files with 166 additions and 60 deletions.
67 changes: 66 additions & 1 deletion ...ira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,31 @@

import lombok.Getter;
import lombok.Setter;
import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean;
import org.opensearch.dataprepper.plugins.source.jira.utils.Constants;
import org.opensearch.dataprepper.plugins.source.jira.utils.JiraContentType;
import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo;

import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.regex.Pattern;

import static org.opensearch.dataprepper.plugins.source.jira.JiraService.CONTENT_TYPE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE;

@Setter
@Getter
Expand Down Expand Up @@ -105,6 +124,52 @@ public JiraItemInfoBuilder withIssueType(String issueType) {
this.issueType = issueType;
return this;
}

public JiraItemInfoBuilder withIssueBean(IssueBean issue) {
Map<String, Object> issueMetadata = new HashMap<>();
if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) {
issueMetadata.put(PROJECT_KEY, ((Map) issue.getFields().get(PROJECT)).get(KEY).toString());
this.project = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString();
}

if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(NAME))) {
issueMetadata.put(PROJECT_NAME, ((Map) issue.getFields().get(PROJECT)).get(NAME).toString());
}

long created = 0;
Pattern JiraDateTimePattern = Pattern.compile(
"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[-+]\\d{4}$");
DateTimeFormatter offsetDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
if (Objects.nonNull(issue.getFields().get(CREATED)) && JiraDateTimePattern.matcher(issue.getFields().get(CREATED)
.toString()).matches()) {
String charSequence = issue.getFields().get(CREATED).toString();
OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter);
new Date(offsetDateTime.toInstant().toEpochMilli());
created = offsetDateTime.toEpochSecond() * 1000;
}
issueMetadata.put(CREATED, String.valueOf(created));

long updated = 0;
if (JiraDateTimePattern.matcher(issue.getFields().get(UPDATED).toString()).matches()) {
String charSequence = issue.getFields().get(UPDATED).toString();
OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter);
new Date(offsetDateTime.toInstant().toEpochMilli());
updated = offsetDateTime.toEpochSecond() * 1000;
}
issueMetadata.put(UPDATED, String.valueOf(updated));

issueMetadata.put(ISSUE_KEY, issue.getKey());
this.id = issue.getKey();

issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType());
this.issueType = JiraContentType.ISSUE.getType();

this.itemId = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey();

this.metadata = issueMetadata;

return this;
}
}

}
}
52 changes: 7 additions & 45 deletions ...jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@

import javax.inject.Named;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -27,16 +25,12 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.LIVE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._PROJECT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.CLOSING_ROUND_BRACKET;
import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.DELIMITER;
Expand Down Expand Up @@ -129,46 +123,14 @@ private void searchForNewTicketsAndAddToQueue(JiraSourceConfig configuration, In
*/
private void addItemsToQueue(List<IssueBean> issueList, Queue<ItemInfo> itemInfoQueue) {
issueList.forEach(issue -> {
Map<String, Object> issueMetadata = new HashMap<>();
if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) {
issueMetadata.put(PROJECT_KEY,
((Map) issue.getFields().get(PROJECT)).get(KEY).toString());
}
if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(NAME))) {
issueMetadata.put(PROJECT_NAME,
((Map) issue.getFields().get(PROJECT)).get(NAME).toString());
}

long created = 0;
if (Objects.nonNull(issue.getFields()) && issue.getFields().get(CREATED)
.toString().length() >= 23) {
String charSequence = issue.getFields().get(CREATED).toString().substring(0, 23) + "Z";
OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence);
new Date(offsetDateTime.toInstant().toEpochMilli());
created = offsetDateTime.toEpochSecond() * 1000;
}
issueMetadata.put(CREATED, String.valueOf(created));

long updated = 0;
if (issue.getFields().get(UPDATED).toString().length() >= 23) {
String charSequence = issue.getFields().get(UPDATED).toString().substring(0, 23) + "Z";
OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence);
new Date(offsetDateTime.toInstant().toEpochMilli());
updated = offsetDateTime.toEpochSecond() * 1000;
}
issueMetadata.put(UPDATED, String.valueOf(updated));

issueMetadata.put(ISSUE_KEY, issue.getKey());
issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType());
String id = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey();

itemInfoQueue.add(createItemInfo(id, issueMetadata));
itemInfoQueue.add(JiraItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build());

if (Objects.nonNull(issueMetadata.get(PROJECT_KEY)) && !jiraProjectCache
.containsKey(issueMetadata.get(PROJECT_KEY))) {
jiraProjectCache.put((String) issueMetadata.get(PROJECT_KEY), LIVE);
if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) {
String projectKey = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString();
if (!jiraProjectCache.containsKey(projectKey)) {
jiraProjectCache.put(projectKey, LIVE);
}
}

});
}

Expand Down Expand Up @@ -247,4 +209,4 @@ private ItemInfo createItemInfo(String key, Map<String, Object> metadata) {
.build();
}

}
}
9 changes: 7 additions & 2 deletions ...source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opensearch.dataprepper.plugins.source.jira.utils.Constants;

import java.time.Instant;
import java.util.Map;
import java.util.UUID;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
Expand Down Expand Up @@ -53,6 +53,11 @@ void testGetters() {
assertEquals(jiraItemInfo.getEventTime(), eventTime);
}

@Test
void testGetKeyAttributes() {
assertInstanceOf(Map.class, jiraItemInfo.getKeyAttributes());
}

@Test
void testSetter() {
jiraItemInfo.setEventTime(Instant.now());
Expand Down Expand Up @@ -89,4 +94,4 @@ void testGetLastModifiedAt() {
assertEquals(Instant.ofEpochMilli(7), jiraItemInfo.getLastModifiedAt());
}

}
}
29 changes: 20 additions & 9 deletions ...-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
Expand Down Expand Up @@ -112,6 +113,8 @@ void testJiraServiceInitialization() throws JsonProcessingException {
JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey);
JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient);
assertNotNull(jiraService);
when(jiraRestClient.getIssue(anyString())).thenReturn("test String");
assertNotNull(jiraService.getIssue("test Key"));
}

@Test
Expand All @@ -125,10 +128,12 @@ public void testGetJiraEntities() throws JsonProcessingException {
JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey);
JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient));
List<IssueBean> mockIssues = new ArrayList<>();
IssueBean issue1 = createIssueBean(false);
IssueBean issue1 = createIssueBean(false, false);
mockIssues.add(issue1);
IssueBean issue2 = createIssueBean(true);
IssueBean issue2 = createIssueBean(true, false);
mockIssues.add(issue2);
IssueBean issue3 = createIssueBean(false, true);
mockIssues.add(issue3);

SearchResults mockSearchResults = mock(SearchResults.class);
when(mockSearchResults.getIssues()).thenReturn(mockIssues);
Expand All @@ -139,7 +144,7 @@ public void testGetJiraEntities() throws JsonProcessingException {
Instant timestamp = Instant.ofEpochSecond(0);
Queue<ItemInfo> itemInfoQueue = new ConcurrentLinkedQueue<>();
jiraService.getJiraEntities(jiraSourceConfig, timestamp, itemInfoQueue);
//one additional item is added for the project

assertEquals(mockIssues.size() + 1, itemInfoQueue.size());
}

Expand All @@ -153,7 +158,7 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep
JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient));
List<IssueBean> mockIssues = new ArrayList<>();
for (int i = 0; i < 50; i++) {
IssueBean issue1 = createIssueBean(false);
IssueBean issue1 = createIssueBean(false, false);
mockIssues.add(issue1);
}

Expand All @@ -177,8 +182,9 @@ public void testBadProjectKeys() throws JsonProcessingException {
issueType.add("Task");
issueStatus.add("Done");
projectKey.add("Bad Project Key");
projectKey.add("");
projectKey.add("A");
projectKey.add("!@#$");
projectKey.add("AAAAAAAAAAAAAA");

JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey);
JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient);
Expand All @@ -204,7 +210,8 @@ public void testGetJiraEntitiesException() throws JsonProcessingException {
assertThrows(RuntimeException.class, () -> jiraService.getJiraEntities(jiraSourceConfig, timestamp, itemInfoQueue));
}

private IssueBean createIssueBean(boolean nullFields) {

private IssueBean createIssueBean(boolean nullFields, boolean createdNull) {
IssueBean issue1 = new IssueBean();
issue1.setId(UUID.randomUUID().toString());
issue1.setKey("issue_1_key");
Expand All @@ -213,12 +220,16 @@ private IssueBean createIssueBean(boolean nullFields) {

Map<String, Object> fieldMap = new HashMap<>();
if (!nullFields) {
fieldMap.put(CREATED, Instant.now());
fieldMap.put(UPDATED, Instant.now());
fieldMap.put(CREATED, "2024-07-06T21:12:23.437-0700");
fieldMap.put(UPDATED, "2024-07-06T21:12:23.106-0700");
// fieldMap.put(UPDATED, Instant.now());
} else {
fieldMap.put(CREATED, 0);
fieldMap.put(UPDATED, 0);
}
if (createdNull) {
fieldMap.put(CREATED, null);
}

Map<String, Object> issueTypeMap = new HashMap<>();
issueTypeMap.put("name", "Task");
Expand Down Expand Up @@ -246,4 +257,4 @@ private IssueBean createIssueBean(boolean nullFields) {
return issue1;
}

}
}
Loading

0 comments on commit 3c15223

Please sign in to comment.