Skip to content

Commit

Permalink
Merge pull request #13 from san81/revert-12-jira-source
Browse files Browse the repository at this point in the history 
Revert "Jira source"
  • Loading branch information
@Galactus22625
Galactus22625 authored Nov 1, 2024
2 parents 6e70427 + 13f56d2 commit a8c3218
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 267 deletions.
65 changes: 0 additions & 65 deletions ...ira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,12 @@

import lombok.Getter;
import lombok.Setter;
import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean;
import org.opensearch.dataprepper.plugins.source.jira.utils.Constants;
import org.opensearch.dataprepper.plugins.source.jira.utils.JiraContentType;
import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo;

import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.regex.Pattern;

import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CONTENT_TYPE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE;

@Setter
@Getter
Expand Down Expand Up @@ -124,52 +105,6 @@ public JiraItemInfoBuilder withIssueType(String issueType) {
this.issueType = issueType;
return this;
}

public JiraItemInfoBuilder withIssueBean(IssueBean issue) {
Map<String, Object> issueMetadata = new HashMap<>();
if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) {
issueMetadata.put(PROJECT_KEY, ((Map) issue.getFields().get(PROJECT)).get(KEY).toString());
this.project = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString();
}

if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(NAME))) {
issueMetadata.put(PROJECT_NAME, ((Map) issue.getFields().get(PROJECT)).get(NAME).toString());
}

long created = 0;
Pattern JiraDateTimePattern = Pattern.compile(
"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[-+]\\d{4}$");
DateTimeFormatter offsetDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
if (Objects.nonNull(issue.getFields().get(CREATED)) && JiraDateTimePattern.matcher(issue.getFields().get(CREATED)
.toString()).matches()) {
String charSequence = issue.getFields().get(CREATED).toString();
OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter);
new Date(offsetDateTime.toInstant().toEpochMilli());
created = offsetDateTime.toEpochSecond() * 1000;
}
issueMetadata.put(CREATED, String.valueOf(created));

long updated = 0;
if (JiraDateTimePattern.matcher(issue.getFields().get(UPDATED).toString()).matches()) {
String charSequence = issue.getFields().get(UPDATED).toString();
OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter);
new Date(offsetDateTime.toInstant().toEpochMilli());
updated = offsetDateTime.toEpochSecond() * 1000;
}
issueMetadata.put(UPDATED, String.valueOf(updated));

issueMetadata.put(ISSUE_KEY, issue.getKey());
this.id = issue.getKey();

issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType());
this.issueType = JiraContentType.ISSUE.getType();

this.itemId = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey();

this.metadata = issueMetadata;

return this;
}
}

}
71 changes: 41 additions & 30 deletions ...jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@

import javax.inject.Named;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -26,38 +28,15 @@
import java.util.stream.Collectors;

import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED;
import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.AUTHORIZATION_ERROR_CODE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BAD_REQUEST_EXCEPTION;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CLOSING_ROUND_BRACKET;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CONTENT_TYPE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.DELIMITER;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.EXPAND_FIELD;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.EXPAND_VALUE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.FIFTY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.GREATER_THAN_EQUALS;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.LIVE;

import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.MAX_RESULT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PREFIX;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY;

import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.RATE_LIMIT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.REST_API_FETCH_ISSUE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.REST_API_SEARCH;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.RETRY_ATTEMPT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.RETRY_ATTEMPT_SLEEP_TIME;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.START_AT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.STATUS_IN;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.SUFFIX;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.TOKEN_EXPIRED;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._PROJECT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.CLOSING_ROUND_BRACKET;
import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.DELIMITER;
Expand Down Expand Up @@ -150,14 +129,46 @@ private void searchForNewTicketsAndAddToQueue(JiraSourceConfig configuration, In
*/
private void addItemsToQueue(List<IssueBean> issueList, Queue<ItemInfo> itemInfoQueue) {
issueList.forEach(issue -> {
itemInfoQueue.add(JiraItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build());

Map<String, Object> issueMetadata = new HashMap<>();
if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) {
String projectKey = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString();
if (!jiraProjectCache.containsKey(projectKey)) {
jiraProjectCache.put(projectKey, LIVE);
}
issueMetadata.put(PROJECT_KEY,
((Map) issue.getFields().get(PROJECT)).get(KEY).toString());
}
if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(NAME))) {
issueMetadata.put(PROJECT_NAME,
((Map) issue.getFields().get(PROJECT)).get(NAME).toString());
}

long created = 0;
if (Objects.nonNull(issue.getFields()) && issue.getFields().get(CREATED)
.toString().length() >= 23) {
String charSequence = issue.getFields().get(CREATED).toString().substring(0, 23) + "Z";
OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence);
new Date(offsetDateTime.toInstant().toEpochMilli());
created = offsetDateTime.toEpochSecond() * 1000;
}
issueMetadata.put(CREATED, String.valueOf(created));

long updated = 0;
if (issue.getFields().get(UPDATED).toString().length() >= 23) {
String charSequence = issue.getFields().get(UPDATED).toString().substring(0, 23) + "Z";
OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence);
new Date(offsetDateTime.toInstant().toEpochMilli());
updated = offsetDateTime.toEpochSecond() * 1000;
}
issueMetadata.put(UPDATED, String.valueOf(updated));

issueMetadata.put(ISSUE_KEY, issue.getKey());
issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType());
String id = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey();

itemInfoQueue.add(createItemInfo(id, issueMetadata));

if (Objects.nonNull(issueMetadata.get(PROJECT_KEY)) && !jiraProjectCache
.containsKey(issueMetadata.get(PROJECT_KEY))) {
jiraProjectCache.put((String) issueMetadata.get(PROJECT_KEY), LIVE);
}

});
}

Expand Down
7 changes: 1 addition & 6 deletions ...source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opensearch.dataprepper.plugins.source.jira.utils.Constants;

import java.time.Instant;
import java.util.Map;
import java.util.UUID;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
Expand Down Expand Up @@ -53,11 +53,6 @@ void testGetters() {
assertEquals(jiraItemInfo.getEventTime(), eventTime);
}

@Test
void testGetKeyAttributes() {
assertInstanceOf(Map.class, jiraItemInfo.getKeyAttributes());
}

@Test
void testSetter() {
jiraItemInfo.setEventTime(Instant.now());
Expand Down
109 changes: 8 additions & 101 deletions ...-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,12 +125,10 @@ public void testGetJiraEntities() throws JsonProcessingException {
JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey);
JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient));
List<IssueBean> mockIssues = new ArrayList<>();
IssueBean issue1 = createIssueBean(false, false);
IssueBean issue1 = createIssueBean(false);
mockIssues.add(issue1);
IssueBean issue2 = createIssueBean(true, false);
IssueBean issue2 = createIssueBean(true);
mockIssues.add(issue2);
IssueBean issue3 = createIssueBean(false, true);
mockIssues.add(issue3);

SearchResults mockSearchResults = mock(SearchResults.class);
when(mockSearchResults.getIssues()).thenReturn(mockIssues);
Expand All @@ -141,7 +139,7 @@ public void testGetJiraEntities() throws JsonProcessingException {
Instant timestamp = Instant.ofEpochSecond(0);
Queue<ItemInfo> itemInfoQueue = new ConcurrentLinkedQueue<>();
jiraService.getJiraEntities(jiraSourceConfig, timestamp, itemInfoQueue);

//one additional item is added for the project
assertEquals(mockIssues.size() + 1, itemInfoQueue.size());
}

Expand All @@ -155,7 +153,7 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep
JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient));
List<IssueBean> mockIssues = new ArrayList<>();
for (int i = 0; i < 50; i++) {
IssueBean issue1 = createIssueBean(false, false);
IssueBean issue1 = createIssueBean(false);
mockIssues.add(issue1);
}

Expand All @@ -179,9 +177,8 @@ public void testBadProjectKeys() throws JsonProcessingException {
issueType.add("Task");
issueStatus.add("Done");
projectKey.add("Bad Project Key");
projectKey.add("A");
projectKey.add("");
projectKey.add("!@#$");
projectKey.add("AAAAAAAAAAAAAA");

JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey);
JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient);
Expand All @@ -207,93 +204,7 @@ public void testGetJiraEntitiesException() throws JsonProcessingException {
assertThrows(RuntimeException.class, () -> jiraService.getJiraEntities(jiraSourceConfig, timestamp, itemInfoQueue));
}


@Test
public void testGetAllIssuesBasic() throws JsonProcessingException {
List<String> issueType = new ArrayList<>();
List<String> issueStatus = new ArrayList<>();
List<String> projectKey = new ArrayList<>();
issueType.add("Task");
JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey);
JiraService jiraService = new JiraService(restTemplate, jiraSourceConfig, authConfig);
SearchResults mockSearchResults = mock(SearchResults.class);
doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class));
SearchResults results = jiraService.getAllIssues(jql, 0, jiraSourceConfig);
assertNotNull(results);
}

@Test
public void testGetAllIssuesOauth2() throws JsonProcessingException {
List<String> issueType = new ArrayList<>();
List<String> issueStatus = new ArrayList<>();
List<String> projectKey = new ArrayList<>();
issueType.add("Task");
JiraSourceConfig jiraSourceConfig = createJiraConfiguration(OAUTH2, issueType, issueStatus, projectKey);
JiraService jiraService = new JiraService(restTemplate, jiraSourceConfig, authConfig);
SearchResults mockSearchResults = mock(SearchResults.class);
doReturn("http://mock-service.jira.com").when(authConfig).getUrl();
doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class));
SearchResults results = jiraService.getAllIssues(jql, 0, jiraSourceConfig);
assertNotNull(results);
}

private JiraSourceConfig createJiraConfiguration(String auth_type, List<String> issueType, List<String> issueStatus, List<String> projectKey) throws JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
Map<String, String> connectorCredentialsMap = new HashMap<>();
connectorCredentialsMap.put("auth_type", auth_type);

Map<String, Object> jiraSourceConfigMap = new HashMap<>();
jiraSourceConfigMap.put("account_url", ACCESSIBLE_RESOURCES);
jiraSourceConfigMap.put("connector_credentials", connectorCredentialsMap);
jiraSourceConfigMap.put("issue_types", issueType);
jiraSourceConfigMap.put("statuses", issueStatus);
jiraSourceConfigMap.put("projects", projectKey);


String jiraSourceConfigJsonString = objectMapper.writeValueAsString(jiraSourceConfigMap);
return objectMapper.readValue(jiraSourceConfigJsonString, JiraSourceConfig.class);
}

@Test
void testInvokeRestApiTokenExpiredInterruptException() throws JsonProcessingException, InterruptedException {
List<String> issueType = new ArrayList<>();
List<String> issueStatus = new ArrayList<>();
List<String> projectKey = new ArrayList<>();
JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey);
JiraService jiraService = new JiraService(restTemplate, jiraSourceConfig, authConfig);
when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key");
when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS));
jiraService.setSleepTimeMultiplier(100000);

Thread testThread = new Thread(() -> {
assertThrows(InterruptedException.class, () -> {
try {
jiraService.getIssue("key");
} catch (Exception e) {
throw new RuntimeException(e);
}
});
});
testThread.start();
Thread.sleep(100);
testThread.interrupt();
}

@ParameterizedTest
@MethodSource("provideHttpStatusCodesWithExceptionClass")
void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptionType) throws JsonProcessingException {
List<String> issueType = new ArrayList<>();
List<String> issueStatus = new ArrayList<>();
List<String> projectKey = new ArrayList<>();
JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey);
JiraService jiraService = new JiraService(restTemplate, jiraSourceConfig, authConfig);
jiraService.setSleepTimeMultiplier(1);
when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key");
when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(statusCode));
assertThrows(expectedExceptionType, () -> jiraService.getIssue("key"));
}

private IssueBean createIssueBean(boolean nullFields, boolean createdNull) {
private IssueBean createIssueBean(boolean nullFields) {
IssueBean issue1 = new IssueBean();
issue1.setId(UUID.randomUUID().toString());
issue1.setKey("issue_1_key");
Expand All @@ -302,16 +213,12 @@ private IssueBean createIssueBean(boolean nullFields, boolean createdNull) {

Map<String, Object> fieldMap = new HashMap<>();
if (!nullFields) {
fieldMap.put(CREATED, "2024-07-06T21:12:23.437-0700");
fieldMap.put(UPDATED, "2024-07-06T21:12:23.106-0700");
// fieldMap.put(UPDATED, Instant.now());
fieldMap.put(CREATED, Instant.now());
fieldMap.put(UPDATED, Instant.now());
} else {
fieldMap.put(CREATED, 0);
fieldMap.put(UPDATED, 0);
}
if (createdNull) {
fieldMap.put(CREATED, null);
}

Map<String, Object> issueTypeMap = new HashMap<>();
issueTypeMap.put("name", "Task");
Expand Down
Loading

0 comments on commit a8c3218

Please sign in to comment.