Jira Source Configuration and Filter Changes and add license headers

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java

@@ -25,9 +25,12 @@
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.DELIMITER;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.GREATER_THAN_EQUALS;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.ISSUE_TYPE_IN;
+import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.ISSUE_TYPE_NOT_IN;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PREFIX;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PROJECT_IN;
+import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PROJECT_NOT_IN;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.STATUS_IN;
+import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.STATUS_NOT_IN;
 import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.SUFFIX;
 
 
@@ -117,26 +120,41 @@ private void addItemsToQueue(List<IssueBean> issueList, Queue<ItemInfo> itemInfo
     private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration, Instant ts) {
 
         log.info("Creating issue filter criteria");
-        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectKeyFilter(configuration))) {
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameExcludeFilter(configuration)) ) {
             validateProjectFilters(configuration);
         }
         StringBuilder jiraQl = new StringBuilder(UPDATED + GREATER_THAN_EQUALS + ts.toEpochMilli());
-        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectKeyFilter(configuration))) {
-            jiraQl.append(PROJECT_IN).append(JiraConfigHelper.getProjectKeyFilter(configuration).stream()
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameIncludeFilter(configuration))) {
+            jiraQl.append(PROJECT_IN).append(JiraConfigHelper.getProjectNameIncludeFilter(configuration).stream()
                             .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
                     .append(CLOSING_ROUND_BRACKET);
         }
-        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeFilter(configuration))) {
-            jiraQl.append(ISSUE_TYPE_IN).append(JiraConfigHelper.getIssueTypeFilter(configuration).stream()
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameExcludeFilter(configuration))) {
+            jiraQl.append(PROJECT_NOT_IN).append(JiraConfigHelper.getProjectNameExcludeFilter(configuration).stream()
                             .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
                     .append(CLOSING_ROUND_BRACKET);
         }
-        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusFilter(configuration))) {
-            jiraQl.append(STATUS_IN).append(JiraConfigHelper.getIssueStatusFilter(configuration).stream()
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeIncludeFilter(configuration))) {
+            jiraQl.append(ISSUE_TYPE_IN).append(JiraConfigHelper.getIssueTypeIncludeFilter(configuration).stream()
                             .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
                     .append(CLOSING_ROUND_BRACKET);
         }
-        log.trace("Created issue filter criteria JiraQl query: {}", jiraQl);
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeExcludeFilter(configuration))) {
+            jiraQl.append(ISSUE_TYPE_NOT_IN).append(JiraConfigHelper.getIssueTypeExcludeFilter(configuration).stream()
+                            .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
+                    .append(CLOSING_ROUND_BRACKET);
+        }
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusIncludeFilter(configuration))) {
+            jiraQl.append(STATUS_IN).append(JiraConfigHelper.getIssueStatusIncludeFilter(configuration).stream()
+                            .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
+                    .append(CLOSING_ROUND_BRACKET);
+        }
+        if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusExcludeFilter(configuration))) {
+            jiraQl.append(STATUS_NOT_IN).append(JiraConfigHelper.getIssueStatusExcludeFilter(configuration).stream()
+                            .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
+                    .append(CLOSING_ROUND_BRACKET);
+        }
+        log.error("Created issue filter criteria JiraQl query: {}", jiraQl);
         return jiraQl;
     }
 
@@ -149,7 +167,13 @@ private void validateProjectFilters(JiraSourceConfig configuration) {
         log.trace("Validating project filters");
         List<String> badFilters = new ArrayList<>();
         Pattern regex = Pattern.compile("[^A-Z0-9]");
-        JiraConfigHelper.getProjectKeyFilter(configuration).forEach(projectFilter -> {
+        JiraConfigHelper.getProjectNameIncludeFilter(configuration).forEach(projectFilter -> {
+            Matcher matcher = regex.matcher(projectFilter);
+            if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) {
+                badFilters.add(projectFilter);
+            }
+        });
+        JiraConfigHelper.getProjectNameExcludeFilter(configuration).forEach(projectFilter -> {
             Matcher matcher = regex.matcher(projectFilter);
             if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) {
                 badFilters.add(projectFilter);

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java

@@ -1,16 +1,14 @@
 package org.opensearch.dataprepper.plugins.source.jira;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import jakarta.validation.constraints.Size;
+import jakarta.validation.Valid;
 import lombok.Getter;
+import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig;
+import org.opensearch.dataprepper.plugins.source.jira.configuration.FilterConfig;
 import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig;
 
 import java.time.Duration;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
-
-import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2;
 
 @Getter
 public class JiraSourceConfig implements CrawlerSourceConfig {
@@ -20,50 +18,23 @@ public class JiraSourceConfig implements CrawlerSourceConfig {
     /**
      * Jira account url
      */
-    @JsonProperty("account_url")
-    private String accountUrl;
-
-    /**
-     * A map of connector credentials specific to this source
-     */
-    @JsonProperty("connector_credentials")
-    private Map<String, String> connectorCredentials;
-
-    /**
-     * List of projects to ingest
-     */
-    @JsonProperty("projects")
-    @Size(max = 1000, message = "Project type filter should not be more than 1000")
-    private List<String> project = new ArrayList<>();
+    @JsonProperty("hosts")
+    private List<String> hosts;
 
     /**
-     * List of specific issue types to ingest.
-     * Ex: Story, Epic, Task etc
+     * Authentication Config to Access Jira
      */
-    @JsonProperty("issue_types")
-    @Size(max = 1000, message = "Issue type filter should be less than 1000")
-    private List<String> issueType = new ArrayList<>();
+    @JsonProperty("authentication")
+    @Valid
+    private AuthenticationConfig authenticationConfig;
 
-    /**
-     * Optional Inclusion patterns for filtering some tickets
-     */
-    @JsonProperty("inclusion_patterns")
-    @Size(max = 100, message = "inclusion pattern filters should not be more than 1000")
-    private List<String> inclusionPatterns;
 
     /**
-     * Optional Exclusion patterns for excluding some tickets
+     * Filter Config to filter what tickets get ingested
      */
-    @JsonProperty("exclusion_patterns")
-    @Size(max = 1000, message = "exclusion pattern filter should be less than 1000")
-    private List<String> exclusionPatterns;
+    @JsonProperty("filter")
+    private FilterConfig filterConfig;
 
-    /**
-     * Optional Status filter to ingest the tickets
-     */
-    @JsonProperty("statuses")
-    @Size(max = 1000, message = "Status filter should be less than 1000")
-    private List<String> status = new ArrayList<>();
 
     /**
      * Number of worker threads to spawn to parallel source fetching
@@ -78,43 +49,11 @@ public class JiraSourceConfig implements CrawlerSourceConfig {
     @JsonProperty("backoff_time")
     private Duration backOff = DEFAULT_BACKOFF_MILLIS;
 
-    public String getJiraId() {
-        return this.getConnectorCredentials().get("jira_id");
-    }
-
-    public String getJiraCredential() {
-        return this.getConnectorCredentials().get("jira_credential");
+    public String getAccountUrl() {
+        return this.getHosts().get(0);
     }
 
     public String getAuthType() {
-        return this.getConnectorCredentials().get("auth_type");
-    }
-
-    public String getAccessToken() {
-        return fetchGivenOAuthAttribute("access_token");
-    }
-
-    public String getRefreshToken() {
-        return fetchGivenOAuthAttribute("refresh_token");
-    }
-
-    public String getClientId() {
-        return fetchGivenOAuthAttribute("client_id");
-    }
-
-    public String getClientSecret() {
-        return fetchGivenOAuthAttribute("client_secret");
+        return this.getAuthenticationConfig().getAuthType();
     }
-
-    private String fetchGivenOAuthAttribute(String givenAttribute) {
-        if (!OAUTH2.equals(getAuthType())) {
-            throw new RuntimeException("Authentication Type is not OAuth2.");
-        }
-        String attributeValue = this.getConnectorCredentials().get(givenAttribute);
-        if (attributeValue == null || attributeValue.isEmpty()) {
-            throw new RuntimeException(String.format("%s is required for OAuth2 AuthType", givenAttribute));
-        }
-        return attributeValue;
-    }
-
 }

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/AuthenticationConfig.java

@@ -0,0 +1,35 @@
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.AssertTrue;
+import lombok.Getter;
+
+import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC;
+import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2;
+
+@Getter
+public class AuthenticationConfig {
+    @JsonProperty("basic")
+    @Valid
+    private BasicConfig basicConfig;
+
+    @JsonProperty("oauth2")
+    @Valid
+    private Oauth2Config oauth2Config;
+
+    @AssertTrue(message = "Authentication config should have either basic or oauth2")
+    private boolean isValidAuthenticationConfig() {
+        boolean hasBasic = basicConfig != null;
+        boolean hasOauth = oauth2Config != null;
+        return hasBasic ^ hasOauth;
+    }
+
+    public String getAuthType() {
+        if (basicConfig != null) {
+            return BASIC;
+        } else {
+            return OAUTH2;
+        }
+    }
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/BasicConfig.java

@@ -0,0 +1,19 @@
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.AssertTrue;
+import lombok.Getter;
+
+@Getter
+public class BasicConfig {
+    @JsonProperty("username")
+    private String username;
+
+    @JsonProperty("password")
+    private String password;
+
+    @AssertTrue(message = "Username and Password are both required for Basic Auth")
+    private boolean isBasicConfigValid() {
+        return username != null && password != null;
+    }
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/FilterConfig.java

@@ -0,0 +1,16 @@
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+
+@Getter
+public class FilterConfig {
+    @JsonProperty("project")
+    private ProjectConfig  projectConfig;
+
+    @JsonProperty("status")
+    private StatusConfig statusConfig;
+
+    @JsonProperty("issue_type")
+    private IssueTypeConfig issueTypeConfig;
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/IssueTypeConfig.java

@@ -0,0 +1,19 @@
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.Size;
+import lombok.Getter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Getter
+public class IssueTypeConfig {
+    @JsonProperty("include")
+    @Size(max = 1000, message = "Issue type filter should not be more than 1000")
+    private List<String> include = new ArrayList<>();
+
+    @JsonProperty("exclude")
+    @Size(max = 1000, message = "Issue type filter should not be more than 1000")
+    private List<String> exclude = new ArrayList<>();
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/NameConfig.java

@@ -0,0 +1,19 @@
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.Size;
+import lombok.Getter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Getter
+public class NameConfig {
+    @JsonProperty("include")
+    @Size(max = 1000, message = "Project name type filter should not be more than 1000")
+    private List<String> include = new ArrayList<>();
+
+    @JsonProperty("exclude")
+    @Size(max = 1000, message = "Project name type filter should not be more than 1000")
+    private List<String> exclude = new ArrayList<>();
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/Oauth2Config.java

@@ -0,0 +1,25 @@
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.AssertTrue;
+import lombok.Getter;
+
+@Getter
+public class Oauth2Config {
+    @JsonProperty("client_id")
+    private String clientId;
+
+    @JsonProperty("client_secret")
+    private String clientSecret;
+
+    @JsonProperty("access_token")
+    private String accessToken;
+
+    @JsonProperty("refresh_token")
+    private String refreshToken;
+
+    @AssertTrue(message = "Client ID, Client Secret, Access Token, and Refresh Token are both required for Oauth2")
+    private boolean isOauth2ConfigValid() {
+        return clientId != null && clientSecret != null && accessToken != null && refreshToken != null;
+    }
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/ProjectConfig.java

@@ -0,0 +1,10 @@
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+
+@Getter
+public class ProjectConfig {
+    @JsonProperty("key")
+    private NameConfig nameConfig;
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/StatusConfig.java

@@ -0,0 +1,19 @@
+package org.opensearch.dataprepper.plugins.source.jira.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.Size;
+import lombok.Getter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Getter
+public class StatusConfig {
+    @JsonProperty("include")
+    @Size(max = 1000, message = "status type filter should not be more than 1000")
+    private List<String> include = new ArrayList<>();
+
+    @JsonProperty("exclude")
+    @Size(max = 1000, message = "status type filter should not be more than 1000")
+    private List<String> exclude = new ArrayList<>();
+}

data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptor.java

@@ -17,8 +17,8 @@ public class BasicAuthInterceptor implements ClientHttpRequestInterceptor {
     private final String password;
 
     public BasicAuthInterceptor(JiraSourceConfig config) {
-        this.username = config.getJiraId();
-        this.password = config.getJiraCredential();
+        this.username = config.getAuthenticationConfig().getBasicConfig().getUsername();
+        this.password = config.getAuthenticationConfig().getBasicConfig().getPassword();;
     }
 
     @Override