Merge branch 'opensearch-project:main' into main

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @chenqi0805 @engechas @graytaylor0 @dinujoh @kkondaka @asifsmohammed @dlvenable @oeyh
+*   @chenqi0805 @engechas @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh

.github/workflows/kafka-plugin-integration-tests.yml

@@ -41,7 +41,7 @@ jobs:
         run: |
           echo 'KAFKA_VERSION=${{ matrix.kafka }}' > data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/.env
           docker compose --project-directory data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/zookeeper --env-file  data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/.env up -d
-          sleep 10
+          sleep 2
       - name: Wait for Kafka
         run: |
           ./gradlew data-prepper-plugins:kafka-plugins:integrationTest -Dtests.kafka.bootstrap_servers=localhost:9092 -Dtests.kafka.authconfig.username=admin -Dtests.kafka.authconfig.password=admin --tests KafkaStartIT

.github/workflows/release.yml

@@ -42,19 +42,19 @@ jobs:
         aws-region: us-east-1
 
     - name: Build Archives
-      run: ./gradlew --parallel --max-workers 2 :release:archives:buildArchives -Prelease
+      run: ./gradlew --parallel --max-workers 2 :release:archives:buildArchives
 
     - name: Build Maven Artifacts
       run: ./gradlew publish
 
     - name: Build Docker Image
-      run: ./gradlew :release:docker:docker -Prelease
+      run: ./gradlew :release:docker:docker
 
     - name: Upload Archives to Archives Bucket
-      run: ./gradlew :release:archives:uploadArchives -Prelease -Pregion=us-east-1 -Pbucket=${{ secrets.ARCHIVES_BUCKET_NAME }} -Pprofile=default -PbuildNumber=${{ github.run_number }}
+      run: ./gradlew :release:archives:uploadArchives -Pregion=us-east-1 -Pbucket=${{ secrets.ARCHIVES_BUCKET_NAME }} -Pprofile=default -PbuildNumber=${{ github.run_number }}
 
     - name: Upload Maven Artifacts to Archives Bucket
-      run: ./gradlew :release:maven:uploadArtifacts -Prelease -Pregion=us-east-1 -Pbucket=${{ secrets.ARCHIVES_BUCKET_NAME }} -Pprofile=default -PbuildNumber=${{ github.run_number }}
+      run: ./gradlew :release:maven:uploadArtifact -Pregion=us-east-1 -Pbucket=${{ secrets.ARCHIVES_BUCKET_NAME }} -Pprofile=default -PbuildNumber=${{ github.run_number }}
 
     - name: Log into Amazon ECR Public
       id: login-ecr

.github/workflows/testing-resources-cdk-check.yml

@@ -0,0 +1,34 @@
+name: Testing Resources CDK App Check
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'testing/aws-testing-cdk/**'
+  pull_request:
+    paths:
+      - 'testing/aws-testing-cdk/**'
+  workflow_dispatch:
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: ./testing/aws-testing-cdk
+    steps:
+    - name: Set up Node.js
+      uses: actions/setup-node@v2
+      with:
+        node-version: '18'
+
+    - name: Checkout Data Prepper
+      uses: actions/checkout@v2
+
+    - name: Install NPM Dependencies
+      run: npm install
+    - name: Test
+      run: npm run test
+    - name: Lint
+      run: npm run lint

.gitignore

@@ -13,3 +13,6 @@ gradle/tools
 # Ignore config file generated by test
 data-prepper-main/src/test/resources/logstash-filter.yaml
 data-prepper-main/src/test/resources/logstash-conf/logstash-filter.yaml
+
+# Python virtual environments
+.venv

.whitesource

@@ -5,11 +5,29 @@
     "projectToken": "",
     "baseBranches": []
   },
+  "scanSettingsSAST": {
+    "enableScan": false,
+    "scanPullRequests": false,
+    "incrementalScan": true,
+    "baseBranches": [],
+    "snippetSize": 10
+  },
   "checkRunSettings": {
     "vulnerableCheckRunConclusionLevel": "failure",
-    "displayMode": "diff"
+    "displayMode": "diff",
+    "useMendCheckNames": true
+  },
+  "checkRunSettingsSAST": {
+    "checkRunConclusionLevel": "failure",
+    "severityThreshold": "high"
   },
   "issueSettings": {
-    "minSeverityLevel": "LOW"
+    "minSeverityLevel": "LOW",
+    "issueType": "DEPENDENCY"
+  },
+  "remediateSettings": {
+    "workflowRules": {
+      "enabled": true
+    }
   }
 }

CODE_OF_CONDUCT.md

@@ -1,5 +1,5 @@
-This code of conduct applies to all spaces provided by the OpenSource project including in code, documentation, issue trackers, mailing lists, chat channels, wikis, blogs, social media and any other communication channels used by the project.
 
+This code of conduct applies to all spaces provided by the OpenSource project including in code, documentation, issue trackers, mailing lists, chat channels, wikis, blogs, social media, events, conferences, meetings, and any other communication channels used by the project.
 
 **Our open source communities endeavor to:**
 
@@ -8,7 +8,6 @@ This code of conduct applies to all spaces provided by the OpenSource project in
 * Be Respectful: We are committed to encouraging differing viewpoints, accepting constructive criticism and work collaboratively towards decisions that help the project grow. Disrespectful and unacceptable behavior will not be tolerated.
 * Be Collaborative: We are committed to supporting what is best for our community and users. When we build anything for the benefit of the project, we should document the work we do and communicate to others on how this affects their work.
 
-
 **Our Responsibility. As contributors, members, or bystanders we each individually have the responsibility to behave professionally and respectfully at all times. Disrespectful and unacceptable behaviors include, but are not limited to:**
 
 * The use of violent threats, abusive, discriminatory, or derogatory language;
@@ -19,6 +18,7 @@ This code of conduct applies to all spaces provided by the OpenSource project in
 * Publishing private information, such as physical or electronic address, without permission;
 * Other conduct which could reasonably be considered inappropriate in a professional setting;
 * Advocating for or encouraging any of the above behaviors.
-* Enforcement and Reporting Code of Conduct Issues:
+
+**Enforcement and Reporting Code of Conduct Issues:**
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be reported. [Contact us](mailto:[email protected]). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances.

MAINTAINERS.md

@@ -4,16 +4,17 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 
 ## Current Maintainers
 
-| Maintainer           | GitHub ID                                             | Affiliation |
-| -------------------- | ----------------------------------------------------- | ----------- |
-| Qi Chen              | [chenqi0805](https://github.com/chenqi0805)           | Amazon      |
-| Chase Engelbrecht    | [engechas](https://github.com/engechas)               | Amazon      |
-| Taylor Gray          | [graytaylor0](https://github.com/graytaylor0)         | Amazon      |
-| Dinu John            | [dinujoh](https://github.com/dinujoh)                 | Amazon      |
-| Krishna Kondaka      | [kkondaka](https://github.com/kkondaka)               | Amazon      |
-| Asif Sohail Mohammed | [asifsmohammed](https://github.com/asifsmohammed)     | Amazon      |
-| David Venable        | [dlvenable](https://github.com/dlvenable)             | Amazon      |
-| Hai Yan              | [oeyh](https://github.com/oeyh)                       | Amazon      |
+| Maintainer           | GitHub ID                                                 | Affiliation |
+| -------------------- | --------------------------------------------------------- | ----------- |
+| Qi Chen              | [chenqi0805](https://github.com/chenqi0805)               | Amazon      |
+| Chase Engelbrecht    | [engechas](https://github.com/engechas)                   | Amazon      |
+| Taylor Gray          | [graytaylor0](https://github.com/graytaylor0)             | Amazon      |
+| Dinu John            | [dinujoh](https://github.com/dinujoh)                     | Amazon      |
+| Krishna Kondaka      | [kkondaka](https://github.com/kkondaka)                   | Amazon      |
+| Asif Sohail Mohammed | [asifsmohammed](https://github.com/asifsmohammed)         | Amazon      |
+| Karsten Schnitter    | [KarstenSchnitter](https://github.com/KarstenSchnitter)   | SAP         |
+| David Venable        | [dlvenable](https://github.com/dlvenable)                 | Amazon      |
+| Hai Yan              | [oeyh](https://github.com/oeyh)                           | Amazon      |
 
 
 ## Emeritus

RELEASING.md

@@ -1,18 +1,122 @@
 # Releasing
 
 This document outlines the process for releasing Data Prepper.
+It is a guide for maintainers of the Data Prepper project to release a new version.
 
-## Release prerequisites
+## Overview
 
-Be sure you have:
+This document has three broad categories of steps to follow:
 
-* Created a release branch.
-* Updated the version in the release branch.
-* Updated the THIRD-PARTY file.
-* Created the release notes file
-* Created the changelog file
+1. [Release setup](#release-setup)
+2. [Performing a release](#performing-a-release)
+3. [Post-release](#post-release)
 
-## Performing a release
+
+## <a name="release-setup">Release setup</a>
+
+### Branch setup
+
+Data Prepper uses a release branch for releasing.
+The [Developer Guide](docs/developer_guide.md#backporting) discusses this in detail.
+
+The repository has a release branch for a major/minor version.
+Patch versions will continue on the same branch.
+For example, Data Prepper `2.6.0` was released from the `2.6` branch.
+Additionally, Data Prepper `2.6.1` and `2.6.2` were also released from the `2.6` branch.
+
+If you are creating a new major/minor release, then you will need to create the release branch.
+Use GitHub to create a new branch.
+
+Steps:
+1. Go to the [branches page here](https://github.com/opensearch-project/data-prepper/branches).
+2. Select `New branch`
+3. Name the branch `{major}.{minor}`. For example, `2.6`.
+4. Select `main` as the source.
+
+Create a backport label for the release branch.
+
+Steps:
+1. Go to the [labels page here](https://github.com/opensearch-project/data-prepper/labels).
+2. Select `New label`
+3. Name the branch `backport {major}.minor`. For example, `backport 2.6`
+
+### Update versions
+
+The Data Prepper version is defined in the [`gradle.properties`](https://github.com/opensearch-project/data-prepper/blob/main/gradle.properties)  file.
+We must update this whenever we create a new release.
+We will need two PRs to update it.
+
+#### Update on release branch
+
+For the current release branch, update the version to the new version.
+You may either need to update by removing the `-SNAPSHOT` or by updating the patch version.
+
+For example, when we released `2.6.2`, the property was set as `version=2.6.2`.
+You can see the [exact commit here](https://github.com/opensearch-project/data-prepper/blob/2.6.2/gradle.properties#L8).
+
+Create a PR that targets the release branch with the change.
+Do not create a PR to `main` for this.
+
+#### Update on the main branch
+
+If you have just created a release branch, you should also create a PR on the `main` branch to bump the version.
+
+For example, if you have started the `2.7` branch, you will need to update the `main` branch from `2.6.0-SNAPSHOT` to `2.7.0-SNAPSHOT`.
+
+#### Update the version of DataPrepperVersion
+
+If you have just created a release branch, you should also create a PR on the `main` branch to bump the version in `DataPrepperVersion`.
+
+Steps:
+1. Modify the `DataPrepperVersion` class to update `CURRENT_VERSION` to the next version.
+2. Create a PR targeting `main`
+
+Note: This step can be automated through [#4877](https://github.com/opensearch-project/data-prepper/issues/4877).
+
+### Update the THIRD-PARTY file
+
+We should update the `THIRD-PARTY` file for every release.
+Data Prepper has a GitHub action that will generate this and create a PR with the updated file.
+
+Steps:
+* Go the [Third Party Generate action](https://github.com/opensearch-project/data-prepper/actions/workflows/third-party-generate.yml)
+* Select `Run workflow`
+* Choose the branch you are releasing. e.g. `2.6`
+* Press `Run workflow`
+* Wait for a new PR to be created
+* Spot check the PR, approve and merge
+
+
+### Prepare release notes
+
+Prepare release notes and check them into the `main` branch in the [`release-notes` directory](https://github.com/opensearch-project/data-prepper/tree/main/release/release-notes).
+The format for the release notes file is `data-prepper.release-notes.{major}.{minor}.{patch}.md`.
+
+You can use a script to help you generate these.
+See the [README](release/script/release-notes/README.md) for the script for instructions.
+
+Once the release notes are ready, create a PR to merge them into `main`.
+Also tag this with the `backport {major}.{minor}` to create a PR that you can merge into your release branch.
+
+### Create changelog
+
+You can create a changelog using [git-release-notes](https://github.com/ariatemplates/git-release-notes).
+
+```
+git fetch upstream
+git switch {major}.{minor}
+git fetch upstream --tags
+git pull
+git-release-notes {previousMajor}.{previousMinor}.{previousPatch}..HEAD markdown > release/release-notes/data-prepper.change-log-{major}.{minor}.{patch}.md
+git switch main
+```
+
+Once the change log ready, create a PR to merge it into `main`.
+Also tag this with the `backport {major}.{minor}` to create a PR that you can merge into your release branch.
+
+
+
+## <a name="performing-a-release">Performing a release</a>
 
 This section outlines how to perform a Data Prepper release using GitHub Actions and the OpenSearch build infrastructure.
 The audience for this section are Data Prepper maintainers.
@@ -27,14 +131,14 @@ Select the "Run workflow" option from the GitHub Actions UI. GitHub will prompt
 #### Use workflow for
 
 Select the release branch which you are releasing for.
-Typically, this will be a branch such as `2.4`.
+Typically, this will be a branch such as `2.6`.
 However, you may select `main` for testing.
 
 #### Whether to create major tag of docker image or not.
 
 This will create a tag such as `2` which points to this version
 
-All releases have a full version tag. For example, `2.4.0`.
+All releases have a full version tag. For example, `2.6.0`.
 The latest release on a major series can also have a tag applied such as `2`.
 Check this option if you are releasing the latest version withing that series of major versions.
 This value can be true for old major series as well such as the 1.x series.
@@ -69,3 +173,28 @@ You can also deny a release by using _deny_ or _denied_ in the comment.
 ### Further details
 
 For more details on the release build, or to setup your own GitHub repository, see [release/README.md](release/README.md).
+
+
+
+## <a name="post-release">Post release</a>
+
+After the release, there are a few other steps to clean up the repository.
+
+### Update the release notes
+
+The release process will have created a draft release for the new version.
+The next step is to update the draft release with the release notes created before the release.
+
+Steps:
+* Go to the [releases page](https://github.com/opensearch-project/data-prepper/releases)
+* Find the new draft release. It should be at the top.
+* Replace the auto-generated release notes with the release notes created previous to the release.
+* Release it
+
+### Close the GitHub milestone
+
+Steps:
+* Go to the [milestones](https://github.com/opensearch-project/data-prepper/milestones) page.
+* Find the milestone for the release.
+* Make sure there are no issues. If there are any triage them by closing, or changing the milestone.
+* Click the "Close" button