-
Notifications
You must be signed in to change notification settings - Fork 214
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'opensearch-project:main' into secrets-variable-interface
- Loading branch information
Showing
34 changed files
with
404 additions
and
78 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,2 @@ | ||
| # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams | ||
| * @sb2k16 @chenqi0805 @engechas @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh | ||
| * @sb2k16 @chenqi0805 @engechas @san81 @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 44 additions & 0 deletions
44
...-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AttributeHandler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| * | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.source.sqs; | ||
|
|
||
| import org.opensearch.dataprepper.model.event.Event; | ||
| import org.opensearch.dataprepper.model.event.EventMetadata; | ||
| import software.amazon.awssdk.services.sqs.model.Message; | ||
| import software.amazon.awssdk.services.sqs.model.MessageAttributeValue; | ||
| import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
|
|
||
| public class AttributeHandler { | ||
| public static Map<String, String> collectMetadataAttributes(final Message message, final String queueUrl) { | ||
| final Map<String, String> metadataMap = new HashMap<>(); | ||
| metadataMap.put("queueUrl", queueUrl); | ||
|
|
||
| for (Map.Entry<MessageSystemAttributeName, String> entry : message.attributes().entrySet()) { | ||
| String originalKey = entry.getKey().toString(); | ||
| String key = originalKey.substring(0, 1).toLowerCase() + originalKey.substring(1); | ||
| metadataMap.put(key, entry.getValue()); | ||
| } | ||
|
|
||
| for (Map.Entry<String, MessageAttributeValue> entry : message.messageAttributes().entrySet()) { | ||
| String originalKey = entry.getKey().toString(); | ||
| String key = originalKey.substring(0, 1).toLowerCase() + originalKey.substring(1); | ||
| metadataMap.put(key, entry.getValue().stringValue()); | ||
| } | ||
| return metadataMap; | ||
| } | ||
|
|
||
| public static void applyMetadataAttributes(final Event event, final Map<String, String> attributes) { | ||
| final EventMetadata metadata = event.getMetadata(); | ||
| attributes.forEach(metadata::setAttribute); | ||
| } | ||
| } |
5 changes: 5 additions & 0 deletions
5
...src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
5 changes: 5 additions & 0 deletions
5
...src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptions.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
...ain/java/org/opensearch/dataprepper/plugins/source/sqs/CodecBulkMessageFieldStrategy.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| * | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.source.sqs; | ||
|
|
||
| import org.opensearch.dataprepper.model.codec.InputCodec; | ||
| import org.opensearch.dataprepper.model.event.Event; | ||
| import org.opensearch.dataprepper.model.record.Record; | ||
| import java.io.ByteArrayInputStream; | ||
| import java.nio.charset.StandardCharsets; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.function.Consumer; | ||
|
|
||
| public class CodecBulkMessageFieldStrategy implements MessageFieldStrategy { | ||
|
|
||
| private final InputCodec codec; | ||
|
|
||
| public CodecBulkMessageFieldStrategy(final InputCodec codec) { | ||
| this.codec = codec; | ||
| } | ||
|
|
||
| @Override | ||
| public List<Event> parseEvents(final String messageBody) { | ||
| final List<Event> events = new ArrayList<>(); | ||
| final ByteArrayInputStream inputStream = new ByteArrayInputStream(messageBody.getBytes(StandardCharsets.UTF_8)); | ||
| try { | ||
| codec.parse(inputStream, (Consumer<Record<Event>>) record -> events.add(record.getData())); | ||
| } catch (Exception e) { | ||
| throw new RuntimeException("Failed to parse events from SQS body.", e); | ||
| } | ||
| return events; | ||
| } | ||
| } |
21 changes: 21 additions & 0 deletions
21
...rce/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/MessageFieldStrategy.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| * | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.source.sqs; | ||
|
|
||
| import org.opensearch.dataprepper.model.event.Event; | ||
| import java.util.List; | ||
|
|
||
| public interface MessageFieldStrategy { | ||
| /** | ||
| * Converts the SQS message body into one or more events. | ||
| */ | ||
| List<Event> parseEvents(String messageBody); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.