-
Notifications
You must be signed in to change notification settings - Fork 211
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Address Scale Items for lambda plugin (#5032)
* Add support for lambda async client in lambda processor Signed-off-by: Srikanth Govindarajan <[email protected]> Refactor aws lambda plugin to have a class for common methods between processor and sink Signed-off-by: Srikanth Govindarajan <[email protected]> Add support for lambda async client in lambda sink Signed-off-by: Srikanth Govindarajan <[email protected]> * Changes to Lambda Plugin Integration Test Signed-off-by: Srikanth Govindarajan <[email protected]> * Add JsonPropertyDescription to all Config, add debug logs and add ITs Signed-off-by: Srikanth Govindarajan <[email protected]> * Address Acknowledgements for processor and sink; Add request and response codec Signed-off-by: Srikanth Govindarajan <[email protected]> * Address comments Signed-off-by: Srikanth Govindarajan <[email protected]> * Add response processing mode to processor configuration Signed-off-by: Srikanth Govindarajan <[email protected]> * Add Response Handling Strategy; Make InvocationType and ResponseCardinality enums; Change reponse_processing_mode option to response_cardinality Signed-off-by: Srikanth Govindarajan <[email protected]> * Address Enum Signed-off-by: Srikanth Govindarajan <[email protected]> * Address Enum2 Signed-off-by: Srikanth Govindarajan <[email protected]> * Fix checkstyle Signed-off-by: Srikanth Govindarajan <[email protected]> --------- Signed-off-by: Srikanth Govindarajan <[email protected]>
- Loading branch information
1 parent
cfaf19d
commit e311f0d
Showing
35 changed files
with
1,947 additions
and
1,076 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
66 changes: 66 additions & 0 deletions
66
...a/src/main/java/org/opensearch/dataprepper/plugins/lambda/common/LambdaCommonHandler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| package org.opensearch.dataprepper.plugins.lambda.common; | ||
|
|
||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import org.opensearch.dataprepper.plugins.lambda.common.accumlator.Buffer; | ||
| import org.opensearch.dataprepper.plugins.lambda.common.accumlator.BufferFactory; | ||
| import org.slf4j.Logger; | ||
| import software.amazon.awssdk.services.lambda.LambdaAsyncClient; | ||
| import software.amazon.awssdk.services.lambda.model.InvokeResponse; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.concurrent.CompletableFuture; | ||
|
|
||
| public class LambdaCommonHandler { | ||
| private final Logger LOG; | ||
| private final LambdaAsyncClient lambdaAsyncClient; | ||
| private final String functionName; | ||
| private final String invocationType; | ||
| BufferFactory bufferFactory; | ||
| private final ObjectMapper objectMapper = new ObjectMapper(); | ||
|
|
||
| public LambdaCommonHandler( | ||
| final Logger log, | ||
| final LambdaAsyncClient lambdaAsyncClient, | ||
| final String functionName, | ||
| final String invocationType, | ||
| BufferFactory bufferFactory){ | ||
| this.LOG = log; | ||
| this.lambdaAsyncClient = lambdaAsyncClient; | ||
| this.functionName = functionName; | ||
| this.invocationType = invocationType; | ||
| this.bufferFactory = bufferFactory; | ||
| } | ||
|
|
||
| public Buffer createBuffer(Buffer currentBuffer) { | ||
| try { | ||
| LOG.debug("Resetting buffer"); | ||
| currentBuffer = bufferFactory.getBuffer(lambdaAsyncClient, functionName, invocationType); | ||
| return currentBuffer; | ||
| } catch (IOException e) { | ||
| throw new RuntimeException("Failed to reset buffer", e); | ||
| } | ||
| } | ||
|
|
||
| public boolean checkStatusCode(InvokeResponse response) { | ||
| int statusCode = response.statusCode(); | ||
| if (statusCode < 200 || statusCode >= 300) { | ||
| LOG.error("Lambda invocation returned with non-success status code: {}", statusCode); | ||
| return false; | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| public void waitForFutures(List<CompletableFuture<Void>> futureList) { | ||
| if (!futureList.isEmpty()) { | ||
| try { | ||
| CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join(); | ||
| LOG.info("All {} Lambda invocations have completed", futureList.size()); | ||
| } catch (Exception e) { | ||
| LOG.warn("Exception while waiting for Lambda invocations to complete", e); | ||
| } finally { | ||
| futureList.clear(); | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.