From 22a7b19f77f5e749f9b6d6e333270281db1c688e Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 20 Feb 2024 13:01:43 +0900 Subject: [PATCH] Beyond 4096b cf param sar (#627) * add second trigger parameters in cf template * update docs * changelog * add PR # on changelog --- .internal/aws/cloudformation/application.yaml | 8 +++ .internal/aws/cloudformation/macro.yaml | 70 +++++++++++++++++++ .internal/aws/cloudformation/template.yaml | 32 +++++++++ CHANGELOG.md | 9 ++- ...ploy-elastic-serverless-forwarder.asciidoc | 8 +++ 5 files changed, 126 insertions(+), 1 deletion(-) diff --git a/.internal/aws/cloudformation/application.yaml b/.internal/aws/cloudformation/application.yaml index 450ae2bd..d15ac650 100644 --- a/.internal/aws/cloudformation/application.yaml +++ b/.internal/aws/cloudformation/application.yaml @@ -14,12 +14,20 @@ Parameters: Type: CommaDelimitedList ElasticServerlessForwarderSQSEvents: Type: CommaDelimitedList + ElasticServerlessForwarderSQSEvents2: + Type: CommaDelimitedList ElasticServerlessForwarderS3SQSEvents: Type: CommaDelimitedList + ElasticServerlessForwarderS3SQSEvents2: + Type: CommaDelimitedList ElasticServerlessForwarderKinesisEvents: Type: CommaDelimitedList + ElasticServerlessForwarderKinesisEvents2: + Type: CommaDelimitedList ElasticServerlessForwarderCloudWatchLogsEvents: Type: CommaDelimitedList + ElasticServerlessForwarderCloudWatchLogsEvents2: + Type: CommaDelimitedList ElasticServerlessForwarderS3Buckets: Type: CommaDelimitedList ElasticServerlessForwarderSecurityGroups: diff --git a/.internal/aws/cloudformation/macro.yaml b/.internal/aws/cloudformation/macro.yaml index c6b19218..a8288330 100644 --- a/.internal/aws/cloudformation/macro.yaml +++ b/.internal/aws/cloudformation/macro.yaml @@ -38,6 +38,23 @@ Resources: } } + if "ElasticServerlessForwarderKinesisEvents2" in parameters: + for kinesis_event in parameters["ElasticServerlessForwarderKinesisEvents2"]: + kinesis_event = kinesis_event.strip() + if len(kinesis_event) == 0: + continue + + kinesis_event_name = f"KinesisEvent{hex_suffix(kinesis_event)}" + events_fragment[kinesis_event_name] = { + "Type": "Kinesis", + "Properties": { + "Stream": kinesis_event, + "StartingPosition": "TRIM_HORIZON", + "BatchSize": 10, + "Enabled": True, + } + } + if "ElasticServerlessForwarderSQSEvents" in parameters: for sqs_event in parameters["ElasticServerlessForwarderSQSEvents"]: sqs_event = sqs_event.strip() @@ -54,6 +71,23 @@ Resources: } } + + if "ElasticServerlessForwarderSQSEvents2" in parameters: + for sqs_event in parameters["ElasticServerlessForwarderSQSEvents2"]: + sqs_event = sqs_event.strip() + if len(sqs_event) == 0: + continue + + sqs_event_name = f"SQSEvent{hex_suffix(sqs_event)}" + events_fragment[sqs_event_name] = { + "Type": "SQS", + "Properties": { + "Queue": sqs_event, + "BatchSize": 10, + "Enabled": True, + } + } + if "ElasticServerlessForwarderS3SQSEvents" in parameters: for s3_sqs_event in parameters["ElasticServerlessForwarderS3SQSEvents"]: s3_sqs_event = s3_sqs_event.strip() @@ -70,6 +104,23 @@ Resources: } } + + if "ElasticServerlessForwarderS3SQSEvents2" in parameters: + for s3_sqs_event in parameters["ElasticServerlessForwarderS3SQSEvents2"]: + s3_sqs_event = s3_sqs_event.strip() + if len(s3_sqs_event) == 0: + continue + + s3_sqs_event_name = f"S3SQSEvent{hex_suffix(s3_sqs_event)}" + events_fragment[s3_sqs_event_name] = { + "Type": "SQS", + "Properties": { + "Queue": s3_sqs_event, + "BatchSize": 10, + "Enabled": True, + } + } + if "ElasticServerlessForwarderCloudWatchLogsEvents" in parameters: for cloudwatch_logs_event in parameters["ElasticServerlessForwarderCloudWatchLogsEvents"]: cloudwatch_logs_event = cloudwatch_logs_event.strip() @@ -88,6 +139,25 @@ Resources: } } + + if "ElasticServerlessForwarderCloudWatchLogsEvents2" in parameters: + for cloudwatch_logs_event in parameters["ElasticServerlessForwarderCloudWatchLogsEvents2"]: + cloudwatch_logs_event = cloudwatch_logs_event.strip() + if len(cloudwatch_logs_event) == 0: + continue + + arn_components = cloudwatch_logs_event.split(":") + cloudwatch_logs_group_name = arn_components[6] + + cloudwatch_logs_event_name = f"CloudWatchLogsEvent{hex_suffix(cloudwatch_logs_group_name)}" + events_fragment[cloudwatch_logs_event_name] = { + "Type": "CloudWatchLogs", + "Properties": { + "FilterPattern": "", + "LogGroupName": cloudwatch_logs_group_name, + } + } + return events_fragment diff --git a/.internal/aws/cloudformation/template.yaml b/.internal/aws/cloudformation/template.yaml index e9409d31..1e82ca74 100644 --- a/.internal/aws/cloudformation/template.yaml +++ b/.internal/aws/cloudformation/template.yaml @@ -22,18 +22,34 @@ Parameters: Type: CommaDelimitedList Default: "" Description: Comma delimited list of Direct SQS queues ARNs to set as event triggers for the Lambda + ElasticServerlessForwarderSQSEvents2: + Type: CommaDelimitedList + Default: "" + Description: Comma delimited list of Direct SQS queues ARNs to set as event triggers for the Lambda, if limit is reached on ElasticServerlessForwarderSQSEvents ElasticServerlessForwarderS3SQSEvents: Type: CommaDelimitedList Default: "" Description: Comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the Lambda + ElasticServerlessForwarderS3SQSEvents2: + Type: CommaDelimitedList + Default: "" + Description: Comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the Lambda, if limit is reached on ElasticServerlessForwarderS3SQSEvents ElasticServerlessForwarderKinesisEvents: Type: CommaDelimitedList Default: "" Description: Comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the Lambda + ElasticServerlessForwarderKinesisEvents2: + Type: CommaDelimitedList + Default: "" + Description: Comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the Lambda, if limit is reached on ElasticServerlessForwarderKinesisEvents ElasticServerlessForwarderCloudWatchLogsEvents: Type: CommaDelimitedList Default: "" Description: Comma delimited list of Cloudwatch Logs Log Groups ARNs to set subscription filters on the Lambda for + ElasticServerlessForwarderCloudWatchLogsEvents2: + Type: CommaDelimitedList + Default: "" + Description: Comma delimited list of Cloudwatch Logs Log Groups ARNs to set subscription filters on the Lambda for, if limit is reached on ElasticServerlessForwarderCloudWatchLogsEvents ElasticServerlessForwarderS3Buckets: Type: CommaDelimitedList Default: "" @@ -64,9 +80,13 @@ Resources: ElasticServerlessForwarderSSMSecrets: !Join [",", !Ref ElasticServerlessForwarderSSMSecrets] ElasticServerlessForwarderKMSKeys: !Join [",", !Ref ElasticServerlessForwarderKMSKeys] ElasticServerlessForwarderSQSEvents: !Join [",", !Ref ElasticServerlessForwarderSQSEvents] + ElasticServerlessForwarderSQSEvents2: !Join [",", !Ref ElasticServerlessForwarderSQSEvents2] ElasticServerlessForwarderS3SQSEvents: !Join [",", !Ref ElasticServerlessForwarderS3SQSEvents] + ElasticServerlessForwarderS3SQSEvents2: !Join [",", !Ref ElasticServerlessForwarderS3SQSEvents2] ElasticServerlessForwarderKinesisEvents: !Join [",", !Ref ElasticServerlessForwarderKinesisEvents] + ElasticServerlessForwarderKinesisEvents2: !Join [",", !Ref ElasticServerlessForwarderKinesisEvents2] ElasticServerlessForwarderCloudWatchLogsEvents: !Join [",", !Ref ElasticServerlessForwarderCloudWatchLogsEvents] + ElasticServerlessForwarderCloudWatchLogsEvents2: !Join [",", !Ref ElasticServerlessForwarderCloudWatchLogsEvents2] ElasticServerlessForwarderS3Buckets: !Join [",", !Ref ElasticServerlessForwarderS3Buckets] ElasticServerlessForwarderSecurityGroups: !Join [",", !Ref ElasticServerlessForwarderSecurityGroups] ElasticServerlessForwarderSubnets: !Join [",", !Ref ElasticServerlessForwarderSubnets] @@ -84,9 +104,13 @@ Metadata: default: Elastic Serverless Forwarder Events options Parameters: - ElasticServerlessForwarderSQSEvents + - ElasticServerlessForwarderSQSEvents2 - ElasticServerlessForwarderS3SQSEvents + - ElasticServerlessForwarderS3SQSEvents2 - ElasticServerlessForwarderKinesisEvents + - ElasticServerlessForwarderKinesisEvents2 - ElasticServerlessForwarderCloudWatchLogsEvents + - ElasticServerlessForwarderCloudWatchLogsEvents2 - Label: default: Elastic Serverless Forwarder S3 Buckets Permissions options Parameters: @@ -105,12 +129,20 @@ Metadata: default: "AWS KMS Keys" ElasticServerlessForwarderCloudWatchLogsEvents: default: "Cloudwatch Logs subscription filters" + ElasticServerlessForwarderCloudWatchLogsEvents2: + default: "Cloudwatch Logs subscription filters, if limit is reached on ElasticServerlessForwarderCloudWatchLogsEvents" ElasticServerlessForwarderKinesisEvents: default: "Kinesis Data Stream event triggers" + ElasticServerlessForwarderKinesisEvents2: + default: "Kinesis Data Stream event triggers, if limit is reached on ElasticServerlessForwarderKinesisEvents" ElasticServerlessForwarderS3SQSEvents: default: "S3 SQS Event Notifications event triggers" + ElasticServerlessForwarderS3SQSEvents2: + default: "S3 SQS Event Notifications event triggers, if limit is reached on ElasticServerlessForwarderS3SQSEvents" ElasticServerlessForwarderSQSEvents: default: "Direct SQS queues event triggers" + ElasticServerlessForwarderSQSEvents2: + default: "Direct SQS queues event triggers, if limit is reached on ElasticServerlessForwarderSQSEvents" ElasticServerlessForwarderS3Buckets: default: "S3 buckets of S3 SQS Event Notifications" ElasticServerlessForwarderSecurityGroups: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b0ac5db..a7c92fac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,15 @@ +### v1.13.0 - 2024/02/23 +##### Features +* Go beyond 4096b limit on CF Parameter for event triggers on SAR deployment [#627](https://github.com/elastic/elastic-serverless-forwarder/pull/627) + +### v1.12.0 - 2024/02/13 +##### Features +* Add outputs for Lambda function ARN and IAM Role ARN [#552](https://github.com/elastic/elastic-serverless-forwarder/pull/552) + ### v1.11.0 - 2023/12/08 ##### Features * Add user agent with information about ESF version and host environment: [#537](https://github.com/elastic/elastic-serverless-forwarder/pull/537) * Remove calls to `sqs.DeleteMessage` and refactor storage decorators: [#544](https://github.com/elastic/elastic-serverless-forwarder/pull/544) -* Add outputs for Lambda function ARN and IAM Role ARN [#552](https://github.com/elastic/elastic-serverless-forwarder/pull/552) ##### Bug fixes * Fix regression when both `json_content_type: single` and `expand_event_list_from_field` are set: [#553](https://github.com/elastic/elastic-serverless-forwarder/pull/553) diff --git a/docs/en/aws-deploy-elastic-serverless-forwarder.asciidoc b/docs/en/aws-deploy-elastic-serverless-forwarder.asciidoc index e9665e45..7fda4e69 100644 --- a/docs/en/aws-deploy-elastic-serverless-forwarder.asciidoc +++ b/docs/en/aws-deploy-elastic-serverless-forwarder.asciidoc @@ -259,9 +259,13 @@ Make sure you include all the KMS keys used to encrypt the data. For example, S3 These parameters define your specific <> or 'event triggers'. - `ElasticServerlessForwarderSQSEvents`: Add a comma delimited list of Direct SQS queue ARNs to set as event triggers for the forwarder (if any). +- `ElasticServerlessForwarderSQSEvents2`: Add a comma delimited list of Direct SQS queue ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderSQSEvents). - `ElasticServerlessForwarderS3SQSEvents`: Add a comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the forwarder (if any). +- `ElasticServerlessForwarderS3SQSEvents2`: Add a comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderS3SQSEvents). - `ElasticServerlessForwarderKinesisEvents`: Add a comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the forwarder (if any). +- `ElasticServerlessForwarderKinesisEvents2`: Add a comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderKinesisEvents). - `ElasticServerlessForwarderCloudWatchLogsEvents`: Add a comma delimited list of Cloudwatch Logs log group ARNs to set subscription filters on the forwarder (if any). +- `ElasticServerlessForwarderCloudWatchLogsEvents2`: Add a comma delimited list of Cloudwatch Logs log group ARNs to set subscription filters on the forwarder (if limit is reach on ElasticServerlessForwarderCloudWatchLogsEvents). [NOTE] ==== @@ -361,9 +365,13 @@ aws serverlessrepo list-application-versions --application-id arn:aws:serverless ElasticServerlessForwarderSSMSecrets: "" ElasticServerlessForwarderKMSKeys: "" ElasticServerlessForwarderSQSEvents: "" + ElasticServerlessForwarderSQSEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 ElasticServerlessForwarderS3SQSEvents: "" + ElasticServerlessForwarderS3SQSEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 ElasticServerlessForwarderKinesisEvents: "" + ElasticServerlessForwarderKinesisEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 ElasticServerlessForwarderCloudWatchLogsEvents: "" + ElasticServerlessForwarderCloudWatchLogsEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0 ElasticServerlessForwarderS3Buckets: "" ElasticServerlessForwarderSecurityGroups: "" ElasticServerlessForwarderSubnets: ""