Beyond 4096b cf param sar (#627)

* add second trigger parameters in cf template

* update docs

* changelog

* add PR # on changelog

.internal/aws/cloudformation/application.yaml

@@ -14,12 +14,20 @@ Parameters:
     Type: CommaDelimitedList
   ElasticServerlessForwarderSQSEvents:
     Type: CommaDelimitedList
+  ElasticServerlessForwarderSQSEvents2:
+    Type: CommaDelimitedList
   ElasticServerlessForwarderS3SQSEvents:
     Type: CommaDelimitedList
+  ElasticServerlessForwarderS3SQSEvents2:
+    Type: CommaDelimitedList
   ElasticServerlessForwarderKinesisEvents:
     Type: CommaDelimitedList
+  ElasticServerlessForwarderKinesisEvents2:
+    Type: CommaDelimitedList
   ElasticServerlessForwarderCloudWatchLogsEvents:
     Type: CommaDelimitedList
+  ElasticServerlessForwarderCloudWatchLogsEvents2:
+    Type: CommaDelimitedList
   ElasticServerlessForwarderS3Buckets:
     Type: CommaDelimitedList
   ElasticServerlessForwarderSecurityGroups:

.internal/aws/cloudformation/macro.yaml

@@ -38,6 +38,23 @@ Resources:
                         }
                     }
 
+            if "ElasticServerlessForwarderKinesisEvents2" in parameters:
+                for kinesis_event in parameters["ElasticServerlessForwarderKinesisEvents2"]:
+                    kinesis_event = kinesis_event.strip()
+                    if len(kinesis_event) == 0:
+                        continue
+
+                    kinesis_event_name = f"KinesisEvent{hex_suffix(kinesis_event)}"
+                    events_fragment[kinesis_event_name] = {
+                        "Type": "Kinesis",
+                        "Properties": {
+                            "Stream": kinesis_event,
+                            "StartingPosition": "TRIM_HORIZON",
+                            "BatchSize": 10,
+                            "Enabled": True,
+                        }
+                    }
+
             if "ElasticServerlessForwarderSQSEvents" in parameters:
                 for sqs_event in parameters["ElasticServerlessForwarderSQSEvents"]:
                     sqs_event = sqs_event.strip()
@@ -54,6 +71,23 @@ Resources:
                         }
                     }
 
+
+            if "ElasticServerlessForwarderSQSEvents2" in parameters:
+                for sqs_event in parameters["ElasticServerlessForwarderSQSEvents2"]:
+                    sqs_event = sqs_event.strip()
+                    if len(sqs_event) == 0:
+                        continue
+
+                    sqs_event_name = f"SQSEvent{hex_suffix(sqs_event)}"
+                    events_fragment[sqs_event_name] = {
+                        "Type": "SQS",
+                        "Properties": {
+                            "Queue": sqs_event,
+                            "BatchSize": 10,
+                            "Enabled": True,
+                        }
+                    }
+
             if "ElasticServerlessForwarderS3SQSEvents" in parameters:
                 for s3_sqs_event in parameters["ElasticServerlessForwarderS3SQSEvents"]:
                     s3_sqs_event = s3_sqs_event.strip()
@@ -70,6 +104,23 @@ Resources:
                         }
                     }
 
+
+            if "ElasticServerlessForwarderS3SQSEvents2" in parameters:
+                for s3_sqs_event in parameters["ElasticServerlessForwarderS3SQSEvents2"]:
+                    s3_sqs_event = s3_sqs_event.strip()
+                    if len(s3_sqs_event) == 0:
+                        continue
+
+                    s3_sqs_event_name = f"S3SQSEvent{hex_suffix(s3_sqs_event)}"
+                    events_fragment[s3_sqs_event_name] = {
+                        "Type": "SQS",
+                        "Properties": {
+                            "Queue": s3_sqs_event,
+                            "BatchSize": 10,
+                            "Enabled": True,
+                        }
+                    }
+
             if "ElasticServerlessForwarderCloudWatchLogsEvents" in parameters:
                 for cloudwatch_logs_event in parameters["ElasticServerlessForwarderCloudWatchLogsEvents"]:
                     cloudwatch_logs_event = cloudwatch_logs_event.strip()
@@ -88,6 +139,25 @@ Resources:
                         }
                     }
 
+
+            if "ElasticServerlessForwarderCloudWatchLogsEvents2" in parameters:
+                for cloudwatch_logs_event in parameters["ElasticServerlessForwarderCloudWatchLogsEvents2"]:
+                    cloudwatch_logs_event = cloudwatch_logs_event.strip()
+                    if len(cloudwatch_logs_event) == 0:
+                        continue
+
+                    arn_components = cloudwatch_logs_event.split(":")
+                    cloudwatch_logs_group_name = arn_components[6]
+
+                    cloudwatch_logs_event_name = f"CloudWatchLogsEvent{hex_suffix(cloudwatch_logs_group_name)}"
+                    events_fragment[cloudwatch_logs_event_name] = {
+                        "Type": "CloudWatchLogs",
+                        "Properties": {
+                            "FilterPattern": "",
+                            "LogGroupName": cloudwatch_logs_group_name,
+                        }
+                    }
+
             return events_fragment
 
 

.internal/aws/cloudformation/template.yaml

@@ -22,18 +22,34 @@ Parameters:
     Type: CommaDelimitedList
     Default: ""
     Description: Comma delimited list of Direct SQS queues ARNs to set as event triggers for the Lambda
+  ElasticServerlessForwarderSQSEvents2:
+    Type: CommaDelimitedList
+    Default: ""
+    Description: Comma delimited list of Direct SQS queues ARNs to set as event triggers for the Lambda, if limit is reached on ElasticServerlessForwarderSQSEvents
   ElasticServerlessForwarderS3SQSEvents:
     Type: CommaDelimitedList
     Default: ""
     Description: Comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the Lambda
+  ElasticServerlessForwarderS3SQSEvents2:
+    Type: CommaDelimitedList
+    Default: ""
+    Description: Comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the Lambda, if limit is reached on ElasticServerlessForwarderS3SQSEvents
   ElasticServerlessForwarderKinesisEvents:
     Type: CommaDelimitedList
     Default: ""
     Description: Comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the Lambda
+  ElasticServerlessForwarderKinesisEvents2:
+    Type: CommaDelimitedList
+    Default: ""
+    Description: Comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the Lambda, if limit is reached on ElasticServerlessForwarderKinesisEvents
   ElasticServerlessForwarderCloudWatchLogsEvents:
     Type: CommaDelimitedList
     Default: ""
     Description: Comma delimited list of Cloudwatch Logs Log Groups ARNs to set subscription filters on the Lambda for
+  ElasticServerlessForwarderCloudWatchLogsEvents2:
+    Type: CommaDelimitedList
+    Default: ""
+    Description: Comma delimited list of Cloudwatch Logs Log Groups ARNs to set subscription filters on the Lambda for, if limit is reached on ElasticServerlessForwarderCloudWatchLogsEvents
   ElasticServerlessForwarderS3Buckets:
     Type: CommaDelimitedList
     Default: ""
@@ -64,9 +80,13 @@ Resources:
         ElasticServerlessForwarderSSMSecrets: !Join [",", !Ref ElasticServerlessForwarderSSMSecrets]
         ElasticServerlessForwarderKMSKeys: !Join [",", !Ref ElasticServerlessForwarderKMSKeys]
         ElasticServerlessForwarderSQSEvents: !Join [",", !Ref ElasticServerlessForwarderSQSEvents]
+        ElasticServerlessForwarderSQSEvents2: !Join [",", !Ref ElasticServerlessForwarderSQSEvents2]
         ElasticServerlessForwarderS3SQSEvents: !Join [",", !Ref ElasticServerlessForwarderS3SQSEvents]
+        ElasticServerlessForwarderS3SQSEvents2: !Join [",", !Ref ElasticServerlessForwarderS3SQSEvents2]
         ElasticServerlessForwarderKinesisEvents: !Join [",", !Ref ElasticServerlessForwarderKinesisEvents]
+        ElasticServerlessForwarderKinesisEvents2: !Join [",", !Ref ElasticServerlessForwarderKinesisEvents2]
         ElasticServerlessForwarderCloudWatchLogsEvents: !Join [",", !Ref ElasticServerlessForwarderCloudWatchLogsEvents]
+        ElasticServerlessForwarderCloudWatchLogsEvents2: !Join [",", !Ref ElasticServerlessForwarderCloudWatchLogsEvents2]
         ElasticServerlessForwarderS3Buckets: !Join [",", !Ref ElasticServerlessForwarderS3Buckets]
         ElasticServerlessForwarderSecurityGroups: !Join [",", !Ref ElasticServerlessForwarderSecurityGroups]
         ElasticServerlessForwarderSubnets: !Join [",", !Ref ElasticServerlessForwarderSubnets]
@@ -84,9 +104,13 @@ Metadata:
           default: Elastic Serverless Forwarder Events options
         Parameters:
           - ElasticServerlessForwarderSQSEvents
+          - ElasticServerlessForwarderSQSEvents2
           - ElasticServerlessForwarderS3SQSEvents
+          - ElasticServerlessForwarderS3SQSEvents2
           - ElasticServerlessForwarderKinesisEvents
+          - ElasticServerlessForwarderKinesisEvents2
           - ElasticServerlessForwarderCloudWatchLogsEvents
+          - ElasticServerlessForwarderCloudWatchLogsEvents2
       - Label:
           default: Elastic Serverless Forwarder S3 Buckets Permissions options
         Parameters:
@@ -105,12 +129,20 @@ Metadata:
         default: "AWS KMS Keys"
       ElasticServerlessForwarderCloudWatchLogsEvents:
         default: "Cloudwatch Logs subscription filters"
+      ElasticServerlessForwarderCloudWatchLogsEvents2:
+        default: "Cloudwatch Logs subscription filters, if limit is reached on ElasticServerlessForwarderCloudWatchLogsEvents"
       ElasticServerlessForwarderKinesisEvents:
         default: "Kinesis Data Stream event triggers"
+      ElasticServerlessForwarderKinesisEvents2:
+        default: "Kinesis Data Stream event triggers, if limit is reached on ElasticServerlessForwarderKinesisEvents"
       ElasticServerlessForwarderS3SQSEvents:
         default: "S3 SQS Event Notifications event triggers"
+      ElasticServerlessForwarderS3SQSEvents2:
+        default: "S3 SQS Event Notifications event triggers, if limit is reached on ElasticServerlessForwarderS3SQSEvents"
       ElasticServerlessForwarderSQSEvents:
         default: "Direct SQS queues event triggers"
+      ElasticServerlessForwarderSQSEvents2:
+        default: "Direct SQS queues event triggers, if limit is reached on ElasticServerlessForwarderSQSEvents"
       ElasticServerlessForwarderS3Buckets:
         default: "S3 buckets of S3 SQS Event Notifications"
       ElasticServerlessForwarderSecurityGroups:

CHANGELOG.md

@@ -1,8 +1,15 @@
+### v1.13.0 - 2024/02/23
+##### Features
+* Go beyond 4096b limit on CF Parameter for event triggers on SAR deployment [#627](https://github.com/elastic/elastic-serverless-forwarder/pull/627)
+
+### v1.12.0 - 2024/02/13
+##### Features
+* Add outputs for Lambda function ARN and IAM Role ARN [#552](https://github.com/elastic/elastic-serverless-forwarder/pull/552)
+
 ### v1.11.0 - 2023/12/08
 ##### Features
 * Add user agent with information about ESF version and host environment: [#537](https://github.com/elastic/elastic-serverless-forwarder/pull/537)
 * Remove calls to `sqs.DeleteMessage` and refactor storage decorators: [#544](https://github.com/elastic/elastic-serverless-forwarder/pull/544)
-* Add outputs for Lambda function ARN and IAM Role ARN [#552](https://github.com/elastic/elastic-serverless-forwarder/pull/552)
 ##### Bug fixes
 * Fix regression when both `json_content_type: single` and `expand_event_list_from_field` are set: [#553](https://github.com/elastic/elastic-serverless-forwarder/pull/553)
 

docs/en/aws-deploy-elastic-serverless-forwarder.asciidoc

@@ -259,9 +259,13 @@ Make sure you include all the KMS keys used to encrypt the data. For example, S3
 These parameters define your specific <<aws-serverless-forwarder-inputs>> or 'event triggers'.
 
 - `ElasticServerlessForwarderSQSEvents`: Add a comma delimited list of Direct SQS queue ARNs to set as event triggers for the forwarder (if any).
+- `ElasticServerlessForwarderSQSEvents2`: Add a comma delimited list of Direct SQS queue ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderSQSEvents).
 - `ElasticServerlessForwarderS3SQSEvents`: Add a comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the forwarder (if any).
+- `ElasticServerlessForwarderS3SQSEvents2`: Add a comma delimited list of S3 SQS Event Notifications ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderS3SQSEvents).
 - `ElasticServerlessForwarderKinesisEvents`: Add a comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the forwarder (if any).
+- `ElasticServerlessForwarderKinesisEvents2`: Add a comma delimited list of Kinesis Data Stream ARNs to set as event triggers for the forwarder (if limit is reach on ElasticServerlessForwarderKinesisEvents).
 - `ElasticServerlessForwarderCloudWatchLogsEvents`: Add a comma delimited list of Cloudwatch Logs log group ARNs to set subscription filters on the forwarder (if any).
+- `ElasticServerlessForwarderCloudWatchLogsEvents2`: Add a comma delimited list of Cloudwatch Logs log group ARNs to set subscription filters on the forwarder (if limit is reach on ElasticServerlessForwarderCloudWatchLogsEvents).
 
 [NOTE]
 ====
@@ -361,9 +365,13 @@ aws serverlessrepo list-application-versions --application-id arn:aws:serverless
             ElasticServerlessForwarderSSMSecrets: ""
             ElasticServerlessForwarderKMSKeys: ""
             ElasticServerlessForwarderSQSEvents: ""
+            ElasticServerlessForwarderSQSEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0
             ElasticServerlessForwarderS3SQSEvents: ""
+            ElasticServerlessForwarderS3SQSEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0
             ElasticServerlessForwarderKinesisEvents: ""
+            ElasticServerlessForwarderKinesisEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0
             ElasticServerlessForwarderCloudWatchLogsEvents: ""
+            ElasticServerlessForwarderCloudWatchLogsEvents2: "" ## IF SEMANTIC VERSION GREATER THAN 1.12.0
             ElasticServerlessForwarderS3Buckets: ""
             ElasticServerlessForwarderSecurityGroups: ""
             ElasticServerlessForwarderSubnets: ""