diff --git a/.internal/aws/cloudformation/macro.yaml b/.internal/aws/cloudformation/macro.yaml index df58b39a..ecce4fb3 100644 --- a/.internal/aws/cloudformation/macro.yaml +++ b/.internal/aws/cloudformation/macro.yaml @@ -5,9 +5,17 @@ Description: > SAM Template for the macro, not intended to be deployed on its own +Conditions: + MacroESFAWS: !Equals + - %awsOrGov% + - aws + MacroESFAWSGov: !Equals + - %awsOrGov% + - aws-us-gov Resources: MacroElasticServerlessForwarderFunction: Type: AWS::Serverless::Function + Condition: MacroESFAWS Properties: InlineCode: | import hashlib @@ -243,12 +251,258 @@ Resources: return {"status": "SUCCESS", "requestId": event["requestId"], "fragment": event["fragment"]} Handler: index.handler Runtime: python3.9 + MacroElasticServerlessForwarderFunctionAWSGov: + Type: AWS::Serverless::Function + Condition: MacroESFAWSGov + Properties: + InlineCode: | + import hashlib + import random + import string + + + def hex_suffix(src): + return hashlib.sha256(src.encode("utf-8")).hexdigest()[:10] + + def create_events(event): + events_fragment = {} + parameters = event["templateParameterValues"] + if "ElasticServerlessForwarderKinesisEvents" in parameters: + for kinesis_event in parameters["ElasticServerlessForwarderKinesisEvents"]: + kinesis_event = kinesis_event.strip() + if len(kinesis_event) == 0: + continue + + kinesis_event_name = f"KinesisEvent{hex_suffix(kinesis_event)}" + events_fragment[kinesis_event_name] = { + "Type": "Kinesis", + "Properties": { + "Stream": kinesis_event, + "StartingPosition": "TRIM_HORIZON", + "BatchSize": 10, + "Enabled": True, + } + } + + if "ElasticServerlessForwarderSQSEvents" in parameters: + for sqs_event in parameters["ElasticServerlessForwarderSQSEvents"]: + sqs_event = sqs_event.strip() + if len(sqs_event) == 0: + continue + + sqs_event_name = f"SQSEvent{hex_suffix(sqs_event)}" + events_fragment[sqs_event_name] = { + "Type": "SQS", + "Properties": { + "Queue": sqs_event, + "BatchSize": 10, + "Enabled": True, + } + } + + if "ElasticServerlessForwarderS3SQSEvents" in parameters: + for s3_sqs_event in parameters["ElasticServerlessForwarderS3SQSEvents"]: + s3_sqs_event = s3_sqs_event.strip() + if len(s3_sqs_event) == 0: + continue + + s3_sqs_event_name = f"S3SQSEvent{hex_suffix(s3_sqs_event)}" + events_fragment[s3_sqs_event_name] = { + "Type": "SQS", + "Properties": { + "Queue": s3_sqs_event, + "BatchSize": 10, + "Enabled": True, + } + } + + if "ElasticServerlessForwarderCloudWatchLogsEvents" in parameters: + for cloudwatch_logs_event in parameters["ElasticServerlessForwarderCloudWatchLogsEvents"]: + cloudwatch_logs_event = cloudwatch_logs_event.strip() + if len(cloudwatch_logs_event) == 0: + continue + + arn_components = cloudwatch_logs_event.split(":") + cloudwatch_logs_group_name = arn_components[6] + + cloudwatch_logs_event_name = f"CloudWatchLogsEvent{hex_suffix(cloudwatch_logs_group_name)}" + events_fragment[cloudwatch_logs_event_name] = { + "Type": "CloudWatchLogs", + "Properties": { + "FilterPattern": "", + "LogGroupName": cloudwatch_logs_group_name, + } + } + + return events_fragment + + + def create_policy(event): + policy_fragment = { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyName": { + "Fn::Join": ["-", ["elastic-serverless-forwarder-policy", { + "Fn::Select": [4, { + "Fn::Split": ["-", { + "Fn::Select": [2, { + "Fn::Split": ["/", { + "Ref": "AWS::StackId" + }] + }] + }] + }] + }]] + }, + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [] + }, + "Roles": [{ + "Ref": "ApplicationElasticServerlessForwarderRole" + }] + } + } + + parameters = event["templateParameterValues"] + if "ElasticServerlessForwarderS3ConfigFile" in parameters: + bucket_name_and_object_key = parameters["ElasticServerlessForwarderS3ConfigFile"].replace("s3://", "") + resource = f"arn:aws-us-gov:s3:::{bucket_name_and_object_key}" + if len(resource) > 0: + policy_fragment["Properties"]["PolicyDocument"]["Statement"].append( + { + "Effect": "Allow", + "Action": "s3:GetObject", + "Resource": resource + } + ) + + if "ElasticServerlessForwarderSSMSecrets" in parameters: + ssm_secrets_arn = [x for x in parameters["ElasticServerlessForwarderSSMSecrets"] if len(x.strip()) > 0] + + if len(ssm_secrets_arn) > 0: + policy_fragment["Properties"]["PolicyDocument"]["Statement"].append( + { + "Effect": "Allow", + "Action": "secretsmanager:GetSecretValue", + "Resource": ssm_secrets_arn + } + ) + + if "ElasticServerlessForwarderKMSKeys" in parameters: + kms_keys_arn = [x for x in parameters["ElasticServerlessForwarderKMSKeys"] if len(x.strip()) > 0] + if len(kms_keys_arn) > 0: + policy_fragment["Properties"]["PolicyDocument"]["Statement"].append( + { + "Effect": "Allow", + "Action": "kms:Decrypt", + "Resource": kms_keys_arn + } + ) + + if "ElasticServerlessForwarderS3Buckets" in parameters: + s3_buckets_arn = [x for x in parameters["ElasticServerlessForwarderS3Buckets"] if len(x.strip()) > 0] + if len(s3_buckets_arn) > 0: + policy_fragment["Properties"]["PolicyDocument"]["Statement"].append( + { + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": s3_buckets_arn + } + ) + + resources = [] + for s3_bucket_with_notification in s3_buckets_arn: + resources.append(f"{s3_bucket_with_notification}/*") + + if len(resources) > 0: + policy_fragment["Properties"]["PolicyDocument"]["Statement"].append( + { + "Effect": "Allow", + "Action": "s3:GetObject", + "Resource": resources + } + ) + + policy_fragment["Properties"]["PolicyDocument"]["Statement"].append( + { + "Effect": "Allow", + "Action": "sqs:SendMessage", + "Resource": [ + {"Fn::GetAtt": ["ElasticServerlessForwarderReplayQueue", "Arn"]}, + {"Fn::GetAtt": ["ElasticServerlessForwarderContinuingQueue", "Arn"]}, + ] + } + ) + + policy_fragment["Properties"]["PolicyDocument"]["Statement"].append( + { + "Effect": "Allow", + "Action": "ec2:DescribeRegions", + "Resource": "*", + } + ) + + return policy_fragment + + + def create_vpc_config(event): + parameters = event["templateParameterValues"] + vpc_config_fragment = {} + + security_groups = [] + if "ElasticServerlessForwarderSecurityGroups" in parameters: + security_groups = [x for x in parameters["ElasticServerlessForwarderSecurityGroups"] if len(x.strip()) > 0] + + subnets = [] + if "ElasticServerlessForwarderSubnets" in parameters: + subnets = [x for x in parameters["ElasticServerlessForwarderSubnets"] if len(x.strip()) > 0] + + if len(security_groups) > 0: + vpc_config_fragment["SecurityGroupIds"] = security_groups + + if len(subnets) > 0: + vpc_config_fragment["SubnetIds"] = subnets + + if "SubnetIds" in vpc_config_fragment and "SecurityGroupIds" not in vpc_config_fragment: + vpc_config_fragment["SecurityGroupIds"] = [] + + if "SecurityGroupIds" in vpc_config_fragment and "SubnetIds" not in vpc_config_fragment: + vpc_config_fragment["SubnetIds"] = [] + + return vpc_config_fragment + + def handler(event, context): + vpc_config = create_vpc_config(event) + if vpc_config: + event["fragment"]["ApplicationElasticServerlessForwarder"]["Properties"]["VpcConfig"] = vpc_config + + created_events = create_events(event) + for created_event in created_events: + event["fragment"]["ApplicationElasticServerlessForwarder"]["Properties"]["Events"][created_event] = created_events[created_event] + + created_policy = create_policy(event) + event["fragment"]["ElasticServerlessForwarderPolicy"] = created_policy + event["fragment"]["ApplicationElasticServerlessForwarder"]["DependsOn"] = "ElasticServerlessForwarderPolicy" + + + return {"status": "SUCCESS", "requestId": event["requestId"], "fragment": event["fragment"]} + Handler: index.handler + Runtime: python3.9 MacroElasticServerlessForwarder: Type: AWS::CloudFormation::Macro + Condition: MacroESFAWS Properties: - Description: Expand parameters to Events and Policy for %sarAppName% + Description: Expand parameters to Events and Policy for %sarAppName% in %awsOrGov% FunctionName: !GetAtt MacroElasticServerlessForwarderFunction.Arn Name: %sarAppName%-macro + MacroElasticServerlessForwarderGov: + Type: AWS::CloudFormation::Macro + Condition: MacroESFAWSGov + Properties: + Description: Expand parameters to Events and Policy for %sarAppName% in %awsOrGov% + FunctionName: !GetAtt MacroElasticServerlessForwarderFunctionAWSGov.Arn + Name: %sarAppName%-macro Metadata: AWS::ServerlessRepo::Application: Name: helper-macro-%sarAppName% diff --git a/.internal/aws/cloudformation/template.yaml b/.internal/aws/cloudformation/template.yaml index ecc72ccb..b7364115 100644 --- a/.internal/aws/cloudformation/template.yaml +++ b/.internal/aws/cloudformation/template.yaml @@ -51,13 +51,13 @@ Resources: Type: AWS::Serverless::Application Properties: Location: - ApplicationId: arn:aws:serverlessrepo:%awsRegion%:%accountID%:applications/helper-macro-%sarAppName% + ApplicationId: arn:%awsOrGov%:serverlessrepo:%awsRegion%:%accountID%:applications/helper-macro-%sarAppName% SemanticVersion: %semanticVersion% ElasticServerlessForwarderApplication: Type: AWS::Serverless::Application Properties: Location: - ApplicationId: arn:aws:serverlessrepo:%awsRegion%:%accountID%:applications/helper-application-%sarAppName% + ApplicationId: arn:%awsOrGov%:serverlessrepo:%awsRegion%:%accountID%:applications/helper-application-%sarAppName% SemanticVersion: %semanticVersion% Parameters: ElasticServerlessForwarderS3ConfigFile: !Ref ElasticServerlessForwarderS3ConfigFile diff --git a/.internal/aws/scripts/dist.sh b/.internal/aws/scripts/dist.sh index a5b53f2b..009acf99 100755 --- a/.internal/aws/scripts/dist.sh +++ b/.internal/aws/scripts/dist.sh @@ -35,7 +35,18 @@ CODE_URI="${TMPDIR}/sources" trap "rm -rf ${TMPDIR}" EXIT -aws s3api get-bucket-location --bucket "${BUCKET}" || aws s3api create-bucket --acl private --bucket "${BUCKET}" --region "${REGION}" --create-bucket-configuration LocationConstraint="${REGION}" +aws s3api get-bucket-location --bucket "${BUCKET}" --region "${REGION}" || aws s3api create-bucket --acl private --bucket "${BUCKET}" --region "${REGION}" --create-bucket-configuration LocationConstraint="${REGION}" + +# Check if region is in AWS GovCloud and create bucket arn +if [[ ${REGION} == *"$gov"* ]]; then + BUCKET_ARN="arn:aws-us-gov:s3:::${BUCKET}" + AWS_OR_AWS_GOV="aws-us-gov" +else + BUCKET_ARN="arn:aws:s3:::${BUCKET}" + AWS_OR_AWS_GOV="aws" +fi + +BUCKET_RESOURCE="${BUCKET_ARN}/*" cat < "${TMPDIR}/policy.json" { @@ -47,7 +58,7 @@ cat < "${TMPDIR}/policy.json" "Service": "serverlessrepo.amazonaws.com" }, "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::${BUCKET}/*", + "Resource": "${BUCKET_RESOURCE}", "Condition" : { "StringEquals": { "aws:SourceAccount": "${ACCOUNT_ID}" @@ -58,7 +69,7 @@ cat < "${TMPDIR}/policy.json" } EOF -aws s3api put-bucket-policy --bucket "${BUCKET}" --policy "file://${TMPDIR}/policy.json" +aws s3api put-bucket-policy --bucket "${BUCKET}" --region "${REGION}" --policy "file://${TMPDIR}/policy.json" mkdir -v -p "${CODE_URI}" cp -v requirements.txt "${CODE_URI}/" cp -v main_aws.py "${CODE_URI}/" @@ -67,8 +78,8 @@ find {handlers,share,shippers,storage} -not -name "*__pycache__*" -name "*.py" - cp -v LICENSE.txt "${CODE_URI}/LICENSE.txt" cp -v docs/README-AWS.md "${CODE_URI}/README.md" -sed -e "s|%codeUri%|${CODE_URI}|g" -e "s/%sarAppName%/${SAR_APP_NAME}/g" -e "s/%sarAuthorName%/${SAR_AUTHOR_NAME}/g" -e "s/%semanticVersion%/${SEMANTIC_VERSION}/g" -e "s/%awsRegion%/${REGION}/g" .internal/aws/cloudformation/macro.yaml > "${TMPDIR}/macro.yaml" -sed -e "s|%codeUri%|${CODE_URI}|g" -e "s/%sarAppName%/${SAR_APP_NAME}/g" -e "s/%sarAuthorName%/${SAR_AUTHOR_NAME}/g" -e "s/%semanticVersion%/${SEMANTIC_VERSION}/g" -e "s/%awsRegion%/${REGION}/g" -e "s/%accountID%/${ACCOUNT_ID}/g" .internal/aws/cloudformation/template.yaml > "${TMPDIR}/template.yaml" +sed -e "s|%codeUri%|${CODE_URI}|g" -e "s/%sarAppName%/${SAR_APP_NAME}/g" -e "s/%sarAuthorName%/${SAR_AUTHOR_NAME}/g" -e "s/%semanticVersion%/${SEMANTIC_VERSION}/g" -e "s/%awsRegion%/${REGION}/g" -e "s/%awsOrGov%/${AWS_OR_AWS_GOV}/g" .internal/aws/cloudformation/macro.yaml > "${TMPDIR}/macro.yaml" +sed -e "s|%codeUri%|${CODE_URI}|g" -e "s/%sarAppName%/${SAR_APP_NAME}/g" -e "s/%sarAuthorName%/${SAR_AUTHOR_NAME}/g" -e "s/%semanticVersion%/${SEMANTIC_VERSION}/g" -e "s/%awsRegion%/${REGION}/g" -e "s/%accountID%/${ACCOUNT_ID}/g" -e "s/%awsOrGov%/${AWS_OR_AWS_GOV}/g" .internal/aws/cloudformation/template.yaml > "${TMPDIR}/template.yaml" sed -e "s|%codeUri%|${CODE_URI}|g" -e "s/%sarAppName%/${SAR_APP_NAME}/g" -e "s/%sarAuthorName%/${SAR_AUTHOR_NAME}/g" -e "s/%semanticVersion%/${SEMANTIC_VERSION}/g" -e "s/%awsRegion%/${REGION}/g" -e "s/%codeURIBucket%/${BUCKET}/g" .internal/aws/cloudformation/application.yaml > "${TMPDIR}/application.yaml" sam build --debug --use-container --build-dir "${TMPDIR}/.aws-sam/build/macro" --template-file "${TMPDIR}/macro.yaml" --region "${REGION}"