diff --git a/mwaa/README.md b/mwaa/README.md new file mode 100644 index 0000000..a25b2ce --- /dev/null +++ b/mwaa/README.md @@ -0,0 +1,180 @@ +# Amazon Managed Workflows for Apache Airflow(MWAA) Module + +This terraform module can be used to deploy [Amazon Managed Workflows for Apache Airflow(MWAA)](https://docs.aws.amazon.com/mwaa/latest/userguide/what-is-mwaa.html) environment. + + ✅ Deployment examples can be found under [examples](https://github.com/aws-ia/terraform-aws-mwaa/tree/main/examples) folder. + + ✅ Amazon MWAA documentation for more details about [Amazon MWAA](https://docs.aws.amazon.com/mwaa/index.html) + + ✅ Amazon MWAA for Analytics [Workshop](https://catalog.us-east-1.prod.workshops.aws/workshops/795e88bb-17e2-498f-82d1-2104f4824168/en-US) + +## Amazon MWAA Architecture + +

+ example of Amazon MWAA Architecture for an example public deployment +

+ +## Usage + +The example below builds Amazon MWAA environment with existing VPC and Private Subnets. +Amazon MWAA supporting resources S3 bucket, IAM role and Security groups created by this module by default. +This module allows you to bring your own S3 bucket, IAM role and Security group. + +```hcl +module "mwaa" { + source = "aws-ia/mwaa/aws" + + name = "basic-mwaa" + airflow_version = "2.2.2" + environment_class = "mw1.medium" + + vpc_id = "" + private_subnet_ids = ["",""] + + min_workers = 1 + max_workers = 25 + webserver_access_mode = "PUBLIC_ONLY" # Default PRIVATE_ONLY for production environments + + iam_role_additional_policies = { + "additional-policy-1" = "" + "additional-policy-2" = "" + } + + logging_configuration = { + dag_processing_logs = { + enabled = true + log_level = "INFO" + } + + scheduler_logs = { + enabled = true + log_level = "INFO" + } + + task_logs = { + enabled = true + log_level = "INFO" + } + + webserver_logs = { + enabled = true + log_level = "INFO" + } + + worker_logs = { + enabled = true + log_level = "INFO" + } + } + + airflow_configuration_options = { + "core.load_default_connections" = "false" + "core.load_examples" = "false" + "webserver.dag_default_view" = "tree" + "webserver.dag_orientation" = "TB" + "logging.logging_level" = "INFO" + } +} +``` + +## Security + +See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. + +## License + +Apache-2.0 Licensed. See [LICENSE](https://github.com/aws-ia/terraform-aws-mwaa/blob/main/LICENSE). + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [aws](#requirement\_aws) | >= 4.63.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 4.63.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_mwaa_environment.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/mwaa_environment) | resource | +| [aws_s3_bucket.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket_public_access_block.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | +| [aws_s3_bucket_server_side_encryption_configuration.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | +| [aws_s3_bucket_versioning.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource | +| [aws_security_group.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.mwaa_sg_inbound](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.mwaa_sg_inbound_vpn](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.mwaa_sg_outbound](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.mwaa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.mwaa_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [airflow\_configuration\_options](#input\_airflow\_configuration\_options) | (Optional) The airflow\_configuration\_options parameter specifies airflow override options. | `any` | `null` | no | +| [airflow\_version](#input\_airflow\_version) | (Optional) Airflow version of your environment, will be set by default to the latest version that MWAA supports. | `string` | `null` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Create IAM role for MWAA | `bool` | `true` | no | +| [create\_s3\_bucket](#input\_create\_s3\_bucket) | Create new S3 bucket for MWAA. | `string` | `true` | no | +| [create\_security\_group](#input\_create\_security\_group) | Create security group for MWAA | `bool` | `true` | no | +| [dag\_s3\_path](#input\_dag\_s3\_path) | (Required) The relative path to the DAG folder on your Amazon S3 storage bucket. For example, dags. | `string` | `"dags"` | no | +| [environment\_class](#input\_environment\_class) | (Optional) Environment class for the cluster. Possible options are mw1.small, mw1.medium, mw1.large, mw1.xlarge, mw1.2xlarge.
Will be set by default to mw1.small. Please check the AWS Pricing for more information about the environment classes. | `string` | `"mw1.small"` | no | +| [execution\_role\_arn](#input\_execution\_role\_arn) | (Required) The Amazon Resource Name (ARN) of the task execution role that the Amazon MWAA and its environment can assume
Mandatory if `create_iam_role=false` | `string` | `null` | no | +| [force\_detach\_policies](#input\_force\_detach\_policies) | IAM role Force detach policies | `bool` | `false` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | IAM Role Name to be created if execution\_role\_arn is null | `string` | `null` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `"/"` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | IAM role Permission boundary | `string` | `null` | no | +| [kms\_key](#input\_kms\_key) | (Optional) The Amazon Resource Name (ARN) of your KMS key that you want to use for encryption.
Will be set to the ARN of the managed KMS key aws/airflow by default. | `string` | `null` | no | +| [logging\_configuration](#input\_logging\_configuration) | (Optional) The Apache Airflow logs which will be send to Amazon CloudWatch Logs. | `any` | `null` | no | +| [max\_workers](#input\_max\_workers) | (Optional) The maximum number of workers that can be automatically scaled up.
Value need to be between 1 and 25. Will be 10 by default | `number` | `10` | no | +| [min\_workers](#input\_min\_workers) | (Optional) The minimum number of workers that you want to run in your environment. Will be 1 by default. | `number` | `1` | no | +| [name](#input\_name) | (Required) The name of the Apache Airflow MWAA Environment | `string` | n/a | yes | +| [plugins\_s3\_object\_version](#input\_plugins\_s3\_object\_version) | (Optional) The plugins.zip file version you want to use. | `string` | `null` | no | +| [plugins\_s3\_path](#input\_plugins\_s3\_path) | (Optional) The relative path to the plugins.zip file on your Amazon S3 storage bucket. For example, plugins.zip. If a relative path is provided in the request, then plugins\_s3\_object\_version is required. | `string` | `null` | no | +| [private\_subnet\_ids](#input\_private\_subnet\_ids) | (Required) The private subnet IDs in which the environment should be created.
MWAA requires two subnets. | `list(string)` | n/a | yes | +| [requirements\_s3\_object\_version](#input\_requirements\_s3\_object\_version) | (Optional) The requirements.txt file version you want to use. | `string` | `null` | no | +| [requirements\_s3\_path](#input\_requirements\_s3\_path) | (Optional) The relative path to the requirements.txt file on your Amazon S3 storage bucket. For example, requirements.txt. If a relative path is provided in the request, then requirements\_s3\_object\_version is required. | `string` | `null` | no | +| [schedulers](#input\_schedulers) | (Optional) The number of schedulers that you want to run in your environment. | `string` | `null` | no | +| [security\_group\_ids](#input\_security\_group\_ids) | Security group IDs for MWAA | `list(string)` | `[]` | no | +| [source\_bucket\_arn](#input\_source\_bucket\_arn) | (Required) The Amazon Resource Name (ARN) of your Amazon S3 storage bucket. For example, arn:aws:s3:::airflow-mybucketname | `string` | `null` | no | +| [source\_bucket\_name](#input\_source\_bucket\_name) | New bucket will be created with the given name for MWAA when create\_s3\_bucket=true.
If set to null, then the default bucket name prefix will be set, irrespective of the value of `var.use_source_bucket_name_as_prefix` | `string` | `null` | no | +| [source\_cidr](#input\_source\_cidr) | (Required) Source CIDR block which will be allowed on MWAA SG to access Airflow UI
Used only if `create_security_group=true` | `list(string)` | `[]` | no | +| [startup\_script\_s3\_object\_version](#input\_startup\_script\_s3\_object\_version) | (Optional) The version of the startup shell script you want to use. You must specify the version ID that Amazon S3 assigns to the file every time you update the script. | `string` | `null` | no | +| [startup\_script\_s3\_path](#input\_startup\_script\_s3\_path) | (Optional) The relative path to the script hosted in your bucket. The script runs as your environment starts before starting the Apache Airflow process. Use this script to install dependencies, modify configuration options, and set environment variables. | `string` | `null` | no | +| [tags](#input\_tags) | (Optional) A map of resource tags to associate with the resource | `map(string)` | `{}` | no | +| [use\_source\_bucket\_name\_as\_prefix](#input\_use\_source\_bucket\_name\_as\_prefix) | Whether or not to use the `var.source_bucket_name` as the S3 bucket name prefix | `bool` | `true` | no | +| [vpc\_id](#input\_vpc\_id) | (Required) VPC ID to deploy the MWAA Environment.
Mandatory if `create_security_group=true` | `string` | `""` | no | +| [webserver\_access\_mode](#input\_webserver\_access\_mode) | (Optional) Specifies whether the webserver should be accessible over the internet or via your specified VPC. Possible options: PRIVATE\_ONLY (default) and PUBLIC\_ONLY | `string` | `"PRIVATE_ONLY"` | no | +| [weekly\_maintenance\_window\_start](#input\_weekly\_maintenance\_window\_start) | (Optional) Specifies the start date for the weekly maintenance window | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [aws\_s3\_bucket\_name](#output\_aws\_s3\_bucket\_name) | S3 bucket Name of the MWAA Environment | +| [mwaa\_arn](#output\_mwaa\_arn) | The ARN of the MWAA Environment | +| [mwaa\_role\_arn](#output\_mwaa\_role\_arn) | IAM Role ARN of the MWAA Environment | +| [mwaa\_role\_name](#output\_mwaa\_role\_name) | IAM role name of the MWAA Environment | +| [mwaa\_security\_group\_id](#output\_mwaa\_security\_group\_id) | Security group id of the MWAA Environment | +| [mwaa\_service\_role\_arn](#output\_mwaa\_service\_role\_arn) | The Service Role ARN of the Amazon MWAA Environment | +| [mwaa\_status](#output\_mwaa\_status) | The status of the Amazon MWAA Environment | +| [mwaa\_webserver\_url](#output\_mwaa\_webserver\_url) | The webserver URL of the MWAA Environment | + diff --git a/mwaa/data.tf b/mwaa/data.tf new file mode 100644 index 0000000..1a8e7b9 --- /dev/null +++ b/mwaa/data.tf @@ -0,0 +1,196 @@ +data "aws_partition" "current" {} + +data "aws_region" "current" {} + +data "aws_caller_identity" "current" {} + +# --------------------------------------------------------------------------------------------------------------------- +# MWAA Role +# --------------------------------------------------------------------------------------------------------------------- +data "aws_iam_policy_document" "mwaa_assume" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["airflow.amazonaws.com"] + } + + principals { + type = "Service" + identifiers = ["airflow-env.amazonaws.com"] + } + + principals { + type = "Service" + identifiers = ["batch.amazonaws.com"] + } + + principals { + type = "Service" + identifiers = ["ssm.amazonaws.com"] + } + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + principals { + type = "Service" + identifiers = ["s3.amazonaws.com"] + } + } +} +#tfsec:ignore:AWS099 +data "aws_iam_policy_document" "mwaa" { + statement { + effect = "Allow" + actions = [ + "airflow:PublishMetrics", + "airflow:CreateWebLoginToken" + ] + resources = [ + "arn:${data.aws_partition.current.id}:airflow:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:environment/${var.name}" + ] + } + statement { + effect = "Allow" + actions = [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ] + resources = [ + local.source_bucket_arn, + "${local.source_bucket_arn}/*" + ] + } + + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents", + "logs:GetLogEvents", + "logs:GetLogRecord", + "logs:GetLogGroupFields", + "logs:GetQueryResults" + ] + resources = [ + "arn:${data.aws_partition.current.id}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:airflow-${var.name}-*" + ] + } + + statement { + effect = "Allow" + actions = [ + "logs:DescribeLogGroups", + "cloudwatch:PutMetricData", + "s3:GetAccountPublicAccessBlock" + ] + resources = [ + "*" + ] + } + + statement { + effect = "Allow" + actions = [ + "sqs:ChangeMessageVisibility", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:SendMessage" + ] + resources = [ + "arn:${data.aws_partition.current.id}:sqs:${data.aws_region.current.name}:*:airflow-celery-*" + ] + } + + # See note in https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-create-role.html + # if MWAA is using a AWS managed KMS key, we have to give permission to the key in ?? account + # We don't know what account AWS puts their key in so we use not_resources to grant access to all + # accounts except for ours + dynamic "statement" { + for_each = var.kms_key != null ? [] : [1] + content { + effect = "Allow" + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:GenerateDataKey*", + "kms:Encrypt" + ] + not_resources = [ + "arn:${data.aws_partition.current.id}:kms:*:${data.aws_caller_identity.current.account_id}:key/*" + ] + condition { + test = "StringLike" + variable = "kms:ViaService" + + values = [ + "sqs.${data.aws_region.current.name}.amazonaws.com" + ] + } + } + } + + dynamic "statement" { + for_each = var.kms_key != null ? [1] : [] + content { + effect = "Allow" + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:GenerateDataKey*", + "kms:Encrypt" + ] + resources = [ + var.kms_key + ] + condition { + test = "StringLike" + variable = "kms:ViaService" + + values = [ + "sqs.${data.aws_region.current.name}.amazonaws.com" + ] + } + } + } + + statement { + effect = "Allow" + actions = [ + "batch:*", + ] + resources = [ + "arn:${data.aws_partition.current.id}:batch:*:${data.aws_caller_identity.current.account_id}:*" + ] + } + + statement { + effect = "Allow" + actions = [ + "ssm:*" + ] + resources = [ + "arn:${data.aws_partition.current.id}:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/*" + ] + } + + statement { + effect = "Allow" + actions = [ + "logs:*" + ] + resources = ["arn:${data.aws_partition.current.id}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*"] + } + + statement { + effect = "Allow" + actions = ["cloudwatch:*"] + resources = ["arn:${data.aws_partition.current.id}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*"] + } +} diff --git a/mwaa/locals.tf b/mwaa/locals.tf new file mode 100644 index 0000000..8c83d27 --- /dev/null +++ b/mwaa/locals.tf @@ -0,0 +1,19 @@ +locals { + execution_role_arn = var.create_iam_role ? aws_iam_role.mwaa[0].arn : var.execution_role_arn + + security_group_ids = var.create_security_group ? concat([aws_security_group.mwaa[0].id], var.security_group_ids) : var.security_group_ids + + source_bucket_arn = var.create_s3_bucket ? aws_s3_bucket.mwaa[0].arn : var.source_bucket_arn + + source_bucket_prefix = var.source_bucket_name == null ? format("%s-%s-", "mwaa", data.aws_caller_identity.current.account_id) : (var.use_source_bucket_name_as_prefix ? var.source_bucket_name : null) + + source_bucket_name = local.source_bucket_prefix != null ? null : var.source_bucket_name + + default_airflow_configuration_options = { + "logging.logging_level" = "INFO" + } + + airflow_configuration_options = merge(local.default_airflow_configuration_options, var.airflow_configuration_options) + + iam_role_additional_policies = { for k, v in var.iam_role_additional_policies : k => v if var.create_iam_role } +} diff --git a/mwaa/main.tf b/mwaa/main.tf new file mode 100644 index 0000000..e689ee8 --- /dev/null +++ b/mwaa/main.tf @@ -0,0 +1,200 @@ +# --------------------------------------------------------------------------------------------------------------------- +# MWAA Environment +# --------------------------------------------------------------------------------------------------------------------- +resource "aws_mwaa_environment" "mwaa" { + name = var.name + airflow_version = var.airflow_version + environment_class = var.environment_class + min_workers = var.min_workers + max_workers = var.max_workers + kms_key = var.kms_key + + dag_s3_path = var.dag_s3_path + plugins_s3_object_version = var.plugins_s3_object_version + plugins_s3_path = var.plugins_s3_path + requirements_s3_path = var.requirements_s3_path + requirements_s3_object_version = var.requirements_s3_object_version + startup_script_s3_path = var.startup_script_s3_path + startup_script_s3_object_version = var.startup_script_s3_object_version + schedulers = var.schedulers + execution_role_arn = local.execution_role_arn + airflow_configuration_options = local.airflow_configuration_options + + source_bucket_arn = local.source_bucket_arn + webserver_access_mode = var.webserver_access_mode + weekly_maintenance_window_start = var.weekly_maintenance_window_start + + tags = var.tags + + network_configuration { + security_group_ids = local.security_group_ids + subnet_ids = var.private_subnet_ids + } + + logging_configuration { + dag_processing_logs { + enabled = try(var.logging_configuration.dag_processing_logs.enabled, true) + log_level = try(var.logging_configuration.dag_processing_logs.log_level, "DEBUG") + } + + scheduler_logs { + enabled = try(var.logging_configuration.scheduler_logs.enabled, true) + log_level = try(var.logging_configuration.scheduler_logs.log_level, "INFO") + } + + task_logs { + enabled = try(var.logging_configuration.task_logs.enabled, true) + log_level = try(var.logging_configuration.task_logs.log_level, "WARNING") + } + + webserver_logs { + enabled = try(var.logging_configuration.webserver_logs.enabled, true) + log_level = try(var.logging_configuration.webserver_logs.log_level, "ERROR") + } + + worker_logs { + enabled = try(var.logging_configuration.worker_logs.enabled, true) + log_level = try(var.logging_configuration.worker_logs.log_level, "CRITICAL") + } + } + + lifecycle { + ignore_changes = [ + plugins_s3_object_version, + requirements_s3_object_version, + startup_script_s3_object_version + ] + } +} + +# --------------------------------------------------------------------------------------------------------------------- +# IAM Role +# --------------------------------------------------------------------------------------------------------------------- +resource "aws_iam_role" "mwaa" { + count = var.create_iam_role ? 1 : 0 + + name = var.iam_role_name != null ? var.iam_role_name : null + name_prefix = var.iam_role_name != null ? null : "mwaa-executor" + description = "MWAA IAM Role" + assume_role_policy = data.aws_iam_policy_document.mwaa_assume.json + force_detach_policies = var.force_detach_policies + path = var.iam_role_path + permissions_boundary = var.iam_role_permissions_boundary + + tags = var.tags +} + +resource "aws_iam_role_policy" "mwaa" { + count = var.create_iam_role ? 1 : 0 + + name_prefix = "mwaa-executor" + role = aws_iam_role.mwaa[0].id + policy = data.aws_iam_policy_document.mwaa.json +} + +resource "aws_iam_role_policy_attachment" "mwaa" { + for_each = local.iam_role_additional_policies + policy_arn = each.value + role = aws_iam_role.mwaa[0].id +} + +# --------------------------------------------------------------------------------------------------------------------- +# MWAA S3 Bucket +# --------------------------------------------------------------------------------------------------------------------- +#tfsec:ignore:AWS017 tfsec:ignore:AWS002 tfsec:ignore:AWS077 +resource "aws_s3_bucket" "mwaa" { + count = var.create_s3_bucket ? 1 : 0 + + bucket = local.source_bucket_name + bucket_prefix = local.source_bucket_prefix + + tags = var.tags +} + +#tfsec:ignore:aws-s3-encryption-customer-key +resource "aws_s3_bucket_server_side_encryption_configuration" "mwaa" { + count = var.create_s3_bucket ? 1 : 0 + + bucket = aws_s3_bucket.mwaa[0].id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_versioning" "mwaa" { + count = var.create_s3_bucket ? 1 : 0 + + bucket = aws_s3_bucket.mwaa[0].id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_public_access_block" "mwaa" { + count = var.create_s3_bucket ? 1 : 0 + + bucket = aws_s3_bucket.mwaa[0].id + + block_public_acls = true + block_public_policy = true + restrict_public_buckets = true + ignore_public_acls = true +} + +# --------------------------------------------------------------------------------------------------------------------- +# MWAA Security Group +# --------------------------------------------------------------------------------------------------------------------- +resource "aws_security_group" "mwaa" { + count = var.create_security_group ? 1 : 0 + + name_prefix = "mwaa-" + description = "Security group for MWAA environment" + vpc_id = var.vpc_id + + lifecycle { + create_before_destroy = true + } + + tags = var.tags +} + +resource "aws_security_group_rule" "mwaa_sg_inbound" { + count = var.create_security_group ? 1 : 0 + + type = "ingress" + from_port = 0 + to_port = 0 + protocol = "all" + source_security_group_id = aws_security_group.mwaa[0].id + security_group_id = aws_security_group.mwaa[0].id + description = "Amazon MWAA inbound access" +} + +resource "aws_security_group_rule" "mwaa_sg_inbound_vpn" { + count = var.create_security_group && length(var.source_cidr) > 0 ? 1 : 0 + + type = "ingress" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = var.source_cidr + security_group_id = aws_security_group.mwaa[0].id + description = "VPN Access for Airflow UI" +} + +#tfsec:ignore:aws-vpc-no-public-egress-sgr +resource "aws_security_group_rule" "mwaa_sg_outbound" { + count = var.create_security_group ? 1 : 0 + + type = "egress" + from_port = 0 + to_port = 0 + protocol = "all" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.mwaa[0].id + description = "Amazon MWAA outbound access" +} diff --git a/mwaa/outputs.tf b/mwaa/outputs.tf new file mode 100644 index 0000000..36a092e --- /dev/null +++ b/mwaa/outputs.tf @@ -0,0 +1,39 @@ +output "mwaa_webserver_url" { + description = "The webserver URL of the MWAA Environment" + value = aws_mwaa_environment.mwaa.webserver_url +} + +output "mwaa_arn" { + description = "The ARN of the MWAA Environment" + value = aws_mwaa_environment.mwaa.arn +} + +output "mwaa_service_role_arn" { + description = "The Service Role ARN of the Amazon MWAA Environment" + value = aws_mwaa_environment.mwaa.service_role_arn +} + +output "mwaa_status" { + description = "The status of the Amazon MWAA Environment" + value = aws_mwaa_environment.mwaa.status +} + +output "mwaa_role_arn" { + description = "IAM Role ARN of the MWAA Environment" + value = var.execution_role_arn == null ? aws_iam_role.mwaa[0].arn : "" +} + +output "mwaa_role_name" { + description = "IAM role name of the MWAA Environment" + value = var.execution_role_arn == null ? aws_iam_role.mwaa[0].id : "" +} + +output "mwaa_security_group_id" { + description = "Security group id of the MWAA Environment" + value = var.create_security_group == true ? aws_security_group.mwaa[0].id : "" +} + +output "aws_s3_bucket_name" { + description = "S3 bucket Name of the MWAA Environment" + value = var.source_bucket_arn == null ? aws_s3_bucket.mwaa[0].id : "" +} diff --git a/mwaa/tfsec.yaml b/mwaa/tfsec.yaml new file mode 100644 index 0000000..48a6a86 --- /dev/null +++ b/mwaa/tfsec.yaml @@ -0,0 +1,5 @@ +exclude: + - aws-iam-no-policy-wildcards # Wildcards required in addon IAM policies + - aws-vpc-no-excessive-port-access # VPC settings left up to user implementation for recommended practices + - aws-vpc-no-public-ingress-acl # VPC settings left up to user implementation for recommended practices + - aws-vpc-no-public-egress-sgr # Added in v1.22 diff --git a/mwaa/variables.tf b/mwaa/variables.tf new file mode 100644 index 0000000..b58fbb5 --- /dev/null +++ b/mwaa/variables.tf @@ -0,0 +1,256 @@ +variable "name" { + description = "(Required) The name of the Apache Airflow MWAA Environment" + type = string +} + +variable "private_subnet_ids" { + description = <<-EOD + (Required) The private subnet IDs in which the environment should be created. + MWAA requires two subnets. + EOD + type = list(string) +} + +variable "airflow_configuration_options" { + description = "(Optional) The airflow_configuration_options parameter specifies airflow override options." + type = any + default = null +} + +variable "airflow_version" { + description = "(Optional) Airflow version of your environment, will be set by default to the latest version that MWAA supports." + type = string + default = null +} + +variable "dag_s3_path" { + description = "(Required) The relative path to the DAG folder on your Amazon S3 storage bucket. For example, dags." + type = string + default = "dags" +} + +variable "environment_class" { + description = <<-EOD + (Optional) Environment class for the cluster. Possible options are mw1.small, mw1.medium, mw1.large, mw1.xlarge, mw1.2xlarge. + Will be set by default to mw1.small. Please check the AWS Pricing for more information about the environment classes. + EOD + type = string + default = "mw1.small" + + validation { + condition = contains(["mw1.small", "mw1.medium", "mw1.large", "mw1.xlarge", "mw1.2xlarge"], var.environment_class) + error_message = "Invalid input, options: \"mw1.small\", \"mw1.medium\", \"mw1.large\", \"mw1.xlarge\", \"mw1.2xlarge\"." + } +} + +variable "kms_key" { + description = <<-EOD + (Optional) The Amazon Resource Name (ARN) of your KMS key that you want to use for encryption. + Will be set to the ARN of the managed KMS key aws/airflow by default. + EOD + type = string + default = null +} + +variable "logging_configuration" { + description = "(Optional) The Apache Airflow logs which will be send to Amazon CloudWatch Logs." + type = any + default = null +} + +variable "max_workers" { + description = <<-EOD + (Optional) The maximum number of workers that can be automatically scaled up. + Value need to be between 1 and 25. Will be 10 by default + EOD + type = number + default = 10 + + validation { + condition = var.max_workers > 0 && var.max_workers < 26 + error_message = "Error: Value need to be between 1 and 25." + } +} + +variable "min_workers" { + description = "(Optional) The minimum number of workers that you want to run in your environment. Will be 1 by default." + type = number + default = 1 +} + +variable "plugins_s3_object_version" { + description = "(Optional) The plugins.zip file version you want to use." + type = string + default = null +} + +variable "plugins_s3_path" { + description = "(Optional) The relative path to the plugins.zip file on your Amazon S3 storage bucket. For example, plugins.zip. If a relative path is provided in the request, then plugins_s3_object_version is required." + type = string + default = null +} + +variable "requirements_s3_object_version" { + description = "(Optional) The requirements.txt file version you want to use." + type = string + default = null +} + +variable "requirements_s3_path" { + description = "(Optional) The relative path to the requirements.txt file on your Amazon S3 storage bucket. For example, requirements.txt. If a relative path is provided in the request, then requirements_s3_object_version is required." + type = string + default = null +} + +variable "startup_script_s3_object_version" { + description = "(Optional) The version of the startup shell script you want to use. You must specify the version ID that Amazon S3 assigns to the file every time you update the script." + type = string + default = null +} + +variable "startup_script_s3_path" { + description = "(Optional) The relative path to the script hosted in your bucket. The script runs as your environment starts before starting the Apache Airflow process. Use this script to install dependencies, modify configuration options, and set environment variables." + type = string + default = null +} + +variable "schedulers" { + description = "(Optional) The number of schedulers that you want to run in your environment." + type = string + default = null +} + +variable "webserver_access_mode" { + description = "(Optional) Specifies whether the webserver should be accessible over the internet or via your specified VPC. Possible options: PRIVATE_ONLY (default) and PUBLIC_ONLY" + type = string + default = "PRIVATE_ONLY" + + validation { + condition = contains(["PRIVATE_ONLY", "PUBLIC_ONLY"], var.webserver_access_mode) + error_message = "Invalid input, options: \"PRIVATE_ONLY\", \"PUBLIC_ONLY\"." + } +} + +variable "weekly_maintenance_window_start" { + description = "(Optional) Specifies the start date for the weekly maintenance window" + type = string + default = null +} + +variable "tags" { + description = "(Optional) A map of resource tags to associate with the resource" + type = map(string) + default = {} +} +#---------------------------------------------------------------- +# MWAA IAM Role +#---------------------------------------------------------------- +variable "create_iam_role" { + description = "Create IAM role for MWAA" + type = bool + default = true +} + +variable "iam_role_name" { + description = "IAM Role Name to be created if execution_role_arn is null" + type = string + default = null +} + +variable "iam_role_permissions_boundary" { + description = "IAM role Permission boundary" + type = string + default = null +} + +variable "force_detach_policies" { + description = "IAM role Force detach policies" + type = bool + default = false +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = map(string) + default = {} +} + +variable "iam_role_path" { + description = "IAM role path" + type = string + default = "/" +} + +variable "execution_role_arn" { + description = <<-EOD + (Required) The Amazon Resource Name (ARN) of the task execution role that the Amazon MWAA and its environment can assume + Mandatory if `create_iam_role=false` + EOD + type = string + default = null +} +#---------------------------------------------------------------- +# MWAA S3 Bucket +#---------------------------------------------------------------- +variable "create_s3_bucket" { + description = "Create new S3 bucket for MWAA. " + type = string + default = true +} + +variable "source_bucket_name" { + description = <<-EOD + New bucket will be created with the given name for MWAA when create_s3_bucket=true. + If set to null, then the default bucket name prefix will be set, irrespective of the value of `var.use_source_bucket_name_as_prefix` + EOD + type = string + default = null +} + +variable "use_source_bucket_name_as_prefix" { + description = <<-EOD + Whether or not to use the `var.source_bucket_name` as the S3 bucket name prefix + EOD + type = bool + default = true +} + + +variable "source_bucket_arn" { + description = "(Required) The Amazon Resource Name (ARN) of your Amazon S3 storage bucket. For example, arn:aws:s3:::airflow-mybucketname" + type = string + default = null +} + +#---------------------------------------------------------------- +# MWAA Security groups +#---------------------------------------------------------------- +variable "create_security_group" { + description = "Create security group for MWAA" + type = bool + default = true +} + +variable "security_group_ids" { + description = "Security group IDs for MWAA" + type = list(string) + default = [] +} + +variable "vpc_id" { + description = <<-EOD + (Required) VPC ID to deploy the MWAA Environment. + Mandatory if `create_security_group=true` + EOD + type = string + default = "" +} + +variable "source_cidr" { + description = <<-EOD + (Required) Source CIDR block which will be allowed on MWAA SG to access Airflow UI + Used only if `create_security_group=true` + EOD + type = list(string) + default = [] +} diff --git a/mwaa/versions.tf b/mwaa/versions.tf new file mode 100644 index 0000000..b398951 --- /dev/null +++ b/mwaa/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.63.0" + } + } +}