From 27a834d55f1859c987e5a6d9513bf0b9649353bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20N=C3=A9meth?= Date: Tue, 25 Jun 2024 11:03:06 +0200 Subject: [PATCH] Service endpoint alias fix (#57) --- README.md | 1 + bin/tflocal | 100 +++++++++++++++++++++++++++++++------------- setup.cfg | 2 +- tests/test_apply.py | 49 ++++++++++++++++++---- 4 files changed, 116 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index a5c45ca..e06d99d 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ please refer to the man pages of `terraform --help`. ## Change Log +* v0.18.2: Fix warning on aliased custom endpoint names * v0.18.1: Fix issue with not proxied commands * v0.18.0: Add `DRY_RUN` and patch S3 backend entrypoints * v0.17.1: Add `packaging` module to install requirements diff --git a/bin/tflocal b/bin/tflocal index 68e89bd..32ccd26 100755 --- a/bin/tflocal +++ b/bin/tflocal @@ -70,6 +70,73 @@ terraform { } """ PROCESS = None +# some services have aliases which are mutually exclusive to each other +# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/custom-service-endpoints#available-endpoint-customizations +SERVICE_ALIASES = [ + ("amp", "prometheus", "prometheusservice"), + ("appautoscaling", "applicationautoscaling"), + ("appintegrations", "appintegrationsservice"), + ("ce", "costexplorer"), + ("cloudcontrol", "cloudcontrolapi"), + ("cloudhsmv2", "cloudhsm"), + ("cognitoidp", "cognitoidentityprovider"), + ("configservice", "config"), + ("cur", "costandusagereportservice"), + ("deploy", "codedeploy"), + ("dms", "databasemigration", "databasemigrationservice"), + ("ds", "directoryservice"), + ("elasticbeanstalk", "beanstalk"), + ("elasticsearch", "es", "elasticsearchservice"), + ("elb", "elasticloadbalancing"), + ("elbv2", "elasticloadbalancingv2"), + ("events", "eventbridge", "cloudwatchevents"), + ("evidently", "cloudwatchevidently"), + ("grafana", "managedgrafana", "amg"), + ("inspector2", "inspectorv2"), + ("kafka", "msk"), + ("lexmodels", "lexmodelbuilding", "lexmodelbuildingservice", "lex"), + ("lexv2models", "lexmodelsv2"), + ("location", "locationservice"), + ("logs", "cloudwatchlog", "cloudwatchlogs"), + ("oam", "cloudwatchobservabilityaccessmanager"), + ("opensearch", "opensearchservice"), + ("osis", "opensearchingestion"), + ("rbin", "recyclebin"), + ("redshiftdata", "redshiftdataapiservice"), + ("resourcegroupstaggingapi", "resourcegroupstagging"), + ("rum", "cloudwatchrum"), + ("s3", "s3api"), + ("serverlessrepo", "serverlessapprepo", "serverlessapplicationrepository"), + ("servicecatalogappregistry", "appregistry"), + ("sfn", "stepfunctions"), + ("simpledb", "sdb"), + ("transcribe", "transcribeservice"), +] +# service names to be excluded (not yet available in TF) +SERVICE_EXCLUSIONS = ["meteringmarketplace"] +# maps services to be replaced with alternative names +# skip services which do not have equivalent endpoint overrides +# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/custom-service-endpoints +SERVICE_REPLACEMENTS = { + "apigatewaymanagementapi": "", + "appconfigdata": "", + "ce": "costexplorer", + "dynamodbstreams": "", + "edge": "", + "emrserverless": "", + "iotdata": "", + "ioteventsdata": "", + "iotjobsdata": "", + "iotwireless": "", + "logs": "cloudwatchlogs", + "mediastoredata": "", + "qldbsession": "", + "rdsdata": "", + "sagemakerruntime": "", + "support": "", + "timestream": "", + "timestreamquery": "", +} # --- @@ -79,40 +146,17 @@ PROCESS = None def create_provider_config_file(provider_aliases=None): provider_aliases = provider_aliases or [] - # maps services to be replaced with alternative names - # skip services which do not have equivalent endpoint overrides - # see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/custom-service-endpoints - service_replaces = { - "apigatewaymanagementapi": "", - "appconfigdata": "", - "ce": "costexplorer", - "dynamodbstreams": "", - "edge": "", - "emrserverless": "", - "iotdata": "", - "ioteventsdata": "", - "iotjobsdata": "", - "iotwireless": "", - "logs": "cloudwatchlogs", - "mediastoredata": "", - "qldbsession": "", - "rdsdata": "", - "sagemakerruntime": "", - "support": "", - "timestream": "", - "timestreamquery": "", - } - # service names to be excluded (not yet available in TF) - service_excludes = ["meteringmarketplace"] + # Force service alias replacements + SERVICE_REPLACEMENTS.update({alias: alias_pairs[0] for alias_pairs in SERVICE_ALIASES for alias in alias_pairs if alias != alias_pairs[0]}) # create list of service names services = list(config.get_service_ports()) - services = [srvc for srvc in services if srvc not in service_excludes] + services = [srvc for srvc in services if srvc not in SERVICE_EXCLUSIONS] services = [s.replace("-", "") for s in services] - for old, new in service_replaces.items(): + for old, new in SERVICE_REPLACEMENTS.items(): try: services.remove(old) - if new: + if new and new not in services: services.append(new) except ValueError: pass diff --git a/setup.cfg b/setup.cfg index f994f56..7ec32a5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = terraform-local -version = 0.18.1 +version = 0.18.2 url = https://github.com/localstack/terraform-local author = LocalStack Team author_email = info@localstack.cloud diff --git a/tests/test_apply.py b/tests/test_apply.py index 69a471a..7e3c16c 100644 --- a/tests/test_apply.py +++ b/tests/test_apply.py @@ -99,7 +99,9 @@ def test_access_key_override_by_provider(monkeypatch): def test_s3_path_addressing(): - bucket_name = f"bucket.{short_uid()}" + # Temporarily change "." -> "-" as aws provider >5.55.0 fails with LocalStack + # by calling aws-global pseudo region at S3 bucket creation instead of us-east-1 + bucket_name = f"bucket-{short_uid()}" config = """ resource "aws_s3_bucket" "test-bucket" { bucket = "%s" @@ -164,7 +166,9 @@ def test_provider_aliases(): def test_s3_backend(): state_bucket = f"tf-state-{short_uid()}" state_table = f"tf-state-{short_uid()}" - bucket_name = f"bucket.{short_uid()}" + # Temporarily change "." -> "-" as aws provider >5.55.0 fails with LocalStack + # by calling aws-global pseudo region at S3 bucket creation instead of us-east-1 + bucket_name = f"bucket-{short_uid()}" config = """ terraform { backend "s3" { @@ -203,7 +207,9 @@ def test_dry_run(monkeypatch): monkeypatch.setenv("DRY_RUN", "1") state_bucket = "tf-state-dry-run" state_table = "tf-state-dry-run" - bucket_name = "bucket.dry-run" + # Temporarily change "." -> "-" as aws provider >5.55.0 fails with LocalStack + # by calling aws-global pseudo region at S3 bucket creation instead of us-east-1 + bucket_name = "bucket-dry-run" config = """ terraform { backend "s3" { @@ -224,7 +230,7 @@ def test_dry_run(monkeypatch): override_file = os.path.join(temp_dir, "localstack_providers_override.tf") assert check_override_file_exists(override_file) - assert check_override_file_content(override_file, is_legacy=is_legacy_tf) + assert check_override_file_backend_content(override_file, is_legacy=is_legacy_tf) # assert that bucket with state file exists s3 = client("s3", region_name="us-east-2") @@ -243,6 +249,33 @@ def test_dry_run(monkeypatch): s3.head_bucket(Bucket=bucket_name) +def test_service_endpoint_alias_replacements(monkeypatch): + monkeypatch.setenv("DRY_RUN", "1") + config = """ + provider "aws" { + region = "eu-west-1" + }""" + + temp_dir = deploy_tf_script(config, cleanup=False, user_input="yes") + override_file = os.path.join(temp_dir, "localstack_providers_override.tf") + assert check_override_file_content(override_file) + rmtree(temp_dir) + + +def check_override_file_content(override_file): + try: + with open(override_file, "r") as fp: + result = hcl2.load(fp) + result = result["provider"][0]["aws"] + except Exception as e: + raise Exception(f'Unable to parse "{override_file}" as HCL file: {e}') + + endpoints = result["endpoints"][0] + if "config" in endpoints and "configservice" in endpoints: + return False + return True + + @pytest.mark.parametrize("endpoints", [ '', 'endpoint = "http://s3-localhost.localstack.cloud:4566"', @@ -255,7 +288,9 @@ def test_s3_backend_endpoints_merge(monkeypatch, endpoints: str): monkeypatch.setenv("DRY_RUN", "1") state_bucket = "tf-state-merge" state_table = "tf-state-merge" - bucket_name = "bucket.merge" + # Temporarily change "." -> "-" as aws provider >5.55.0 fails with LocalStack + # by calling aws-global pseudo region at S3 bucket creation instead of us-east-1 + bucket_name = "bucket-merge" config = """ terraform { backend "s3" { @@ -279,7 +314,7 @@ def test_s3_backend_endpoints_merge(monkeypatch, endpoints: str): temp_dir = deploy_tf_script(config, cleanup=False, user_input="yes") override_file = os.path.join(temp_dir, "localstack_providers_override.tf") assert check_override_file_exists(override_file) - assert check_override_file_content(override_file, is_legacy=is_legacy_tf) + assert check_override_file_backend_content(override_file, is_legacy=is_legacy_tf) rmtree(temp_dir) @@ -287,7 +322,7 @@ def check_override_file_exists(override_file): return os.path.isfile(override_file) -def check_override_file_content(override_file, is_legacy: bool = False): +def check_override_file_backend_content(override_file, is_legacy: bool = False): legacy_options = ( "endpoint", "iam_endpoint",