diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c17b5d7..f2e57e87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ +### v1.17.2 - 2024/09/24 +##### Bug fixes +* Remove call to EC2:DescribeRegions API in the cloudwatch-logs input [811](https://github.com/elastic/elastic-serverless-forwarder/pull/811). + ### v1.17.1 - 2024/09/23 ##### Bug fixes -* Cache EC2:DescribeRegion API response to avoid throttling and improve performance [803](https://github.com/elastic/elastic-serverless-forwarder/pull/803). +* Cache EC2:DescribeRegions API response to avoid throttling and improve performance [803](https://github.com/elastic/elastic-serverless-forwarder/pull/803). ### v1.17.0 - 2024/07/10 ##### Features diff --git a/handlers/aws/handler.py b/handlers/aws/handler.py index 9456be29..e543019f 100644 --- a/handlers/aws/handler.py +++ b/handlers/aws/handler.py @@ -31,6 +31,7 @@ expand_event_list_from_field_resolver, get_continuing_original_input_type, get_input_from_log_group_subscription_data, + get_lambda_region, get_shipper_from_input, get_sqs_client, get_trigger_type_and_config_source, @@ -48,7 +49,6 @@ def lambda_handler(lambda_event: dict[str, Any], lambda_context: context_.Contex AWS Lambda handler in handler.aws package Parses the config and acts as front controller for inputs """ - shared_logger.debug("lambda triggered", extra={"invoked_function_arn": lambda_context.invoked_function_arn}) try: @@ -144,11 +144,16 @@ def lambda_handler(lambda_event: dict[str, Any], lambda_context: context_.Contex shared_logger.info("trigger", extra={"size": len(cloudwatch_logs_event["logEvents"])}) + lambda_region = get_lambda_region() + input_id, event_input = get_input_from_log_group_subscription_data( config, cloudwatch_logs_event["owner"], cloudwatch_logs_event["logGroup"], cloudwatch_logs_event["logStream"], + # As of today, the cloudwatch trigger is always in + # the same region as the lambda function. + lambda_region, ) if event_input is None: diff --git a/handlers/aws/utils.py b/handlers/aws/utils.py index 0a4a1c5c..f973b657 100644 --- a/handlers/aws/utils.py +++ b/handlers/aws/utils.py @@ -2,7 +2,6 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; # you may not use this file except in compliance with the Elastic License 2.0. import os -from functools import lru_cache from typing import Any, Callable, Optional import boto3 @@ -32,6 +31,27 @@ INTEGRATION_SCOPE_GENERIC: str = "generic" +def get_lambda_region() -> str: + """ + Get the AWS region where the Lambda function is running. + + Returns the value of the `AWS_REGION` environment variable. If the + `AWS_REGION` variable is not set, it returns the value of the + `AWS_DEFAULT_REGION` variable. + + If neither variable is set, it raises a `ValueError`. + + Returns: + str: The AWS region. + """ + region = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") + + if region is None: + raise ValueError("AWS region not found in environment variables.") + + return region + + def get_sqs_client() -> BotoBaseClient: """ Getter for sqs client @@ -386,49 +406,34 @@ def get_account_id_from_arn(lambda_arn: str) -> str: return arn_components[4] -@lru_cache() -def describe_regions(all_regions: bool = True) -> Any: - """ - Fetches all regions from AWS and returns the response. - - :return: The response from the describe_regions method - """ - return get_ec2_client().describe_regions(AllRegions=all_regions) - - def get_input_from_log_group_subscription_data( - config: Config, account_id: str, log_group_name: str, log_stream_name: str + config: Config, account_id: str, log_group_name: str, log_stream_name: str, region: str ) -> tuple[str, Optional[Input]]: """ - This function is not less resilient than the previous get_log_group_arn_and_region_from_log_group_name() - We avoid to call the describe_log_streams on the logs' client, since we have no way to apply the proper - throttling because we'd need to know the number of concurrent lambda running at the time of the call. - In order to not hardcode the list of regions we rely on ec2 DescribeRegions - as much weird as it is - that I found - no information about having any kind of throttling. We add IAM permissions for it in deployment. - """ - all_regions = describe_regions(all_regions=True) - assert "Regions" in all_regions - for region_data in all_regions["Regions"]: - region = region_data["RegionName"] + Look up for the input in the configuration using the information + from the log event. - aws_or_gov = "aws" - if "gov" in region: - aws_or_gov = "aws-us-gov" + It looks for the log stream arn, if not found it looks for the + log group arn. + """ + partition = "aws" + if "gov" in region: + partition = "aws-us-gov" - log_stream_arn = ( - f"arn:{aws_or_gov}:logs:{region}:{account_id}:log-group:{log_group_name}:log-stream:{log_stream_name}" - ) - event_input = config.get_input_by_id(log_stream_arn) + log_stream_arn = ( + f"arn:{partition}:logs:{region}:{account_id}:log-group:{log_group_name}:log-stream:{log_stream_name}" + ) + event_input = config.get_input_by_id(log_stream_arn) - if event_input is not None: - return log_stream_arn, event_input + if event_input is not None: + return log_stream_arn, event_input - log_group_arn_components = log_stream_arn.split(":") - log_group_arn = f"{':'.join(log_group_arn_components[:-2])}:*" - event_input = config.get_input_by_id(log_group_arn) + log_group_arn_components = log_stream_arn.split(":") + log_group_arn = f"{':'.join(log_group_arn_components[:-2])}:*" + event_input = config.get_input_by_id(log_group_arn) - if event_input is not None: - return log_group_arn, event_input + if event_input is not None: + return log_group_arn, event_input return f"arn:aws:logs:%AWS_REGION%:{account_id}:log-group:{log_group_name}:*", None diff --git a/share/version.py b/share/version.py index b71be610..547fcb47 100644 --- a/share/version.py +++ b/share/version.py @@ -2,4 +2,4 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; # you may not use this file except in compliance with the Elastic License 2.0. -version = "1.17.1" +version = "1.17.2" diff --git a/tests/handlers/aws/test_integrations.py b/tests/handlers/aws/test_integrations.py index f56beb43..fc15d227 100644 --- a/tests/handlers/aws/test_integrations.py +++ b/tests/handlers/aws/test_integrations.py @@ -79,6 +79,8 @@ def setUpClass(cls) -> None: cls.localstack = lsc.start() + os.environ["AWS_REGION"] = _AWS_REGION + session = boto3.Session(region_name=_AWS_REGION) cls.aws_session = session cls.s3_client = session.client("s3", endpoint_url=cls.localstack.get_url()) diff --git a/tests/handlers/aws/test_utils.py b/tests/handlers/aws/test_utils.py index e0a102c0..e2bf845c 100644 --- a/tests/handlers/aws/test_utils.py +++ b/tests/handlers/aws/test_utils.py @@ -3,6 +3,7 @@ # you may not use this file except in compliance with the Elastic License 2.0. +import os import random import string from datetime import datetime @@ -426,45 +427,36 @@ def test_cloudwatch_id_less_than_512bytes(self) -> None: @pytest.mark.unit -class TestDescribeRegions(TestCase): +class TestGetLambdaRegion(TestCase): - @mock.patch("handlers.aws.utils.get_ec2_client", lambda: _ec2_client_mock) - def test_cache_miss(self) -> None: - from handlers.aws.utils import describe_regions + def test_with_aws_region(self) -> None: + from handlers.aws.utils import get_lambda_region - # Reset the cache info before running the test - describe_regions.cache_clear() + os.environ["AWS_REGION"] = "us-west-1" + os.environ["AWS_DEFAULT_REGION"] = "us-west-2" - # First call should be a cache miss - response = describe_regions(all_regions=False) - assert response["Regions"] is not None - assert len(response["Regions"]) == 1 - assert response["Regions"][0]["RegionName"] == "us-west-2" + region = get_lambda_region() - cache_info = describe_regions.cache_info() + assert region == "us-west-1" - assert cache_info.hits == 0 - assert cache_info.misses == 1 - assert cache_info.currsize == 1 + def test_with_aws_default_region(self) -> None: + from handlers.aws.utils import get_lambda_region - @mock.patch("handlers.aws.utils.get_ec2_client", lambda: _ec2_client_mock) - def test_cache_hits(self) -> None: - from handlers.aws.utils import describe_regions + if "AWS_REGION" in os.environ: + del os.environ["AWS_REGION"] + os.environ["AWS_DEFAULT_REGION"] = "us-west-2" - # Reset the cache info before running the test - describe_regions.cache_clear() + region = get_lambda_region() - # First call should be a cache miss and populate the cache - # Second and third calls should be cache hits. - response = describe_regions(all_regions=False) - response = describe_regions(all_regions=False) - response = describe_regions(all_regions=False) - assert response["Regions"] is not None - assert len(response["Regions"]) == 1 - assert response["Regions"][0]["RegionName"] == "us-west-2" + assert region == "us-west-2" - cache_info = describe_regions.cache_info() + def test_without_variables(self) -> None: + from handlers.aws.utils import get_lambda_region - assert cache_info.hits == 2 - assert cache_info.misses == 1 - assert cache_info.currsize == 1 + if "AWS_REGION" in os.environ: + del os.environ["AWS_REGION"] + if "AWS_DEFAULT_REGION" in os.environ: + del os.environ["AWS_DEFAULT_REGION"] + + with pytest.raises(ValueError): + get_lambda_region()