From 35dc0900a107b563d425d41d6db21457beacd5ba Mon Sep 17 00:00:00 2001 From: Davide Girardi <1390902+girodav@users.noreply.github.com> Date: Tue, 28 Nov 2023 12:12:21 +0000 Subject: [PATCH] Add ESF specific User-Agent header in outgoing Elasticsearch requests (#537) * Add ESF specific User-Agent header in outgoing Elasticsearch requests * Bump minimum supported Elastic stack version to 7.17 --- CHANGELOG.md | 4 ++++ .../aws-elastic-serverless-forwarder.asciidoc | 2 +- requirements-tests.txt | 2 +- requirements.txt | 6 +++--- share/environment.py | 17 +++++++++++++++++ share/utils.py | 6 ++++++ share/version.py | 5 +++++ shippers/es.py | 6 ++++++ tests/handlers/aws/test_integrations.py | 2 +- tests/share/test_environment.py | 17 +++++++++++++++++ tests/shippers/test_es.py | 16 +++++++++++++--- tests/testcontainers/es.py | 2 +- tests/testcontainers/logstash.py | 2 +- 13 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 share/environment.py create mode 100644 share/version.py create mode 100644 tests/share/test_environment.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7da1e6c4..d9a6007f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### v1.11.0 - 2023/11/27 +##### Features +* Add user agent with information about ESF version and host environment: [#537](https://github.com/elastic/elastic-serverless-forwarder/pull/537) + ### v1.10.0 - 2023/10/27 ##### Features * Move `_id` field to `@metadata._id` in logstash output: [#507](https://github.com/elastic/elastic-serverless-forwarder/pull/507) diff --git a/docs/en/aws-elastic-serverless-forwarder.asciidoc b/docs/en/aws-elastic-serverless-forwarder.asciidoc index aa2b8ad0..d0331e85 100644 --- a/docs/en/aws-elastic-serverless-forwarder.asciidoc +++ b/docs/en/aws-elastic-serverless-forwarder.asciidoc @@ -11,7 +11,7 @@ The Elastic Serverless Forwarder is an Amazon Web Services ({aws}) Lambda function that ships logs from your {aws} environment to Elastic. -The Elastic Serverless Forwarder works with {stack} 7.16 and later. +The Elastic Serverless Forwarder works with {stack} 7.17 and later. IMPORTANT: Using Elastic Serverless Forwarder may result in additional charges. To learn how to minimize additional charges, refer to <>. diff --git a/requirements-tests.txt b/requirements-tests.txt index fff996f1..49170d6c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,6 +8,6 @@ orjson==3.9.10 pysimdjson==5.0.2 python-rapidjson==1.13 cysimdjson==23.8 -responses==0.23.3 +responses==0.24.1 testcontainers==3.7.1 pyOpenSSL==23.3.0 diff --git a/requirements.txt b/requirements.txt index f75f9839..62cd3bb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ elastic-apm==6.19.0 -boto3==1.28.80 +boto3==1.29.7 ecs_logging==2.1.0 -elasticsearch==7.16.3 +elasticsearch==7.17.9 PyYAML==6.0.1 aws_lambda_typing==2.18.0 ujson==5.8.0 requests==2.31.0 -urllib3==1.26.15 +urllib3==1.26.18 diff --git a/share/environment.py b/share/environment.py new file mode 100644 index 00000000..f8bc0cf4 --- /dev/null +++ b/share/environment.py @@ -0,0 +1,17 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; +# you may not use this file except in compliance with the Elastic License 2.0. + +import os +import platform + + +def is_aws() -> bool: + return os.getenv("AWS_EXECUTION_ENV") is not None + + +def get_environment() -> str: + if is_aws(): + return os.environ["AWS_EXECUTION_ENV"] + else: + return f"Python/{platform.python_version()} {platform.system()}/{platform.machine()}" diff --git a/share/utils.py b/share/utils.py index 4454cd2e..53c48720 100644 --- a/share/utils.py +++ b/share/utils.py @@ -2,7 +2,13 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; # you may not use this file except in compliance with the Elastic License 2.0. import hashlib +import sys def get_hex_prefix(src: str) -> str: return hashlib.sha3_384(src.encode("utf-8")).hexdigest() + + +def create_user_agent(esf_version: str, environment: str = sys.version) -> str: + """Creates the 'User-Agent' header given ESF version and running environment""" + return f"ElasticServerlessForwarder/{esf_version} ({environment})" diff --git a/share/version.py b/share/version.py new file mode 100644 index 00000000..372035dc --- /dev/null +++ b/share/version.py @@ -0,0 +1,5 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; +# you may not use this file except in compliance with the Elastic License 2.0. + +version = "1.11.0" diff --git a/shippers/es.py b/shippers/es.py index e99db5fb..7b7997d6 100644 --- a/shippers/es.py +++ b/shippers/es.py @@ -10,7 +10,10 @@ from elasticsearch.helpers import bulk as es_bulk from elasticsearch.serializer import Serializer +import share.utils from share import json_dumper, json_parser, normalise_event, shared_logger +from share.environment import get_environment +from share.version import version from .shipper import EventIdGeneratorCallable, ReplayHandlerCallable @@ -119,6 +122,9 @@ def _elasticsearch_client(**es_client_kwargs: Any) -> Elasticsearch: es_client_kwargs["max_retries"] = 4 es_client_kwargs["http_compress"] = True es_client_kwargs["retry_on_timeout"] = True + es_client_kwargs["headers"] = { + "User-Agent": share.utils.create_user_agent(esf_version=version, environment=get_environment()) + } return Elasticsearch(**es_client_kwargs) diff --git a/tests/handlers/aws/test_integrations.py b/tests/handlers/aws/test_integrations.py index 8f714fbb..eb185091 100644 --- a/tests/handlers/aws/test_integrations.py +++ b/tests/handlers/aws/test_integrations.py @@ -74,7 +74,7 @@ def setUpClass(cls) -> None: lsc = LocalStackContainer(image="localstack/localstack:1.4.0") lsc.with_env("EAGER_SERVICE_LOADING", "1") - lsc.with_services("kinesis", "logs", "s3", "sqs", "secretsmanager") + lsc.with_services("kinesis", "logs", "s3", "sqs", "secretsmanager", "ec2") cls.localstack = lsc.start() session = boto3.Session(region_name=_AWS_REGION) diff --git a/tests/share/test_environment.py b/tests/share/test_environment.py new file mode 100644 index 00000000..99407731 --- /dev/null +++ b/tests/share/test_environment.py @@ -0,0 +1,17 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; +# you may not use this file except in compliance with the Elastic License 2.0. + +import os +from unittest import mock + +import pytest + +from share.environment import get_environment + + +@pytest.mark.unit +@mock.patch.dict(os.environ, {"AWS_EXECUTION_ENV": "AWS_Lambda_Python3.9"}) +def test_aws_environment() -> None: + environment = get_environment() + assert environment == "AWS_Lambda_Python3.9" diff --git a/tests/shippers/test_es.py b/tests/shippers/test_es.py index 88678ced..aab8c345 100644 --- a/tests/shippers/test_es.py +++ b/tests/shippers/test_es.py @@ -12,6 +12,9 @@ import pytest from elasticsearch import SerializationError +import share +from share.environment import get_environment +from share.version import version from shippers import ElasticsearchShipper, JSONSerializer _now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") @@ -48,7 +51,7 @@ class MockTransport(elasticsearch.Transport): def __init__(self, *args: Any, **kwargs: Any) -> None: - pass + self.kwargs = kwargs class MockClient(elasticsearch.Elasticsearch): @@ -61,13 +64,20 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: _failures = [] -def mock_bulk(client: Any, actions: list[dict[str, Any]], **kwargs: Any) -> tuple[int, list[dict[str, Any]]]: +def mock_bulk( + client: elasticsearch.Elasticsearch, actions: list[dict[str, Any]], **kwargs: Any +) -> tuple[int, list[dict[str, Any]]]: global _documents _documents = [actions] + assert client.transport.kwargs["headers"] == { + "User-Agent": share.utils.create_user_agent(esf_version=version, environment=get_environment()) + } return len(actions), [] -def mock_bulk_failure(client: Any, actions: list[dict[str, Any]], **kwargs: Any) -> tuple[int, list[dict[str, Any]]]: +def mock_bulk_failure( + client: elasticsearch.Elasticsearch, actions: list[dict[str, Any]], **kwargs: Any +) -> tuple[int, list[dict[str, Any]]]: global _failures _failures = list(map(lambda action: {"create": {"_id": action["_id"], "error": "an error"}}, actions)) return len(actions), _failures diff --git a/tests/testcontainers/es.py b/tests/testcontainers/es.py index 3abaf83b..86443440 100644 --- a/tests/testcontainers/es.py +++ b/tests/testcontainers/es.py @@ -31,7 +31,7 @@ class ElasticsearchContainer(DockerContainer): # type: ignore """ _DEFAULT_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch" - _DEFAULT_VERSION = "7.16.3" + _DEFAULT_VERSION = "7.17.9" _DEFAULT_PORT = 9200 _DEFAULT_USERNAME = DEFAULT_USERNAME _DEFAULT_PASSWORD = DEFAULT_PASSWORD diff --git a/tests/testcontainers/logstash.py b/tests/testcontainers/logstash.py index 4e143e91..dba41a00 100644 --- a/tests/testcontainers/logstash.py +++ b/tests/testcontainers/logstash.py @@ -38,7 +38,7 @@ class LogstashContainer(DockerContainer): # type: ignore """ _DEFAULT_IMAGE = "docker.elastic.co/logstash/logstash" - _DEFAULT_VERSION = "7.16.0" + _DEFAULT_VERSION = "7.17.0" _DEFAULT_PORT = 5044 _DEFAULT_API_PORT = 9600 _DEFAULT_USERNAME = "USERNAME"