From 74495f8d031a84c5810e63bf5cb8f397cb9d0ebf Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Thu, 23 Nov 2023 15:33:33 +0000
Subject: [PATCH 01/11] Add user agent in ES client kwargs

---
 share/utils.py   | 8 ++++++++
 share/version.py | 5 +++++
 shippers/es.py   | 2 ++
 3 files changed, 15 insertions(+)
 create mode 100644 share/version.py

diff --git a/share/utils.py b/share/utils.py
index 4454cd2e..4376900d 100644
--- a/share/utils.py
+++ b/share/utils.py
@@ -2,7 +2,15 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0;
 # you may not use this file except in compliance with the Elastic License 2.0.
 import hashlib
+import os
+
+from share.version import version
 
 
 def get_hex_prefix(src: str) -> str:
     return hashlib.sha3_384(src.encode("utf-8")).hexdigest()
+
+
+def create_user_agent() -> str:
+    """Creates the 'User-Agent' header given the library name and version"""
+    return f"ElasticServerlessForwarder/{version} ({os.getenv('AWS_EXECUTION_ENV')}; OTHER)"
diff --git a/share/version.py b/share/version.py
new file mode 100644
index 00000000..80dbc897
--- /dev/null
+++ b/share/version.py
@@ -0,0 +1,5 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0;
+# you may not use this file except in compliance with the Elastic License 2.0.
+
+version = "1.9.0"
diff --git a/shippers/es.py b/shippers/es.py
index e99db5fb..c848e81a 100644
--- a/shippers/es.py
+++ b/shippers/es.py
@@ -10,6 +10,7 @@
 from elasticsearch.helpers import bulk as es_bulk
 from elasticsearch.serializer import Serializer
 
+import share.utils
 from share import json_dumper, json_parser, normalise_event, shared_logger
 
 from .shipper import EventIdGeneratorCallable, ReplayHandlerCallable
@@ -119,6 +120,7 @@ def _elasticsearch_client(**es_client_kwargs: Any) -> Elasticsearch:
         es_client_kwargs["max_retries"] = 4
         es_client_kwargs["http_compress"] = True
         es_client_kwargs["retry_on_timeout"] = True
+        es_client_kwargs["headers"] = {"user-agent": share.utils.create_user_agent()}
 
         return Elasticsearch(**es_client_kwargs)
 

From e0d4720139e62d3ee5adda94a76ec90a205f0f84 Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Thu, 23 Nov 2023 15:40:28 +0000
Subject: [PATCH 02/11] Add comment

---
 share/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/share/utils.py b/share/utils.py
index 4376900d..bffb8ecd 100644
--- a/share/utils.py
+++ b/share/utils.py
@@ -13,4 +13,5 @@ def get_hex_prefix(src: str) -> str:
 
 def create_user_agent() -> str:
     """Creates the 'User-Agent' header given the library name and version"""
-    return f"ElasticServerlessForwarder/{version} ({os.getenv('AWS_EXECUTION_ENV')}; OTHER)"
+    # AWS_EXECUTION_ENV – The runtime identifier, prefixed by AWS_Lambda_ (for example, AWS_Lambda_Python3.9).
+    return f"ElasticServerlessForwarder/{version} ({os.getenv('AWS_EXECUTION_ENV')})"

From 801af4d904dcb84c6d684ffb4894b994f685f006 Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Mon, 27 Nov 2023 10:34:06 +0000
Subject: [PATCH 03/11] Address comments and move version and environment as
 parameters of create_user_agent

---
 share/environment.py | 13 +++++++++++++
 share/utils.py       | 11 ++++-------
 shippers/es.py       |  7 ++++++-
 3 files changed, 23 insertions(+), 8 deletions(-)
 create mode 100644 share/environment.py

diff --git a/share/environment.py b/share/environment.py
new file mode 100644
index 00000000..d0939a24
--- /dev/null
+++ b/share/environment.py
@@ -0,0 +1,13 @@
+import os
+import sys
+
+
+def is_aws() -> bool:
+    return os.getenv('AWS_EXECUTION_ENV') is not None
+
+
+def environment() -> str:
+    if is_aws():
+        return os.getenv('AWS_EXECUTION_ENV')
+    else:
+        return sys.version
diff --git a/share/utils.py b/share/utils.py
index bffb8ecd..53c48720 100644
--- a/share/utils.py
+++ b/share/utils.py
@@ -2,16 +2,13 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0;
 # you may not use this file except in compliance with the Elastic License 2.0.
 import hashlib
-import os
-
-from share.version import version
+import sys
 
 
 def get_hex_prefix(src: str) -> str:
     return hashlib.sha3_384(src.encode("utf-8")).hexdigest()
 
 
-def create_user_agent() -> str:
-    """Creates the 'User-Agent' header given the library name and version"""
-    # AWS_EXECUTION_ENV – The runtime identifier, prefixed by AWS_Lambda_ (for example, AWS_Lambda_Python3.9).
-    return f"ElasticServerlessForwarder/{version} ({os.getenv('AWS_EXECUTION_ENV')})"
+def create_user_agent(esf_version: str, environment: str = sys.version) -> str:
+    """Creates the 'User-Agent' header given ESF version and running environment"""
+    return f"ElasticServerlessForwarder/{esf_version} ({environment})"
diff --git a/shippers/es.py b/shippers/es.py
index c848e81a..f8512c70 100644
--- a/shippers/es.py
+++ b/shippers/es.py
@@ -12,6 +12,8 @@
 
 import share.utils
 from share import json_dumper, json_parser, normalise_event, shared_logger
+from share.environment import environment
+from share.version import version
 
 from .shipper import EventIdGeneratorCallable, ReplayHandlerCallable
 
@@ -120,7 +122,10 @@ def _elasticsearch_client(**es_client_kwargs: Any) -> Elasticsearch:
         es_client_kwargs["max_retries"] = 4
         es_client_kwargs["http_compress"] = True
         es_client_kwargs["retry_on_timeout"] = True
-        es_client_kwargs["headers"] = {"user-agent": share.utils.create_user_agent()}
+        es_client_kwargs["headers"] = {"User-Agent": share.utils.create_user_agent(
+            esf_version=version,
+            environment=environment())
+        }
 
         return Elasticsearch(**es_client_kwargs)
 

From dabe2db20cb3a2f7ee5f47c14265e78cff1ea7ca Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Mon, 27 Nov 2023 10:40:25 +0000
Subject: [PATCH 04/11] Lint fixes

---
 share/environment.py | 8 ++++++--
 shippers/es.py       | 5 ++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/share/environment.py b/share/environment.py
index d0939a24..c290b14c 100644
--- a/share/environment.py
+++ b/share/environment.py
@@ -1,13 +1,17 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0;
+# you may not use this file except in compliance with the Elastic License 2.0.
+
 import os
 import sys
 
 
 def is_aws() -> bool:
-    return os.getenv('AWS_EXECUTION_ENV') is not None
+    return os.getenv("AWS_EXECUTION_ENV") is not None
 
 
 def environment() -> str:
     if is_aws():
-        return os.getenv('AWS_EXECUTION_ENV')
+        return os.environ["AWS_EXECUTION_ENV"]
     else:
         return sys.version
diff --git a/shippers/es.py b/shippers/es.py
index f8512c70..89e823f2 100644
--- a/shippers/es.py
+++ b/shippers/es.py
@@ -122,9 +122,8 @@ def _elasticsearch_client(**es_client_kwargs: Any) -> Elasticsearch:
         es_client_kwargs["max_retries"] = 4
         es_client_kwargs["http_compress"] = True
         es_client_kwargs["retry_on_timeout"] = True
-        es_client_kwargs["headers"] = {"User-Agent": share.utils.create_user_agent(
-            esf_version=version,
-            environment=environment())
+        es_client_kwargs["headers"] = {
+            "User-Agent": share.utils.create_user_agent(esf_version=version, environment=environment())
         }
 
         return Elasticsearch(**es_client_kwargs)

From 40d447aa2c6c8f626c3868301f78eec813768002 Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Mon, 27 Nov 2023 12:11:29 +0000
Subject: [PATCH 05/11] Update requirements

---
 requirements-tests.txt | 2 +-
 requirements.txt       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements-tests.txt b/requirements-tests.txt
index fff996f1..49170d6c 100644
--- a/requirements-tests.txt
+++ b/requirements-tests.txt
@@ -8,6 +8,6 @@ orjson==3.9.10
 pysimdjson==5.0.2
 python-rapidjson==1.13
 cysimdjson==23.8
-responses==0.23.3
+responses==0.24.1
 testcontainers==3.7.1
 pyOpenSSL==23.3.0
diff --git a/requirements.txt b/requirements.txt
index f75f9839..51bffaf7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,9 @@
 elastic-apm==6.19.0
-boto3==1.28.80
+boto3==1.29.7
 ecs_logging==2.1.0
 elasticsearch==7.16.3
 PyYAML==6.0.1
 aws_lambda_typing==2.18.0
 ujson==5.8.0
 requests==2.31.0
-urllib3==1.26.15
+urllib3==1.26.18

From 1f6236943e7e8428d65523db3270c66eecb2c041 Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Mon, 27 Nov 2023 13:57:59 +0000
Subject: [PATCH 06/11] Bump minimum supported stack version to 7.17

---
 docs/en/aws-elastic-serverless-forwarder.asciidoc | 2 +-
 requirements.txt                                  | 2 +-
 tests/handlers/aws/test_integrations.py           | 2 +-
 tests/testcontainers/es.py                        | 2 +-
 tests/testcontainers/logstash.py                  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/en/aws-elastic-serverless-forwarder.asciidoc b/docs/en/aws-elastic-serverless-forwarder.asciidoc
index aa2b8ad0..d0331e85 100644
--- a/docs/en/aws-elastic-serverless-forwarder.asciidoc
+++ b/docs/en/aws-elastic-serverless-forwarder.asciidoc
@@ -11,7 +11,7 @@
 
 The Elastic Serverless Forwarder is an Amazon Web Services ({aws}) Lambda function that ships logs from your {aws} environment to Elastic.
 
-The Elastic Serverless Forwarder works with {stack} 7.16 and later.
+The Elastic Serverless Forwarder works with {stack} 7.17 and later.
 
 IMPORTANT: Using Elastic Serverless Forwarder may result in additional charges. To learn
 how to minimize additional charges, refer to <<preventing-unexpected-costs>>.
diff --git a/requirements.txt b/requirements.txt
index 51bffaf7..62cd3bb5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 elastic-apm==6.19.0
 boto3==1.29.7
 ecs_logging==2.1.0
-elasticsearch==7.16.3
+elasticsearch==7.17.9
 PyYAML==6.0.1
 aws_lambda_typing==2.18.0
 ujson==5.8.0
diff --git a/tests/handlers/aws/test_integrations.py b/tests/handlers/aws/test_integrations.py
index 8f714fbb..eb185091 100644
--- a/tests/handlers/aws/test_integrations.py
+++ b/tests/handlers/aws/test_integrations.py
@@ -74,7 +74,7 @@ def setUpClass(cls) -> None:
 
         lsc = LocalStackContainer(image="localstack/localstack:1.4.0")
         lsc.with_env("EAGER_SERVICE_LOADING", "1")
-        lsc.with_services("kinesis", "logs", "s3", "sqs", "secretsmanager")
+        lsc.with_services("kinesis", "logs", "s3", "sqs", "secretsmanager", "ec2")
         cls.localstack = lsc.start()
 
         session = boto3.Session(region_name=_AWS_REGION)
diff --git a/tests/testcontainers/es.py b/tests/testcontainers/es.py
index 3abaf83b..86443440 100644
--- a/tests/testcontainers/es.py
+++ b/tests/testcontainers/es.py
@@ -31,7 +31,7 @@ class ElasticsearchContainer(DockerContainer):  # type: ignore
     """
 
     _DEFAULT_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch"
-    _DEFAULT_VERSION = "7.16.3"
+    _DEFAULT_VERSION = "7.17.9"
     _DEFAULT_PORT = 9200
     _DEFAULT_USERNAME = DEFAULT_USERNAME
     _DEFAULT_PASSWORD = DEFAULT_PASSWORD
diff --git a/tests/testcontainers/logstash.py b/tests/testcontainers/logstash.py
index 4e143e91..dba41a00 100644
--- a/tests/testcontainers/logstash.py
+++ b/tests/testcontainers/logstash.py
@@ -38,7 +38,7 @@ class LogstashContainer(DockerContainer):  # type: ignore
     """
 
     _DEFAULT_IMAGE = "docker.elastic.co/logstash/logstash"
-    _DEFAULT_VERSION = "7.16.0"
+    _DEFAULT_VERSION = "7.17.0"
     _DEFAULT_PORT = 5044
     _DEFAULT_API_PORT = 9600
     _DEFAULT_USERNAME = "USERNAME"

From 4393dbc3c234895778d7bfd1da8131e44867bd34 Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Mon, 27 Nov 2023 15:18:41 +0000
Subject: [PATCH 07/11] Make default environment part of the user agent
 compliant with RFC

---
 share/environment.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/share/environment.py b/share/environment.py
index c290b14c..8a9b74e4 100644
--- a/share/environment.py
+++ b/share/environment.py
@@ -3,6 +3,7 @@
 # you may not use this file except in compliance with the Elastic License 2.0.
 
 import os
+import platform
 import sys
 
 
@@ -14,4 +15,4 @@ def environment() -> str:
     if is_aws():
         return os.environ["AWS_EXECUTION_ENV"]
     else:
-        return sys.version
+        return f"Python/{platform.python_version()} {platform.system()}/{platform.machine()}"

From 42ba8f6fac73df9e5343f532b91f15e2a574deb5 Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Mon, 27 Nov 2023 15:26:46 +0000
Subject: [PATCH 08/11] Remove unused import

---
 share/environment.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/share/environment.py b/share/environment.py
index 8a9b74e4..4a0a1e91 100644
--- a/share/environment.py
+++ b/share/environment.py
@@ -4,7 +4,6 @@
 
 import os
 import platform
-import sys
 
 
 def is_aws() -> bool:

From 172610f2a4a63eff2ac7866dadd1440e7b152ac3 Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Mon, 27 Nov 2023 17:01:56 +0000
Subject: [PATCH 09/11] Add unit test

---
 share/environment.py            |  2 +-
 shippers/es.py                  |  4 ++--
 tests/share/test_environment.py | 17 +++++++++++++++++
 3 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100644 tests/share/test_environment.py

diff --git a/share/environment.py b/share/environment.py
index 4a0a1e91..f8bc0cf4 100644
--- a/share/environment.py
+++ b/share/environment.py
@@ -10,7 +10,7 @@ def is_aws() -> bool:
     return os.getenv("AWS_EXECUTION_ENV") is not None
 
 
-def environment() -> str:
+def get_environment() -> str:
     if is_aws():
         return os.environ["AWS_EXECUTION_ENV"]
     else:
diff --git a/shippers/es.py b/shippers/es.py
index 89e823f2..7b7997d6 100644
--- a/shippers/es.py
+++ b/shippers/es.py
@@ -12,7 +12,7 @@
 
 import share.utils
 from share import json_dumper, json_parser, normalise_event, shared_logger
-from share.environment import environment
+from share.environment import get_environment
 from share.version import version
 
 from .shipper import EventIdGeneratorCallable, ReplayHandlerCallable
@@ -123,7 +123,7 @@ def _elasticsearch_client(**es_client_kwargs: Any) -> Elasticsearch:
         es_client_kwargs["http_compress"] = True
         es_client_kwargs["retry_on_timeout"] = True
         es_client_kwargs["headers"] = {
-            "User-Agent": share.utils.create_user_agent(esf_version=version, environment=environment())
+            "User-Agent": share.utils.create_user_agent(esf_version=version, environment=get_environment())
         }
 
         return Elasticsearch(**es_client_kwargs)
diff --git a/tests/share/test_environment.py b/tests/share/test_environment.py
new file mode 100644
index 00000000..99407731
--- /dev/null
+++ b/tests/share/test_environment.py
@@ -0,0 +1,17 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0;
+# you may not use this file except in compliance with the Elastic License 2.0.
+
+import os
+from unittest import mock
+
+import pytest
+
+from share.environment import get_environment
+
+
+@pytest.mark.unit
+@mock.patch.dict(os.environ, {"AWS_EXECUTION_ENV": "AWS_Lambda_Python3.9"})
+def test_aws_environment() -> None:
+    environment = get_environment()
+    assert environment == "AWS_Lambda_Python3.9"

From 61f4e08966e057ee3e9147eb6a00859c07ef506f Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Mon, 27 Nov 2023 18:17:31 +0000
Subject: [PATCH 10/11] Fix major version and add changelog

---
 CHANGELOG.md     | 4 ++++
 share/version.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7da1e6c4..d9a6007f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+### v1.11.0 - 2023/11/27
+##### Features
+* Add user agent with information about ESF version and host environment: [#537](https://github.com/elastic/elastic-serverless-forwarder/pull/537)
+
 ### v1.10.0 - 2023/10/27
 ##### Features
 * Move `_id` field to `@metadata._id` in logstash output: [#507](https://github.com/elastic/elastic-serverless-forwarder/pull/507)
diff --git a/share/version.py b/share/version.py
index 80dbc897..372035dc 100644
--- a/share/version.py
+++ b/share/version.py
@@ -2,4 +2,4 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0;
 # you may not use this file except in compliance with the Elastic License 2.0.
 
-version = "1.9.0"
+version = "1.11.0"

From baa8f9455f1681efbdf5e0481789eee0f79ee71f Mon Sep 17 00:00:00 2001
From: girodav <1390902+girodav@users.noreply.github.com>
Date: Tue, 28 Nov 2023 10:11:03 +0000
Subject: [PATCH 11/11] Add User Agent check in ES shipper unit tests

---
 tests/shippers/test_es.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/tests/shippers/test_es.py b/tests/shippers/test_es.py
index 88678ced..aab8c345 100644
--- a/tests/shippers/test_es.py
+++ b/tests/shippers/test_es.py
@@ -12,6 +12,9 @@
 import pytest
 from elasticsearch import SerializationError
 
+import share
+from share.environment import get_environment
+from share.version import version
 from shippers import ElasticsearchShipper, JSONSerializer
 
 _now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
@@ -48,7 +51,7 @@
 
 class MockTransport(elasticsearch.Transport):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
-        pass
+        self.kwargs = kwargs
 
 
 class MockClient(elasticsearch.Elasticsearch):
@@ -61,13 +64,20 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
 _failures = []
 
 
-def mock_bulk(client: Any, actions: list[dict[str, Any]], **kwargs: Any) -> tuple[int, list[dict[str, Any]]]:
+def mock_bulk(
+    client: elasticsearch.Elasticsearch, actions: list[dict[str, Any]], **kwargs: Any
+) -> tuple[int, list[dict[str, Any]]]:
     global _documents
     _documents = [actions]
+    assert client.transport.kwargs["headers"] == {
+        "User-Agent": share.utils.create_user_agent(esf_version=version, environment=get_environment())
+    }
     return len(actions), []
 
 
-def mock_bulk_failure(client: Any, actions: list[dict[str, Any]], **kwargs: Any) -> tuple[int, list[dict[str, Any]]]:
+def mock_bulk_failure(
+    client: elasticsearch.Elasticsearch, actions: list[dict[str, Any]], **kwargs: Any
+) -> tuple[int, list[dict[str, Any]]]:
     global _failures
     _failures = list(map(lambda action: {"create": {"_id": action["_id"], "error": "an error"}}, actions))
     return len(actions), _failures