Limit number of request headers and query string parameters.

README.md

@@ -33,7 +33,7 @@ Kong JSON request logs can be `POST`ed to the `/logs` endpoint. This is designed
 This is currently the only supported input method, but more may be added in the future.
 
 ## Transformation
-Request logs are passed through unchanged by default, but you probably want to enable at least one transformation.
+Request logs are passed through largely unchanged by default, but you probably want to enable at least one transformation.
 
 ### Timestamp Conversion `--convert-ts`
 Kong request logs include a number of UNIX timestamps (some in milliseconds rather than seconds). These are not human readable, and require explicit mappings to be used in Elasticsearch. Enabling this option will convert these timestamps to [RFC3339 date-time strings](https://www.ietf.org/rfc/rfc3339.txt) for readability and automatic Elasticsearch mapping.
@@ -84,6 +84,11 @@ e.g. `--null-path request.headers` will convert the entire `request.headers` obj
 
 If a path doesn't match any field in a given request log it will be ignored.
 
+### Object Limits `--limit-request-headers`/`--limit-request-querystring`
+Requests can contain arbitrary numbers of headers and query string parameters. This can create large numbers of fields in the destination Elasticsearch index, potentially causing performance and indexing issues.
+
+To mitigate this issue, the number of keys in the `request.headers` and `request.querystring` fields are limited to 100 by default - subsequent keys are dropped. The limits can be changed by the `--limit-request-headers` and `--limit-request-querystring` options.
+
 ## Output
 Transformed logs are indexed in Elasticsearch.
 

kong_log_bridge/__init__.py

@@ -50,6 +50,8 @@ def logs():
                             do_hash_cookie=kwargs['hash_cookie'],
                             hash_paths=kwargs['hash_path'],
                             null_paths=kwargs['null_path'],
+                            limit_request_headers=kwargs['limit_request_headers'],
+                            limit_request_querystring=kwargs['limit_request_querystring'],
                             expose_ips=kwargs['expose_ip'])
 
         es_client.index(index=es_index, body=log, request_timeout=30)

kong_log_bridge/transform.py

@@ -182,6 +182,24 @@ def hash_set_cookie(value):
         return _hash_cookie(value)
 
 
+def limit_dict(limit):
+    """Limit the number of entries in a dictionary"""
+
+    def do_limit_dict(value):
+
+        if value is None:
+            return None
+
+        dict_keys = list(value.keys())
+        if len(dict_keys) <= limit:
+            return value
+
+        limited_dict_keys = dict_keys[:limit]
+        return {k: v for k, v in value.items() if k in limited_dict_keys}
+
+    return do_limit_dict
+
+
 def transform_log(log,
                   do_convert_ts=False,
                   do_convert_qs_bools=False,
@@ -190,6 +208,8 @@ def transform_log(log,
                   do_hash_cookie=False,
                   hash_paths=None,
                   null_paths=None,
+                  limit_request_headers=None,
+                  limit_request_querystring=None,
                   expose_ips=None):
 
     if expose_ips is None:
@@ -229,4 +249,10 @@ def transform_log(log,
         for path in null_paths:
             log = update_path(log, path, None)
 
+    if limit_request_headers is not None:
+        log = update_path(log, 'request.headers', limit_dict(limit_request_headers))
+
+    if limit_request_querystring is not None:
+        log = update_path(log, 'request.querystring', limit_dict(limit_request_querystring))
+
     return log

main.py

@@ -44,6 +44,10 @@
 @click.option('--null-path', multiple=True,
               help='A path to a field to set to null. '
                    'Specify multiple paths by providing the option multiple times.')
+@click.option('--limit-request-headers', default=100,
+              help='Limit the number of request headers (default=100)')
+@click.option('--limit-request-querystring', default=100,
+              help='Limit the number of request querystring parameters (default=100)')
 @click.option('--expose-ip', multiple=True,
               help='Hash of an IP to expose i.e. include the raw IP in logs. '
                    'Specify multiple IP hashes by providing the option multiple times.')

requirements.txt

@@ -4,7 +4,7 @@
 # NOTE: This won't override a previously installed version of Bottle,
 #       so remove first with:
 #       > pip3 uninstall bottle
-git+git://github.com/braedon/bottle@improve-wsgi-error-handling-013#egg=bottle
+git+https://github.com/braedon/bottle@improve-wsgi-error-handling-013#egg=bottle
 click==7.1.2
 elasticsearch==7.12.0
 gevent==21.1.2