From 95bd528810df6b30ee378bf236fd4c82b115479a Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 9 Jul 2021 12:44:15 -0700 Subject: [PATCH 01/45] added initial telemetry work --- rollbar/__init__.py | 22 ++++++++++- rollbar/lib/telemetry.py | 82 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 rollbar/lib/telemetry.py diff --git a/rollbar/__init__.py b/rollbar/__init__.py index 5101ea35..2a789585 100644 --- a/rollbar/__init__.py +++ b/rollbar/__init__.py @@ -20,7 +20,7 @@ import requests import six -from rollbar.lib import events, filters, dict_merge, parse_qs, text, transport, urljoin, iteritems, defaultJSONEncode +from rollbar.lib import events, filters, dict_merge, parse_qs, text, transport, telemetry, urljoin, iteritems, defaultJSONEncode __version__ = '0.16.1' @@ -381,6 +381,18 @@ def init(access_token, environment='production', scrub_fields=None, url_fields=N if SETTINGS.get('allow_logging_basic_config'): logging.basicConfig() + if SETTINGS.get('log_telemetry'): + formatter = SETTINGS.get('log_telemetry_formatter') + telemetry.set_log_telemetry(formatter) + if SETTINGS.get('network_telemetry'): + enable_req_headers = SETTINGS.get('enable_req_headers') + enable_response_headers = SETTINGS.get('enable_response_headers') + requests.get = telemetry.request(requests.get, enable_req_headers, enable_response_headers ) + requests.post = telemetry.request(requests.post, enable_req_headers, enable_response_headers) + requests.put = telemetry.request(requests.put, enable_req_headers, enable_response_headers) + requests.patch = telemetry.request(requests.patch, enable_req_headers, enable_response_headers) + requests.delete = telemetry.request(requests.delete, enable_req_headers, enable_response_headers) + if SETTINGS.get('handler') == 'agent': agent_log = _create_agent_log() @@ -777,6 +789,7 @@ def _report_exc_info(exc_info, request, extra_data, payload_data, level=None): _add_request_data(data, request) _add_person_data(data, request) _add_lambda_context_data(data) + _add_telemetry(data) data['server'] = _build_server_data() if payload_data: @@ -857,6 +870,7 @@ def _report_message(message, level, request, extra_data, payload_data): _add_request_data(data, request) _add_person_data(data, request) _add_lambda_context_data(data) + _add_telemetry(data) data['server'] = _build_server_data() if payload_data: @@ -1119,6 +1133,12 @@ def _add_request_data(data, request): data['request'] = request_data +def _add_telemetry(data): + telemetry_data = telemetry.TELEMETRY_QUEUE.get_items() + if telemetry_data: + data['body']['telemetry'] = telemetry_data + + def _check_add_locals(frame, frame_num, total_frames): """ Returns True if we should record local variables for the given frame. diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py new file mode 100644 index 00000000..179d203e --- /dev/null +++ b/rollbar/lib/telemetry.py @@ -0,0 +1,82 @@ +import logging +import threading +import time + +import rollbar + + +class Queue: + def __init__(self, size): + self.size = size + self.items = [] + self.lock = threading.Lock() + + def put(self, item): + with self.lock: + if len(self.items) >= self.size: + self.items = self.items[1:] + self.items.append(item) + + def get_items(self): + with self.lock: + return self.items + + +TELEMETRY_QUEUE_SIZE = 50 +TELEMETRY_QUEUE = Queue(TELEMETRY_QUEUE_SIZE) + + +class TelemetryLogHandler(logging.Handler): + def __init__(self, formatter=None): + super(TelemetryLogHandler, self).__init__() + self.formatter = formatter + + def emit(self, record): + self.setFormatter(self.formatter) + msg = {'message': self.format(record)} + data = {'body': msg, 'source': 'client', 'timestamp_ms': int(time.time()), 'type': 'log', + 'level': record.levelname} + + TELEMETRY_QUEUE.put(data) + + +def set_log_telemetry(log_formatter=None): + logging.getLogger().addHandler(TelemetryLogHandler(log_formatter)) + + +def request(request_function, enable_req_headers, enable_response_headers): + def telemetry(*args, **kwargs): + + def clean_headers(headers): + for h in headers: + if h in rollbar.SETTINGS['scrub_fields']: + del headers[h] + return headers + + data = {'level': 'info'} + data_body = {'status_code': None} + + response = request_function(*args, **kwargs) + + if response: + data_body['status_code'] = response.status_code + if response.status_code >= 500: + data['level'] = 'critical' + elif response.status_code >= 400: + data['level'] = 'error' + if enable_response_headers: + data_body['response'] = {'headers': clean_headers(response.headers)} + if enable_req_headers: + data_body['request_headers'] = clean_headers(kwargs.get('headers')) + data_body['url'] = args[0] + data_body['method'] = request_function.__name__ + data_body['subtype'] = 'http' + data['body'] = data_body + + data['source'] = 'client' + data['timestamp_ms'] = int(time.time()) + data['type'] = 'network' + TELEMETRY_QUEUE.put(data) + + return response + return telemetry From 5430b78304ba524ea747cad8a2ecdf142db5ac07 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Mon, 12 Jul 2021 16:57:58 -0700 Subject: [PATCH 02/45] added telemetry queue size --- rollbar/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rollbar/__init__.py b/rollbar/__init__.py index 2a789585..e45e5a86 100644 --- a/rollbar/__init__.py +++ b/rollbar/__init__.py @@ -381,6 +381,10 @@ def init(access_token, environment='production', scrub_fields=None, url_fields=N if SETTINGS.get('allow_logging_basic_config'): logging.basicConfig() + queue_size = SETTINGS.get('set_custom_queue_size') + if queue_size: + telemetry.TELEMETRY_QUEUE = telemetry.Queue(queue_size) + if SETTINGS.get('log_telemetry'): formatter = SETTINGS.get('log_telemetry_formatter') telemetry.set_log_telemetry(formatter) From d7bf70d6bbba6cf51b9a81975179e85d5adffb54 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Thu, 22 Jul 2021 22:39:55 -0700 Subject: [PATCH 03/45] added unit tests --- rollbar/lib/telemetry.py | 24 +++++++++++++++----- rollbar/test/test_telemetry.py | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 rollbar/test/test_telemetry.py diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index 179d203e..d1557508 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -21,6 +21,10 @@ def get_items(self): with self.lock: return self.items + def clear_items(self): + with self.lock: + self.items = [] + TELEMETRY_QUEUE_SIZE = 50 TELEMETRY_QUEUE = Queue(TELEMETRY_QUEUE_SIZE) @@ -34,7 +38,7 @@ def __init__(self, formatter=None): def emit(self, record): self.setFormatter(self.formatter) msg = {'message': self.format(record)} - data = {'body': msg, 'source': 'client', 'timestamp_ms': int(time.time()), 'type': 'log', + data = {'body': msg, 'source': 'client', 'timestamp_ms': get_current_timestamp(), 'type': 'log', 'level': record.levelname} TELEMETRY_QUEUE.put(data) @@ -44,21 +48,28 @@ def set_log_telemetry(log_formatter=None): logging.getLogger().addHandler(TelemetryLogHandler(log_formatter)) +def get_current_timestamp(): + return int(time.time()) + + def request(request_function, enable_req_headers, enable_response_headers): def telemetry(*args, **kwargs): def clean_headers(headers): + if not headers: + return [] for h in headers: if h in rollbar.SETTINGS['scrub_fields']: del headers[h] return headers - data = {'level': 'info'} data_body = {'status_code': None} + try: + response = request_function(*args, **kwargs) + except: # noqa: E722 + response = None - response = request_function(*args, **kwargs) - - if response: + if response is not None: data_body['status_code'] = response.status_code if response.status_code >= 500: data['level'] = 'critical' @@ -74,9 +85,10 @@ def clean_headers(headers): data['body'] = data_body data['source'] = 'client' - data['timestamp_ms'] = int(time.time()) + data['timestamp_ms'] = get_current_timestamp() data['type'] = 'network' TELEMETRY_QUEUE.put(data) return response + return telemetry diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py new file mode 100644 index 00000000..0fe19a20 --- /dev/null +++ b/rollbar/test/test_telemetry.py @@ -0,0 +1,41 @@ +from rollbar.lib import telemetry +from rollbar.test import BaseTest +import logging +import mock +import requests + + +class RollbarTelemetryTest(BaseTest): + + def setUp(self): + formatter = logging.Formatter('%(name)s :: %(levelname)s :: %(message)s') + telemetry.set_log_telemetry(formatter) + requests.get = telemetry.request(requests.get, False, False) + + @mock.patch('rollbar.telemetry.get_current_timestamp') + def test_telemetry_log(self, timestamp): + timestamp.return_value = 1000000 + logging.warning("test loggin") + items = telemetry.TELEMETRY_QUEUE.get_items() + self.assertEqual(1, len(items)) + + result = {'body': {'message': 'root :: WARNING :: test loggin'}, + 'source': 'client', 'level': 'WARNING', 'type': 'log', + 'timestamp_ms': 1000000} + + self.assertEqual(result, items[0]) + telemetry.TELEMETRY_QUEUE.clear_items() + + @mock.patch('rollbar.telemetry.get_current_timestamp') + def test_telemetry_request(self, timestamp): + timestamp.return_value = 1000000 + + requests.get("http://example.com") + items = telemetry.TELEMETRY_QUEUE.get_items() + self.assertEqual(1, len(items)) + + result = {'body': {'url': 'http://example.com', 'status_code': 200, 'method': 'get', + 'subtype': 'http'}, 'source': 'client', 'timestamp_ms': 1000000,\ + 'type': 'network', 'level': 'info'} + self.assertEqual(result, items[0]) + telemetry.TELEMETRY_QUEUE.clear_items() \ No newline at end of file From 9cc6c8bfdd8eaad61bb1066b662d30f4618a09a5 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Thu, 22 Jul 2021 22:41:33 -0700 Subject: [PATCH 04/45] fixing problem --- rollbar/test/test_telemetry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py index 0fe19a20..250cc6b4 100644 --- a/rollbar/test/test_telemetry.py +++ b/rollbar/test/test_telemetry.py @@ -35,7 +35,7 @@ def test_telemetry_request(self, timestamp): self.assertEqual(1, len(items)) result = {'body': {'url': 'http://example.com', 'status_code': 200, 'method': 'get', - 'subtype': 'http'}, 'source': 'client', 'timestamp_ms': 1000000,\ + 'subtype': 'http'}, 'source': 'client', 'timestamp_ms': 1000000, 'type': 'network', 'level': 'info'} self.assertEqual(result, items[0]) - telemetry.TELEMETRY_QUEUE.clear_items() \ No newline at end of file + telemetry.TELEMETRY_QUEUE.clear_items() From 00b76b717d9dcb1496f07fb8439cb2607f53cca7 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Thu, 22 Jul 2021 23:15:24 -0700 Subject: [PATCH 05/45] corrected tests --- rollbar/test/test_telemetry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py index 250cc6b4..e09d67ed 100644 --- a/rollbar/test/test_telemetry.py +++ b/rollbar/test/test_telemetry.py @@ -7,7 +7,8 @@ class RollbarTelemetryTest(BaseTest): - def setUp(self): + @classmethod + def setUpClass(self): formatter = logging.Formatter('%(name)s :: %(levelname)s :: %(message)s') telemetry.set_log_telemetry(formatter) requests.get = telemetry.request(requests.get, False, False) From 5f85940909fd44b977e9e19bc0df7e0919427a58 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 15:06:40 -0700 Subject: [PATCH 06/45] added dependency --- .github/workflows/ci.yml | 2 +- rollbar/lib/telemetry.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3588da1..52012c55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8] framework: - - FLASK_VERSION=0.10.1 Werkzeug\>=0.7,\<1.0 + - FLASK_VERSION=0.10.1 Werkzeug\>=0.7,\<1.0 charset_normalizer==2.0.3 - FLASK_VERSION=0.11.1 Werkzeug\>=0.7,\<1.0 - FLASK_VERSION=0.12.4 Werkzeug\>=0.7,\<1.0 - FLASK_VERSION=1.0.2 diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index d1557508..1af0a53c 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -60,7 +60,7 @@ def clean_headers(headers): return [] for h in headers: if h in rollbar.SETTINGS['scrub_fields']: - del headers[h] + headers[h] = '[FILTERED]' return headers data = {'level': 'info'} data_body = {'status_code': None} From 5c831e87cbdd003cdad5ab04518edc50e21f5740 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 15:23:04 -0700 Subject: [PATCH 07/45] added chardet==3.0.4 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52012c55..099bec64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8] framework: - - FLASK_VERSION=0.10.1 Werkzeug\>=0.7,\<1.0 charset_normalizer==2.0.3 + - FLASK_VERSION=0.10.1 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 - FLASK_VERSION=0.11.1 Werkzeug\>=0.7,\<1.0 - FLASK_VERSION=0.12.4 Werkzeug\>=0.7,\<1.0 - FLASK_VERSION=1.0.2 From b48929166a04b4a33d0cd2da714677000fb65d60 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 15:30:24 -0700 Subject: [PATCH 08/45] added idna==2.7 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 099bec64..edfea904 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8] framework: - - FLASK_VERSION=0.10.1 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 + - FLASK_VERSION=0.10.1 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 idna==2.7 - FLASK_VERSION=0.11.1 Werkzeug\>=0.7,\<1.0 - FLASK_VERSION=0.12.4 Werkzeug\>=0.7,\<1.0 - FLASK_VERSION=1.0.2 From 5b3979f0f40b0af6412b9ab6482fdf648c3bb572 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 15:33:18 -0700 Subject: [PATCH 09/45] added mock==3.0.5 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edfea904..51475aac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8] framework: - - FLASK_VERSION=0.10.1 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 idna==2.7 + - FLASK_VERSION=0.10.1 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 idna==2.7 mock==3.0.5 - FLASK_VERSION=0.11.1 Werkzeug\>=0.7,\<1.0 - FLASK_VERSION=0.12.4 Werkzeug\>=0.7,\<1.0 - FLASK_VERSION=1.0.2 From 9f5f31725412c3d77d4c270b9fc9a71bac2455ff Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 15:37:29 -0700 Subject: [PATCH 10/45] installing more dependencies --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51475aac..d07f43ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,9 +15,9 @@ jobs: python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8] framework: - FLASK_VERSION=0.10.1 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 idna==2.7 mock==3.0.5 - - FLASK_VERSION=0.11.1 Werkzeug\>=0.7,\<1.0 - - FLASK_VERSION=0.12.4 Werkzeug\>=0.7,\<1.0 - - FLASK_VERSION=1.0.2 + - FLASK_VERSION=0.11.1 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 idna==2.7 mock==3.0.5 + - FLASK_VERSION=0.12.4 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 idna==2.7 mock==3.0.5 + - FLASK_VERSION=1.0.2 chardet==3.0.4 idna==2.7 mock==3.0.5 - TWISTED_VERSION=15.5.0 treq==15.1.0 zope.interface==4.1.3 - TWISTED_VERSION=16.1.0 treq==16.12.0 zope.interface==4.1.3 - TWISTED_VERSION=16.2.0 treq==16.12.0 zope.interface==4.1.3 From c4267fcc249259d73f997ed7e6432cdc551ea4e3 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 15:45:39 -0700 Subject: [PATCH 11/45] added dependencies --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d07f43ad..a3ff3597 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,10 +26,10 @@ jobs: - TWISTED_VERSION=16.5.0 treq==16.12.0 zope.interface==4.5.0 - TWISTED_VERSION=16.6.0 treq==16.12.0 zope.interface==4.5.0 - TWISTED_VERSION=17.1.0 treq==16.12.0 zope.interface==4.5.0 - - DJANGO_VERSION=1.11.20 - - DJANGO_VERSION=2.0.13 - - DJANGO_VERSION=2.1.7 - - DJANGO_VERSION=2.1.15 + - DJANGO_VERSION=1.11.20 chardet==3.0.4 idna==2.7 mock==3.0.5 + - DJANGO_VERSION=2.0.13 chardet==3.0.4 idna==2.7 mock==3.0.5 + - DJANGO_VERSION=2.1.7 chardet==3.0.4 idna==2.7 mock==3.0.5 + - DJANGO_VERSION=2.1.15 chardet==3.0.4 idna==2.7 mock==3.0.5 - PYRAMID_VERSION=1.9.2 - PYRAMID_VERSION=1.10.4 - STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 From 99988fca67efba2cec72b1aec8fdec60d56f2095 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 15:51:38 -0700 Subject: [PATCH 12/45] fixed dependencies --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3ff3597..4981f827 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,9 +27,9 @@ jobs: - TWISTED_VERSION=16.6.0 treq==16.12.0 zope.interface==4.5.0 - TWISTED_VERSION=17.1.0 treq==16.12.0 zope.interface==4.5.0 - DJANGO_VERSION=1.11.20 chardet==3.0.4 idna==2.7 mock==3.0.5 - - DJANGO_VERSION=2.0.13 chardet==3.0.4 idna==2.7 mock==3.0.5 - - DJANGO_VERSION=2.1.7 chardet==3.0.4 idna==2.7 mock==3.0.5 - - DJANGO_VERSION=2.1.15 chardet==3.0.4 idna==2.7 mock==3.0.5 + - DJANGO_VERSION=2.0.13 + - DJANGO_VERSION=2.1.7 + - DJANGO_VERSION=2.1.15 - PYRAMID_VERSION=1.9.2 - PYRAMID_VERSION=1.10.4 - STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 From 8c126a736cbb97273bcab5d71b572319ec5be158 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 16:00:58 -0700 Subject: [PATCH 13/45] added chardet==3.0.4 idna==2.7 mock==3.0.5 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4981f827..89a45cd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - TWISTED_VERSION=16.6.0 treq==16.12.0 zope.interface==4.5.0 - TWISTED_VERSION=17.1.0 treq==16.12.0 zope.interface==4.5.0 - DJANGO_VERSION=1.11.20 chardet==3.0.4 idna==2.7 mock==3.0.5 - - DJANGO_VERSION=2.0.13 + - DJANGO_VERSION=2.0.13 chardet==3.0.4 idna==2.7 mock==3.0.5 - DJANGO_VERSION=2.1.7 - DJANGO_VERSION=2.1.15 - PYRAMID_VERSION=1.9.2 From 5e459f7cdfba205a45abf978a5c71876ba9caa04 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 16:07:36 -0700 Subject: [PATCH 14/45] added dependency to pyramid --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89a45cd9..eb6f0865 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,11 +27,11 @@ jobs: - TWISTED_VERSION=16.6.0 treq==16.12.0 zope.interface==4.5.0 - TWISTED_VERSION=17.1.0 treq==16.12.0 zope.interface==4.5.0 - DJANGO_VERSION=1.11.20 chardet==3.0.4 idna==2.7 mock==3.0.5 - - DJANGO_VERSION=2.0.13 chardet==3.0.4 idna==2.7 mock==3.0.5 + - DJANGO_VERSION=2.0.13 - DJANGO_VERSION=2.1.7 - DJANGO_VERSION=2.1.15 - - PYRAMID_VERSION=1.9.2 - - PYRAMID_VERSION=1.10.4 + - PYRAMID_VERSION=1.9.2 chardet==3.0.4 idna==2.7 mock==3.0.5 + - PYRAMID_VERSION=1.10.4 chardet==3.0.4 idna==2.7 mock==3.0.5 - STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 - STARLETTE_VERSION=0.12.13 httpx==0.18.1 python-multipart==0.0.5 - STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 From 9549e0de9504a471d4807ce9a6ff218ad957ca78 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 16:13:49 -0700 Subject: [PATCH 15/45] fixing versions --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb6f0865..ada9a7a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,8 @@ jobs: framework: DJANGO_VERSION=2.1.7 - python-version: 2.7 framework: DJANGO_VERSION=2.1.15 + - python-version: 3.4 + framework: DJANGO_VERSION=2.0.13 - python-version: 3.4 framework: DJANGO_VERSION=2.1.7 - python-version: 3.4 @@ -154,6 +156,8 @@ jobs: framework: DJANGO_VERSION=1.9.13 - python-version: 3.4 framework: DJANGO_VERSION=1.10.8 + - python-version: 3.4 + framework: DJANGO_VERSION=2.0.13 chardet==3.0.4 idna==2.7 mock==3.0.5 - python-version: 3.5 framework: DJANGO_VERSION=1.8.19 - python-version: 3.5 From 3336002d83d1bdda0702b5bfc8f255fde919c8ad Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 16:26:27 -0700 Subject: [PATCH 16/45] fixing dependencies --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ada9a7a8..6d305312 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,10 @@ jobs: framework: DJANGO_VERSION=2.1.15 - python-version: 3.5 framework: DJANGO_VERSION=2.1.15 + - python-version: 3.5 + framework: DJANGO_VERSION=2.0.13 + - python-version: 3.5 + framework: DJANGO_VERSION=2.1.7 - python-version: 3.6 framework: DJANGO_VERSION=2.1.15 - python-version: 3.7 @@ -164,6 +168,10 @@ jobs: framework: DJANGO_VERSION=1.9.13 - python-version: 3.5 framework: DJANGO_VERSION=1.10.8 + - python-version: 3.5 + framework: DJANGO_VERSION=2.0.13 mock==3.0.5 + - python-version: 3.5 + framework: DJANGO_VERSION=2.1.7 mock==3.0.5 - python-version: 3.7 framework: TWISTED_VERSION=18.9.0 treq==20.4.1 zope.interface==4.5.0 - python-version: 3.7 From f5d91390ebeb5e28ec51a0a0bc17ac3f618871d5 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 16:36:28 -0700 Subject: [PATCH 17/45] fixing dependencies --- .github/workflows/ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d305312..da2611d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,33 +141,33 @@ jobs: - python-version: 2.7 framework: FLASK_VERSION=0.9 - python-version: 3.3 - framework: FLASK_VERSION=0.10.1 + framework: FLASK_VERSION=0.10.1 mock==3.0.5 - python-version: 3.3 - framework: FLASK_VERSION=0.11.1 + framework: FLASK_VERSION=0.11.1 mock==3.0.5 - python-version: 3.3 - framework: FLASK_VERSION=0.12.4 + framework: FLASK_VERSION=0.12.4 mock==3.0.5 - python-version: 3.3 - framework: FLASK_VERSION=1.0.2 + framework: FLASK_VERSION=1.0.2 mock==3.0.5 - python-version: 3.3 - framework: DJANGO_VERSION=1.6.11 + framework: DJANGO_VERSION=1.6.11 mock==3.0.5 - python-version: 3.3 - framework: DJANGO_VERSION=1.8.19 + framework: DJANGO_VERSION=1.8.19 mock==3.0.5 - python-version: 3.4 - framework: DJANGO_VERSION=1.7.11 + framework: DJANGO_VERSION=1.7.11 chardet==3.0.4 idna==2.7 mock==3.0.5 - python-version: 3.4 - framework: DJANGO_VERSION=1.8.19 + framework: DJANGO_VERSION=1.8.19 chardet==3.0.4 idna==2.7 mock==3.0.5 - python-version: 3.4 - framework: DJANGO_VERSION=1.9.13 + framework: DJANGO_VERSION=1.9.13 chardet==3.0.4 idna==2.7 mock==3.0.5 - python-version: 3.4 - framework: DJANGO_VERSION=1.10.8 + framework: DJANGO_VERSION=1.10.8 chardet==3.0.4 idna==2.7 mock==3.0.5 - python-version: 3.4 framework: DJANGO_VERSION=2.0.13 chardet==3.0.4 idna==2.7 mock==3.0.5 - python-version: 3.5 - framework: DJANGO_VERSION=1.8.19 + framework: DJANGO_VERSION=1.8.19 mock==3.0.5 - python-version: 3.5 - framework: DJANGO_VERSION=1.9.13 + framework: DJANGO_VERSION=1.9.13 mock==3.0.5 - python-version: 3.5 - framework: DJANGO_VERSION=1.10.8 + framework: DJANGO_VERSION=1.10.8 mock==3.0.5 - python-version: 3.5 framework: DJANGO_VERSION=2.0.13 mock==3.0.5 - python-version: 3.5 From 7cebc6447ed0b647afffdc63fee20043c46fd105 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 16:42:17 -0700 Subject: [PATCH 18/45] fixing mock version --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da2611d5..4ba4154b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,17 +141,17 @@ jobs: - python-version: 2.7 framework: FLASK_VERSION=0.9 - python-version: 3.3 - framework: FLASK_VERSION=0.10.1 mock==3.0.5 + framework: FLASK_VERSION=0.10.1 mock==4.0.3 - python-version: 3.3 - framework: FLASK_VERSION=0.11.1 mock==3.0.5 + framework: FLASK_VERSION=0.11.1 mock==4.0.3 - python-version: 3.3 - framework: FLASK_VERSION=0.12.4 mock==3.0.5 + framework: FLASK_VERSION=0.12.4 mock==4.0.3 - python-version: 3.3 - framework: FLASK_VERSION=1.0.2 mock==3.0.5 + framework: FLASK_VERSION=1.0.2 mock==4.0.3 - python-version: 3.3 - framework: DJANGO_VERSION=1.6.11 mock==3.0.5 + framework: DJANGO_VERSION=1.6.11 mock==4.0.3 - python-version: 3.3 - framework: DJANGO_VERSION=1.8.19 mock==3.0.5 + framework: DJANGO_VERSION=1.8.19 mock==4.0.3 - python-version: 3.4 framework: DJANGO_VERSION=1.7.11 chardet==3.0.4 idna==2.7 mock==3.0.5 - python-version: 3.4 From ab02383f49845dceefdee79bcf4e0f3108074dbf Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 16:45:19 -0700 Subject: [PATCH 19/45] fixing mock version --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ba4154b..2a86f23c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,15 +143,15 @@ jobs: - python-version: 3.3 framework: FLASK_VERSION=0.10.1 mock==4.0.3 - python-version: 3.3 - framework: FLASK_VERSION=0.11.1 mock==4.0.3 + framework: FLASK_VERSION=0.11.1 mock==2.0.0 - python-version: 3.3 - framework: FLASK_VERSION=0.12.4 mock==4.0.3 + framework: FLASK_VERSION=0.12.4 mock==2.0.0 - python-version: 3.3 - framework: FLASK_VERSION=1.0.2 mock==4.0.3 + framework: FLASK_VERSION=1.0.2 mock==2.0.0 - python-version: 3.3 - framework: DJANGO_VERSION=1.6.11 mock==4.0.3 + framework: DJANGO_VERSION=1.6.11 mock==2.0.0 - python-version: 3.3 - framework: DJANGO_VERSION=1.8.19 mock==4.0.3 + framework: DJANGO_VERSION=1.8.19 mock==2.0.0 - python-version: 3.4 framework: DJANGO_VERSION=1.7.11 chardet==3.0.4 idna==2.7 mock==3.0.5 - python-version: 3.4 From b1628cd9aec1819f3ef3c8734783116302f33105 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 16:49:04 -0700 Subject: [PATCH 20/45] fixing mock version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a86f23c..781e8a56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,7 +141,7 @@ jobs: - python-version: 2.7 framework: FLASK_VERSION=0.9 - python-version: 3.3 - framework: FLASK_VERSION=0.10.1 mock==4.0.3 + framework: FLASK_VERSION=0.10.1 mock==2.0.0 - python-version: 3.3 framework: FLASK_VERSION=0.11.1 mock==2.0.0 - python-version: 3.3 From 6a902d7ea0712033b31b31db6a05b2c62f9b4f40 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 16:53:07 -0700 Subject: [PATCH 21/45] added mock to TWISTED --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 781e8a56..0450ecde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,14 +18,14 @@ jobs: - FLASK_VERSION=0.11.1 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 idna==2.7 mock==3.0.5 - FLASK_VERSION=0.12.4 Werkzeug\>=0.7,\<1.0 chardet==3.0.4 idna==2.7 mock==3.0.5 - FLASK_VERSION=1.0.2 chardet==3.0.4 idna==2.7 mock==3.0.5 - - TWISTED_VERSION=15.5.0 treq==15.1.0 zope.interface==4.1.3 - - TWISTED_VERSION=16.1.0 treq==16.12.0 zope.interface==4.1.3 - - TWISTED_VERSION=16.2.0 treq==16.12.0 zope.interface==4.1.3 - - TWISTED_VERSION=16.3.0 treq==16.12.0 zope.interface==4.2.0 - - TWISTED_VERSION=16.4.0 treq==16.12.0 zope.interface==4.5.0 - - TWISTED_VERSION=16.5.0 treq==16.12.0 zope.interface==4.5.0 - - TWISTED_VERSION=16.6.0 treq==16.12.0 zope.interface==4.5.0 - - TWISTED_VERSION=17.1.0 treq==16.12.0 zope.interface==4.5.0 + - TWISTED_VERSION=15.5.0 treq==15.1.0 zope.interface==4.1.3 mock==2.0.0 + - TWISTED_VERSION=16.1.0 treq==16.12.0 zope.interface==4.1.3 mock==2.0.0 + - TWISTED_VERSION=16.2.0 treq==16.12.0 zope.interface==4.1.3 mock==2.0.0 + - TWISTED_VERSION=16.3.0 treq==16.12.0 zope.interface==4.2.0 mock==2.0.0 + - TWISTED_VERSION=16.4.0 treq==16.12.0 zope.interface==4.5.0 mock==2.0.0 + - TWISTED_VERSION=16.5.0 treq==16.12.0 zope.interface==4.5.0 mock==2.0.0 + - TWISTED_VERSION=16.6.0 treq==16.12.0 zope.interface==4.5.0 mock==2.0.0 + - TWISTED_VERSION=17.1.0 treq==16.12.0 zope.interface==4.5.0 mock==2.0.0 - DJANGO_VERSION=1.11.20 chardet==3.0.4 idna==2.7 mock==3.0.5 - DJANGO_VERSION=2.0.13 - DJANGO_VERSION=2.1.7 From 353dba8712cb3fd1e1ec84899d6869f7db668c69 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 23 Jul 2021 17:05:39 -0700 Subject: [PATCH 22/45] added mock --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0450ecde..79f51232 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,9 +27,9 @@ jobs: - TWISTED_VERSION=16.6.0 treq==16.12.0 zope.interface==4.5.0 mock==2.0.0 - TWISTED_VERSION=17.1.0 treq==16.12.0 zope.interface==4.5.0 mock==2.0.0 - DJANGO_VERSION=1.11.20 chardet==3.0.4 idna==2.7 mock==3.0.5 - - DJANGO_VERSION=2.0.13 - - DJANGO_VERSION=2.1.7 - - DJANGO_VERSION=2.1.15 + - DJANGO_VERSION=2.0.13 mock==2.0.0 + - DJANGO_VERSION=2.1.7 mock==2.0.0 + - DJANGO_VERSION=2.1.15 mock=2.0.0 - PYRAMID_VERSION=1.9.2 chardet==3.0.4 idna==2.7 mock==3.0.5 - PYRAMID_VERSION=1.10.4 chardet==3.0.4 idna==2.7 mock==3.0.5 - STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 From 5a82b304f3dae8e4dc6482bd38cf7c8eedf0f261 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Mon, 26 Jul 2021 10:12:30 -0700 Subject: [PATCH 23/45] added mock to starlette --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79f51232..98113052 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,12 +32,12 @@ jobs: - DJANGO_VERSION=2.1.15 mock=2.0.0 - PYRAMID_VERSION=1.9.2 chardet==3.0.4 idna==2.7 mock==3.0.5 - PYRAMID_VERSION=1.10.4 chardet==3.0.4 idna==2.7 mock==3.0.5 - - STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 - - STARLETTE_VERSION=0.12.13 httpx==0.18.1 python-multipart==0.0.5 - - STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 - - FASTAPI_VERSION=0.40.0 httpx==0.18.1 python-multipart==0.0.5 - - FASTAPI_VERSION=0.50.0 httpx==0.18.1 python-multipart==0.0.5 - - FASTAPI_VERSION=0.63.0 httpx==0.18.1 python-multipart==0.0.5 + - STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 + - STARLETTE_VERSION=0.12.13 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 + - STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 + - FASTAPI_VERSION=0.40.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 + - FASTAPI_VERSION=0.50.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 + - FASTAPI_VERSION=0.63.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 exclude: - python-version: 2.7 framework: DJANGO_VERSION=2.0.13 From 52a831893894943d8ccc7504bb8134c6052b065e Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Mon, 26 Jul 2021 10:16:55 -0700 Subject: [PATCH 24/45] fix tests --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98113052..fa211e7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,11 +40,11 @@ jobs: - FASTAPI_VERSION=0.63.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 exclude: - python-version: 2.7 - framework: DJANGO_VERSION=2.0.13 + framework: DJANGO_VERSION=2.0.13 mock==2.0.0 - python-version: 2.7 - framework: DJANGO_VERSION=2.1.7 + framework: DJANGO_VERSION=2.1.7 mock==2.0.0 - python-version: 2.7 - framework: DJANGO_VERSION=2.1.15 + framework: DJANGO_VERSION=2.1.15 mock==2.0.0 - python-version: 3.4 framework: DJANGO_VERSION=2.0.13 - python-version: 3.4 From 766e10a7b9a8ed5a231f35eeb7f0d83e4a2a94dd Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Mon, 26 Jul 2021 10:24:59 -0700 Subject: [PATCH 25/45] corrected tests --- .github/workflows/ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa211e7d..2ef24c9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,23 +101,23 @@ jobs: framework: TWISTED_VERSION=17.1.0 treq==20.4.1 zope.interface==4.3.0 - python-version: 2.7 - framework: STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 + framework: STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 2.7 - framework: STARLETTE_VERSION=0.12.13 httpx==0.18.1 python-multipart==0.0.5 + framework: STARLETTE_VERSION=0.12.13 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 2.7 - framework: STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 + framework: STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.4 - framework: STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 + framework: STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.4 - framework: STARLETTE_VERSION=0.12.13 httpx==0.18.1 python-multipart==0.0.5 + framework: STARLETTE_VERSION=0.12.13 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.4 - framework: STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 + framework: STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.5 - framework: STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 + framework: STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.5 - framework: STARLETTE_VERSION=0.12.13 httpx==0.18.1 python-multipart==0.0.5 + framework: STARLETTE_VERSION=0.12.13 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.5 - framework: STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 + framework: STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 2.7 framework: FASTAPI_VERSION=0.40.0 httpx==0.18.1 python-multipart==0.0.5 From d2bf64310cbddc2b501aadd7c4bf4483257c7546 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Mon, 26 Jul 2021 10:31:36 -0700 Subject: [PATCH 26/45] corrected tests --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ef24c9a..712d60d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: - DJANGO_VERSION=1.11.20 chardet==3.0.4 idna==2.7 mock==3.0.5 - DJANGO_VERSION=2.0.13 mock==2.0.0 - DJANGO_VERSION=2.1.7 mock==2.0.0 - - DJANGO_VERSION=2.1.15 mock=2.0.0 + - DJANGO_VERSION=2.1.15 mock==2.0.0 - PYRAMID_VERSION=1.9.2 chardet==3.0.4 idna==2.7 mock==3.0.5 - PYRAMID_VERSION=1.10.4 chardet==3.0.4 idna==2.7 mock==3.0.5 - STARLETTE_VERSION=0.12.12 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 @@ -120,23 +120,23 @@ jobs: framework: STARLETTE_VERSION=0.14.2 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 2.7 - framework: FASTAPI_VERSION=0.40.0 httpx==0.18.1 python-multipart==0.0.5 + framework: FASTAPI_VERSION=0.40.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 2.7 - framework: FASTAPI_VERSION=0.50.0 httpx==0.18.1 python-multipart==0.0.5 + framework: FASTAPI_VERSION=0.50.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 2.7 - framework: FASTAPI_VERSION=0.63.0 httpx==0.18.1 python-multipart==0.0.5 + framework: FASTAPI_VERSION=0.63.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.4 - framework: FASTAPI_VERSION=0.40.0 httpx==0.18.1 python-multipart==0.0.5 + framework: FASTAPI_VERSION=0.40.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.4 - framework: FASTAPI_VERSION=0.50.0 httpx==0.18.1 python-multipart==0.0.5 + framework: FASTAPI_VERSION=0.50.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.4 - framework: FASTAPI_VERSION=0.63.0 httpx==0.18.1 python-multipart==0.0.5 + framework: FASTAPI_VERSION=0.63.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.5 - framework: FASTAPI_VERSION=0.40.0 httpx==0.18.1 python-multipart==0.0.5 + framework: FASTAPI_VERSION=0.40.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.5 - framework: FASTAPI_VERSION=0.50.0 httpx==0.18.1 python-multipart==0.0.5 + framework: FASTAPI_VERSION=0.50.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 - python-version: 3.5 - framework: FASTAPI_VERSION=0.63.0 httpx==0.18.1 python-multipart==0.0.5 + framework: FASTAPI_VERSION=0.63.0 httpx==0.18.1 python-multipart==0.0.5 mock==3.0.5 include: - python-version: 2.7 framework: FLASK_VERSION=0.9 From 53132d1dfb3044941afec7b392614cdff498b4fb Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Mon, 26 Jul 2021 10:35:26 -0700 Subject: [PATCH 27/45] corrected tests --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 712d60d5..6bc59bd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,21 +46,21 @@ jobs: - python-version: 2.7 framework: DJANGO_VERSION=2.1.15 mock==2.0.0 - python-version: 3.4 - framework: DJANGO_VERSION=2.0.13 + framework: DJANGO_VERSION=2.0.13 mock==2.0.0 - python-version: 3.4 - framework: DJANGO_VERSION=2.1.7 + framework: DJANGO_VERSION=2.1.7 mock==2.0.0 - python-version: 3.4 - framework: DJANGO_VERSION=2.1.15 + framework: DJANGO_VERSION=2.1.15 mock==2.0.0 - python-version: 3.5 - framework: DJANGO_VERSION=2.1.15 + framework: DJANGO_VERSION=2.1.15 mock==2.0.0 - python-version: 3.5 - framework: DJANGO_VERSION=2.0.13 + framework: DJANGO_VERSION=2.0.13 mock==2.0.0 - python-version: 3.5 - framework: DJANGO_VERSION=2.1.7 + framework: DJANGO_VERSION=2.1.7 mock==2.0.0 - python-version: 3.6 - framework: DJANGO_VERSION=2.1.15 + framework: DJANGO_VERSION=2.1.15 mock==2.0.0 - python-version: 3.7 - framework: DJANGO_VERSION=2.1.15 + framework: DJANGO_VERSION=2.1.15 mock==2.0.0 # twisted/treq setup.py allows: # Twisted < 18.7.0 on python < 3.7 From ec5208f4e8a339fe282edc50b2c65b81a2f83848 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Mon, 26 Jul 2021 10:40:32 -0700 Subject: [PATCH 28/45] added mock to TWISTED --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bc59bd2..0999de84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,17 +173,17 @@ jobs: - python-version: 3.5 framework: DJANGO_VERSION=2.1.7 mock==3.0.5 - python-version: 3.7 - framework: TWISTED_VERSION=18.9.0 treq==20.4.1 zope.interface==4.5.0 + framework: TWISTED_VERSION=18.9.0 treq==20.4.1 zope.interface==4.5.0 mock==3.0.5 - python-version: 3.7 - framework: TWISTED_VERSION=19.10.0 treq==20.4.1 zope.interface==4.6.0 + framework: TWISTED_VERSION=19.10.0 treq==20.4.1 zope.interface==4.6.0 mock==3.0.5 - python-version: 3.7 - framework: TWISTED_VERSION=20.3.0 treq==20.4.1 zope.interface==4.7.0 + framework: TWISTED_VERSION=20.3.0 treq==20.4.1 zope.interface==4.7.0 mock==3.0.5 - python-version: 3.8 - framework: TWISTED_VERSION=18.9.0 treq==20.4.1 zope.interface==4.5.0 + framework: TWISTED_VERSION=18.9.0 treq==20.4.1 zope.interface==4.5.0 mock==3.0.5 - python-version: 3.8 - framework: TWISTED_VERSION=19.10.0 treq==20.4.1 zope.interface==4.6.0 + framework: TWISTED_VERSION=19.10.0 treq==20.4.1 zope.interface==4.6.0 mock==3.0.5 - python-version: 3.8 - framework: TWISTED_VERSION=20.3.0 treq==20.4.1 zope.interface==4.7.0 + framework: TWISTED_VERSION=20.3.0 treq==20.4.1 zope.interface==4.7.0 mock==3.0.5 steps: - uses: actions/checkout@v2 with: From ca9df673f58fb1806b8e088e3266e67baa62eabe Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:39:12 -0700 Subject: [PATCH 29/45] Update rollbar/lib/telemetry.py Co-authored-by: Bart Skowron --- rollbar/lib/telemetry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index 1af0a53c..b36fa752 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -54,7 +54,6 @@ def get_current_timestamp(): def request(request_function, enable_req_headers, enable_response_headers): def telemetry(*args, **kwargs): - def clean_headers(headers): if not headers: return [] From 9f0d61d7fcff49be89fa8797fa7ae8b68b8dd829 Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:39:23 -0700 Subject: [PATCH 30/45] Update rollbar/test/test_telemetry.py Co-authored-by: Bart Skowron --- rollbar/test/test_telemetry.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py index e09d67ed..2601f659 100644 --- a/rollbar/test/test_telemetry.py +++ b/rollbar/test/test_telemetry.py @@ -35,8 +35,17 @@ def test_telemetry_request(self, timestamp): items = telemetry.TELEMETRY_QUEUE.get_items() self.assertEqual(1, len(items)) - result = {'body': {'url': 'http://example.com', 'status_code': 200, 'method': 'get', - 'subtype': 'http'}, 'source': 'client', 'timestamp_ms': 1000000, - 'type': 'network', 'level': 'info'} + result = { + 'body': { + 'url': 'http://example.com', + 'status_code': 200, + 'method': 'get', + 'subtype': 'http', + }, + 'source': 'client', + 'timestamp_ms': 1000000, + 'type': 'network', + 'level': 'info', + } self.assertEqual(result, items[0]) telemetry.TELEMETRY_QUEUE.clear_items() From e59e34c9e5fda81b3cca828e0d237efbe5895018 Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:39:47 -0700 Subject: [PATCH 31/45] Update rollbar/__init__.py Co-authored-by: Bart Skowron --- rollbar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rollbar/__init__.py b/rollbar/__init__.py index e45e5a86..90c02b18 100644 --- a/rollbar/__init__.py +++ b/rollbar/__init__.py @@ -20,7 +20,7 @@ import requests import six -from rollbar.lib import events, filters, dict_merge, parse_qs, text, transport, telemetry, urljoin, iteritems, defaultJSONEncode +from rollbar.lib import events, filters, dict_merge, parse_qs, telemetry, text, transport, urljoin, iteritems, defaultJSONEncode __version__ = '0.16.1' From 5797a30a137f46105142ba2a29df48ad583caebb Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:40:00 -0700 Subject: [PATCH 32/45] Update rollbar/test/test_telemetry.py Co-authored-by: Bart Skowron --- rollbar/test/test_telemetry.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py index 2601f659..63a5e408 100644 --- a/rollbar/test/test_telemetry.py +++ b/rollbar/test/test_telemetry.py @@ -20,9 +20,13 @@ def test_telemetry_log(self, timestamp): items = telemetry.TELEMETRY_QUEUE.get_items() self.assertEqual(1, len(items)) - result = {'body': {'message': 'root :: WARNING :: test loggin'}, - 'source': 'client', 'level': 'WARNING', 'type': 'log', - 'timestamp_ms': 1000000} + result = { + 'body': {'message': 'root :: WARNING :: test loggin'}, + 'source': 'client', + 'level': 'WARNING', + 'type': 'log', + 'timestamp_ms': 1000000, + } self.assertEqual(result, items[0]) telemetry.TELEMETRY_QUEUE.clear_items() From cfafa976791fbd4517d820ee42cf5387046b0948 Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:40:24 -0700 Subject: [PATCH 33/45] Update rollbar/lib/telemetry.py Co-authored-by: Bart Skowron --- rollbar/lib/telemetry.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index b36fa752..cd7dbd3a 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -38,8 +38,13 @@ def __init__(self, formatter=None): def emit(self, record): self.setFormatter(self.formatter) msg = {'message': self.format(record)} - data = {'body': msg, 'source': 'client', 'timestamp_ms': get_current_timestamp(), 'type': 'log', - 'level': record.levelname} + data = { + 'body': msg, + 'source': 'client', + 'timestamp_ms': get_current_timestamp(), + 'type': 'log', + 'level': record.levelname, + } TELEMETRY_QUEUE.put(data) From 0f09f7e1bf3d8af229371260f9ed68115036b1b5 Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:40:57 -0700 Subject: [PATCH 34/45] Update rollbar/lib/telemetry.py Co-authored-by: Bart Skowron --- rollbar/lib/telemetry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index cd7dbd3a..908935f3 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -66,6 +66,7 @@ def clean_headers(headers): if h in rollbar.SETTINGS['scrub_fields']: headers[h] = '[FILTERED]' return headers + data = {'level': 'info'} data_body = {'status_code': None} try: From 16e6a9b61177024ea4d4805300a8ac5fe1161556 Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:41:24 -0700 Subject: [PATCH 35/45] Update rollbar/lib/telemetry.py Co-authored-by: Bart Skowron --- rollbar/lib/telemetry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index 908935f3..6089737d 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -61,7 +61,7 @@ def request(request_function, enable_req_headers, enable_response_headers): def telemetry(*args, **kwargs): def clean_headers(headers): if not headers: - return [] + return headers for h in headers: if h in rollbar.SETTINGS['scrub_fields']: headers[h] = '[FILTERED]' From 070b188a3651fdc2d385ddae1daf5198b734e337 Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:42:22 -0700 Subject: [PATCH 36/45] Update rollbar/lib/telemetry.py Co-authored-by: Bart Skowron --- rollbar/lib/telemetry.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index 6089737d..9bf0a6de 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -62,10 +62,7 @@ def telemetry(*args, **kwargs): def clean_headers(headers): if not headers: return headers - for h in headers: - if h in rollbar.SETTINGS['scrub_fields']: - headers[h] = '[FILTERED]' - return headers + return {h: '[FILTERED]' if h in rollbar.SETTINGS['scrub_fields'] else headers[h] for h in headers} data = {'level': 'info'} data_body = {'status_code': None} From b317564aa650e0ef850c58ddbaf8ffda094fe66f Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:43:36 -0700 Subject: [PATCH 37/45] Update rollbar/lib/telemetry.py Co-authored-by: Bart Skowron --- rollbar/lib/telemetry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index 9bf0a6de..dae13553 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -82,7 +82,7 @@ def clean_headers(headers): if enable_req_headers: data_body['request_headers'] = clean_headers(kwargs.get('headers')) data_body['url'] = args[0] - data_body['method'] = request_function.__name__ + data_body['method'] = request_function.__name__.upper() data_body['subtype'] = 'http' data['body'] = data_body From 88d8da3668b5f12920e569cd82fef766b2ffd1bb Mon Sep 17 00:00:00 2001 From: pawelsz-rb <76971683+pawelsz-rb@users.noreply.github.com> Date: Wed, 4 Aug 2021 16:56:29 -0700 Subject: [PATCH 38/45] Update rollbar/test/test_telemetry.py Co-authored-by: Bart Skowron --- rollbar/test/test_telemetry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py index 63a5e408..861166f7 100644 --- a/rollbar/test/test_telemetry.py +++ b/rollbar/test/test_telemetry.py @@ -6,7 +6,6 @@ class RollbarTelemetryTest(BaseTest): - @classmethod def setUpClass(self): formatter = logging.Formatter('%(name)s :: %(levelname)s :: %(message)s') From 1f1d8da4489e3a6cc582c04dcb34f6b651e9cdb9 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Thu, 5 Aug 2021 11:33:42 -0700 Subject: [PATCH 39/45] addressing PR comments --- rollbar/__init__.py | 25 +++++++++++------------ rollbar/lib/__init__.py | 6 +++++- rollbar/lib/telemetry.py | 43 ++++++++++++++++++++-------------------- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/rollbar/__init__.py b/rollbar/__init__.py index 90c02b18..7257dbfa 100644 --- a/rollbar/__init__.py +++ b/rollbar/__init__.py @@ -10,7 +10,6 @@ import socket import sys import threading -import time import traceback import types import uuid @@ -20,7 +19,7 @@ import requests import six -from rollbar.lib import events, filters, dict_merge, parse_qs, telemetry, text, transport, urljoin, iteritems, defaultJSONEncode +from rollbar.lib import events, filters, dict_merge, parse_qs, telemetry, text, transport, urljoin, iteritems, defaultJSONEncode, get_current_timestamp __version__ = '0.16.1' @@ -322,8 +321,12 @@ def _get_fastapi_request(): 'request_pool_connections': None, 'request_pool_maxsize': None, 'request_max_retries': None, + 'telemetry_queue_size': 50, } + +TELEMETRY_QUEUE = telemetry.Queue(SETTINGS['telemetry_queue_size']) + _CURRENT_LAMBDA_CONTEXT = None _LAST_RESPONSE_STATUS = None @@ -381,21 +384,17 @@ def init(access_token, environment='production', scrub_fields=None, url_fields=N if SETTINGS.get('allow_logging_basic_config'): logging.basicConfig() - queue_size = SETTINGS.get('set_custom_queue_size') + queue_size = SETTINGS.get('telemetry_queue_size') if queue_size: - telemetry.TELEMETRY_QUEUE = telemetry.Queue(queue_size) + TELEMETRY_QUEUE = telemetry.Queue(queue_size) if SETTINGS.get('log_telemetry'): formatter = SETTINGS.get('log_telemetry_formatter') - telemetry.set_log_telemetry(formatter) + telemetry.enable_log_telemetry(formatter) if SETTINGS.get('network_telemetry'): enable_req_headers = SETTINGS.get('enable_req_headers') - enable_response_headers = SETTINGS.get('enable_response_headers') - requests.get = telemetry.request(requests.get, enable_req_headers, enable_response_headers ) - requests.post = telemetry.request(requests.post, enable_req_headers, enable_response_headers) - requests.put = telemetry.request(requests.put, enable_req_headers, enable_response_headers) - requests.patch = telemetry.request(requests.patch, enable_req_headers, enable_response_headers) - requests.delete = telemetry.request(requests.delete, enable_req_headers, enable_response_headers) + enable_resp_headers = SETTINGS.get('enable_resp_headers') + telemetry.enable_network_telemetry(enable_req_headers, enable_resp_headers) if SETTINGS.get('handler') == 'agent': agent_log = _create_agent_log() @@ -905,7 +904,7 @@ def _check_config(): def _build_base_data(request, level='error'): data = { - 'timestamp': int(time.time()), + 'timestamp': get_current_timestamp(), 'environment': SETTINGS['environment'], 'level': level, 'language': 'python %s' % '.'.join(str(x) for x in sys.version_info[:3]), @@ -1138,7 +1137,7 @@ def _add_request_data(data, request): def _add_telemetry(data): - telemetry_data = telemetry.TELEMETRY_QUEUE.get_items() + telemetry_data = TELEMETRY_QUEUE.get_items() if telemetry_data: data['body']['telemetry'] = telemetry_data diff --git a/rollbar/lib/__init__.py b/rollbar/lib/__init__.py index 78a32d87..8eae5e50 100644 --- a/rollbar/lib/__init__.py +++ b/rollbar/lib/__init__.py @@ -4,7 +4,7 @@ import os import sys from array import array -import json +import time try: # Python 3 @@ -229,3 +229,7 @@ def defaultJSONEncode(o): o._setup() return o._wrapped return repr(o) + " is not JSON serializable" + + +def get_current_timestamp(): + return int(time.time()) diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index dae13553..bf2f514f 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -1,8 +1,11 @@ import logging import threading -import time +import requests +import copy import rollbar +from rollbar.lib import transforms +from rollbar.lib.transforms.scrub import ScrubTransform class Queue: @@ -26,10 +29,6 @@ def clear_items(self): self.items = [] -TELEMETRY_QUEUE_SIZE = 50 -TELEMETRY_QUEUE = Queue(TELEMETRY_QUEUE_SIZE) - - class TelemetryLogHandler(logging.Handler): def __init__(self, formatter=None): super(TelemetryLogHandler, self).__init__() @@ -41,29 +40,23 @@ def emit(self, record): data = { 'body': msg, 'source': 'client', - 'timestamp_ms': get_current_timestamp(), + 'timestamp_ms': rollbar.get_current_timestamp(), 'type': 'log', 'level': record.levelname, } - TELEMETRY_QUEUE.put(data) + rollbar.TELEMETRY_QUEUE.put(data) -def set_log_telemetry(log_formatter=None): +def enable_log_telemetry(log_formatter=None): logging.getLogger().addHandler(TelemetryLogHandler(log_formatter)) -def get_current_timestamp(): - return int(time.time()) - - def request(request_function, enable_req_headers, enable_response_headers): + scrubber = ScrubTransform(suffixes=rollbar.SETTINGS['scrub_fields'], redact_char='*', randomize_len=False) + def telemetry(*args, **kwargs): - def clean_headers(headers): - if not headers: - return headers - return {h: '[FILTERED]' if h in rollbar.SETTINGS['scrub_fields'] else headers[h] for h in headers} - + data = {'level': 'info'} data_body = {'status_code': None} try: @@ -78,19 +71,27 @@ def clean_headers(headers): elif response.status_code >= 400: data['level'] = 'error' if enable_response_headers: - data_body['response'] = {'headers': clean_headers(response.headers)} + data_body['response'] = {'headers': transforms.transform(copy.deepcopy(response.headers), scrubber)} if enable_req_headers: - data_body['request_headers'] = clean_headers(kwargs.get('headers')) + data_body['request_headers'] = transforms.transform(copy.deepcopy(kwargs.get('headers')), scrubber) data_body['url'] = args[0] data_body['method'] = request_function.__name__.upper() data_body['subtype'] = 'http' data['body'] = data_body data['source'] = 'client' - data['timestamp_ms'] = get_current_timestamp() + data['timestamp_ms'] = rollbar.get_current_timestamp() data['type'] = 'network' - TELEMETRY_QUEUE.put(data) + rollbar.TELEMETRY_QUEUE.put(data) return response return telemetry + + +def enable_network_telemetry(enable_req_headers, enable_resp_headers): + requests.get = request(requests.get, enable_req_headers, enable_resp_headers) + requests.post = request(requests.post, enable_req_headers, enable_resp_headers) + requests.put = request(requests.put, enable_req_headers, enable_resp_headers) + requests.patch = request(requests.patch, enable_req_headers, enable_resp_headers) + requests.delete = request(requests.delete, enable_req_headers, enable_resp_headers) From 7f57fae7a15f329be045e4cb5cfc6b76a52055ce Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Thu, 5 Aug 2021 12:31:09 -0700 Subject: [PATCH 40/45] corrected tests --- rollbar/test/test_telemetry.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py index 861166f7..c777cc81 100644 --- a/rollbar/test/test_telemetry.py +++ b/rollbar/test/test_telemetry.py @@ -3,20 +3,21 @@ import logging import mock import requests +import rollbar class RollbarTelemetryTest(BaseTest): @classmethod def setUpClass(self): formatter = logging.Formatter('%(name)s :: %(levelname)s :: %(message)s') - telemetry.set_log_telemetry(formatter) - requests.get = telemetry.request(requests.get, False, False) + telemetry.enable_log_telemetry(formatter) + telemetry.enable_network_telemetry(False, False) - @mock.patch('rollbar.telemetry.get_current_timestamp') + @mock.patch('rollbar.get_current_timestamp') def test_telemetry_log(self, timestamp): timestamp.return_value = 1000000 logging.warning("test loggin") - items = telemetry.TELEMETRY_QUEUE.get_items() + items = rollbar.TELEMETRY_QUEUE.get_items() self.assertEqual(1, len(items)) result = { @@ -28,7 +29,7 @@ def test_telemetry_log(self, timestamp): } self.assertEqual(result, items[0]) - telemetry.TELEMETRY_QUEUE.clear_items() + rollbar.TELEMETRY_QUEUE.clear_items() @mock.patch('rollbar.telemetry.get_current_timestamp') def test_telemetry_request(self, timestamp): From 2e09a117e740ecd4111a50d540c70272a03f91cb Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Thu, 5 Aug 2021 12:38:06 -0700 Subject: [PATCH 41/45] corrected tests --- rollbar/test/test_telemetry.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py index c777cc81..21301419 100644 --- a/rollbar/test/test_telemetry.py +++ b/rollbar/test/test_telemetry.py @@ -31,19 +31,19 @@ def test_telemetry_log(self, timestamp): self.assertEqual(result, items[0]) rollbar.TELEMETRY_QUEUE.clear_items() - @mock.patch('rollbar.telemetry.get_current_timestamp') + @mock.patch('rollbar.get_current_timestamp') def test_telemetry_request(self, timestamp): timestamp.return_value = 1000000 requests.get("http://example.com") - items = telemetry.TELEMETRY_QUEUE.get_items() + items = rollbar.TELEMETRY_QUEUE.get_items() self.assertEqual(1, len(items)) result = { 'body': { 'url': 'http://example.com', 'status_code': 200, - 'method': 'get', + 'method': 'GET', 'subtype': 'http', }, 'source': 'client', @@ -52,4 +52,4 @@ def test_telemetry_request(self, timestamp): 'level': 'info', } self.assertEqual(result, items[0]) - telemetry.TELEMETRY_QUEUE.clear_items() + rollbar.TELEMETRY_QUEUE.clear_items() From 1d27b191d30d90ea7269d6c212b5c4efde07e49f Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Thu, 5 Aug 2021 16:02:25 -0700 Subject: [PATCH 42/45] addressing PR comments --- rollbar/__init__.py | 7 ++++--- rollbar/lib/telemetry.py | 27 ++++----------------------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/rollbar/__init__.py b/rollbar/__init__.py index 7257dbfa..e07bbafd 100644 --- a/rollbar/__init__.py +++ b/rollbar/__init__.py @@ -19,6 +19,7 @@ import requests import six +from collections import deque from rollbar.lib import events, filters, dict_merge, parse_qs, telemetry, text, transport, urljoin, iteritems, defaultJSONEncode, get_current_timestamp @@ -325,7 +326,7 @@ def _get_fastapi_request(): } -TELEMETRY_QUEUE = telemetry.Queue(SETTINGS['telemetry_queue_size']) +TELEMETRY_QUEUE = deque([], SETTINGS['telemetry_queue_size']) _CURRENT_LAMBDA_CONTEXT = None _LAST_RESPONSE_STATUS = None @@ -386,7 +387,7 @@ def init(access_token, environment='production', scrub_fields=None, url_fields=N queue_size = SETTINGS.get('telemetry_queue_size') if queue_size: - TELEMETRY_QUEUE = telemetry.Queue(queue_size) + TELEMETRY_QUEUE = deque([], queue_size) if SETTINGS.get('log_telemetry'): formatter = SETTINGS.get('log_telemetry_formatter') @@ -1137,7 +1138,7 @@ def _add_request_data(data, request): def _add_telemetry(data): - telemetry_data = TELEMETRY_QUEUE.get_items() + telemetry_data = list(TELEMETRY_QUEUE) if telemetry_data: data['body']['telemetry'] = telemetry_data diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index bf2f514f..f1660f48 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -8,27 +8,6 @@ from rollbar.lib.transforms.scrub import ScrubTransform -class Queue: - def __init__(self, size): - self.size = size - self.items = [] - self.lock = threading.Lock() - - def put(self, item): - with self.lock: - if len(self.items) >= self.size: - self.items = self.items[1:] - self.items.append(item) - - def get_items(self): - with self.lock: - return self.items - - def clear_items(self): - with self.lock: - self.items = [] - - class TelemetryLogHandler(logging.Handler): def __init__(self, formatter=None): super(TelemetryLogHandler, self).__init__() @@ -45,7 +24,7 @@ def emit(self, record): 'level': record.levelname, } - rollbar.TELEMETRY_QUEUE.put(data) + rollbar.TELEMETRY_QUEUE.append(data) def enable_log_telemetry(log_formatter=None): @@ -63,6 +42,8 @@ def telemetry(*args, **kwargs): response = request_function(*args, **kwargs) except: # noqa: E722 response = None + data_body['status_code'] = 0 + data['level'] = 'critical' if response is not None: data_body['status_code'] = response.status_code @@ -82,7 +63,7 @@ def telemetry(*args, **kwargs): data['source'] = 'client' data['timestamp_ms'] = rollbar.get_current_timestamp() data['type'] = 'network' - rollbar.TELEMETRY_QUEUE.put(data) + rollbar.TELEMETRY_QUEUE.append(data) return response From 2d270df9469c1897a9370c74bb07fbcb29412884 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Thu, 5 Aug 2021 16:07:06 -0700 Subject: [PATCH 43/45] correcting tests --- rollbar/test/test_telemetry.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py index 21301419..e07530c7 100644 --- a/rollbar/test/test_telemetry.py +++ b/rollbar/test/test_telemetry.py @@ -17,7 +17,7 @@ def setUpClass(self): def test_telemetry_log(self, timestamp): timestamp.return_value = 1000000 logging.warning("test loggin") - items = rollbar.TELEMETRY_QUEUE.get_items() + items = list(rollbar.TELEMETRY_QUEUE) self.assertEqual(1, len(items)) result = { @@ -29,14 +29,14 @@ def test_telemetry_log(self, timestamp): } self.assertEqual(result, items[0]) - rollbar.TELEMETRY_QUEUE.clear_items() + rollbar.TELEMETRY_QUEUE.clear() @mock.patch('rollbar.get_current_timestamp') def test_telemetry_request(self, timestamp): timestamp.return_value = 1000000 requests.get("http://example.com") - items = rollbar.TELEMETRY_QUEUE.get_items() + items = list(rollbar.TELEMETRY_QUEUE) self.assertEqual(1, len(items)) result = { @@ -52,4 +52,4 @@ def test_telemetry_request(self, timestamp): 'level': 'info', } self.assertEqual(result, items[0]) - rollbar.TELEMETRY_QUEUE.clear_items() + rollbar.TELEMETRY_QUEUE.clear() From 5d7c1f96dcce60d8e66c6a14efcf3e51739c3e13 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Thu, 5 Aug 2021 17:11:55 -0700 Subject: [PATCH 44/45] added support for urllib --- rollbar/lib/telemetry.py | 37 ++++++++++++++++++++++++---------- rollbar/test/test_telemetry.py | 31 ++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index f1660f48..69542be8 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -1,5 +1,4 @@ import logging -import threading import requests import copy @@ -7,6 +6,13 @@ from rollbar.lib import transforms from rollbar.lib.transforms.scrub import ScrubTransform +try: + # 3.x + import urllib.request as ulib +except ImportError: + # 2.x + import urllib as ulib + class TelemetryLogHandler(logging.Handler): def __init__(self, formatter=None): @@ -31,7 +37,7 @@ def enable_log_telemetry(log_formatter=None): logging.getLogger().addHandler(TelemetryLogHandler(log_formatter)) -def request(request_function, enable_req_headers, enable_response_headers): +def request(request_function, enable_req_headers, enable_response_headers, request_type): scrubber = ScrubTransform(suffixes=rollbar.SETTINGS['scrub_fields'], redact_char='*', randomize_len=False) def telemetry(*args, **kwargs): @@ -46,17 +52,24 @@ def telemetry(*args, **kwargs): data['level'] = 'critical' if response is not None: - data_body['status_code'] = response.status_code - if response.status_code >= 500: + if request_type == 'requests': + data_body['status_code'] = response.status_code + elif request_type == 'urllib': + data_body['status_code'] = response.code + + if data_body['status_code'] >= 500: data['level'] = 'critical' - elif response.status_code >= 400: + elif data_body['status_code'] >= 400: data['level'] = 'error' if enable_response_headers: data_body['response'] = {'headers': transforms.transform(copy.deepcopy(response.headers), scrubber)} if enable_req_headers: data_body['request_headers'] = transforms.transform(copy.deepcopy(kwargs.get('headers')), scrubber) data_body['url'] = args[0] - data_body['method'] = request_function.__name__.upper() + if request_type == 'requests': + data_body['method'] = request_function.__name__.upper() + if request_type == 'urllib': + data_body['method'] = kwargs.get('method', 'GET') data_body['subtype'] = 'http' data['body'] = data_body @@ -71,8 +84,10 @@ def telemetry(*args, **kwargs): def enable_network_telemetry(enable_req_headers, enable_resp_headers): - requests.get = request(requests.get, enable_req_headers, enable_resp_headers) - requests.post = request(requests.post, enable_req_headers, enable_resp_headers) - requests.put = request(requests.put, enable_req_headers, enable_resp_headers) - requests.patch = request(requests.patch, enable_req_headers, enable_resp_headers) - requests.delete = request(requests.delete, enable_req_headers, enable_resp_headers) + requests.get = request(requests.get, enable_req_headers, enable_resp_headers, "requests") + requests.post = request(requests.post, enable_req_headers, enable_resp_headers, "requests") + requests.put = request(requests.put, enable_req_headers, enable_resp_headers, "requests") + requests.patch = request(requests.patch, enable_req_headers, enable_resp_headers, "requests") + requests.delete = request(requests.delete, enable_req_headers, enable_resp_headers, "requests") + + ulib.urlopen = request(ulib.urlopen, enable_req_headers, enable_resp_headers, "urllib") diff --git a/rollbar/test/test_telemetry.py b/rollbar/test/test_telemetry.py index e07530c7..ae30e803 100644 --- a/rollbar/test/test_telemetry.py +++ b/rollbar/test/test_telemetry.py @@ -6,6 +6,14 @@ import rollbar +try: + # 3.x + import urllib.request as ulib +except ImportError: + # 2.x + import urllib as ulib + + class RollbarTelemetryTest(BaseTest): @classmethod def setUpClass(self): @@ -53,3 +61,26 @@ def test_telemetry_request(self, timestamp): } self.assertEqual(result, items[0]) rollbar.TELEMETRY_QUEUE.clear() + + @mock.patch('rollbar.get_current_timestamp') + def test_telemetry_urllib_request(self, timestamp): + timestamp.return_value = 1000000 + + ulib.urlopen("http://example.com") + items = list(rollbar.TELEMETRY_QUEUE) + self.assertEqual(1, len(items)) + + result = { + 'body': { + 'url': 'http://example.com', + 'status_code': 200, + 'method': 'GET', + 'subtype': 'http', + }, + 'source': 'client', + 'timestamp_ms': 1000000, + 'type': 'network', + 'level': 'info', + } + self.assertEqual(result, items[0]) + rollbar.TELEMETRY_QUEUE.clear() From 91f8338d98600492b99a65a201776686a76a6403 Mon Sep 17 00:00:00 2001 From: Pawel Szczodruch Date: Fri, 6 Aug 2021 16:54:00 -0700 Subject: [PATCH 45/45] added support for httpx --- rollbar/lib/telemetry.py | 42 +++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/rollbar/lib/telemetry.py b/rollbar/lib/telemetry.py index 69542be8..9cfff91f 100644 --- a/rollbar/lib/telemetry.py +++ b/rollbar/lib/telemetry.py @@ -14,6 +14,12 @@ import urllib as ulib + +except ImportError: + # 2.x + print('please install httpx for python3') + + class TelemetryLogHandler(logging.Handler): def __init__(self, formatter=None): super(TelemetryLogHandler, self).__init__() @@ -52,7 +58,7 @@ def telemetry(*args, **kwargs): data['level'] = 'critical' if response is not None: - if request_type == 'requests': + if request_type in ['requests', 'httpx']: data_body['status_code'] = response.status_code elif request_type == 'urllib': data_body['status_code'] = response.code @@ -62,11 +68,14 @@ def telemetry(*args, **kwargs): elif data_body['status_code'] >= 400: data['level'] = 'error' if enable_response_headers: - data_body['response'] = {'headers': transforms.transform(copy.deepcopy(response.headers), scrubber)} + response_headers = copy.deepcopy(response.headers) + if not response_headers: + response_headers = {} + data_body['response'] = {'headers': transforms.transform(response_headers, scrubber)} if enable_req_headers: - data_body['request_headers'] = transforms.transform(copy.deepcopy(kwargs.get('headers')), scrubber) + data_body['request_headers'] = transforms.transform(copy.deepcopy(kwargs.get('headers', {})), scrubber) data_body['url'] = args[0] - if request_type == 'requests': + if request_type in ['requests', 'httpx']: data_body['method'] = request_function.__name__.upper() if request_type == 'urllib': data_body['method'] = kwargs.get('method', 'GET') @@ -84,10 +93,21 @@ def telemetry(*args, **kwargs): def enable_network_telemetry(enable_req_headers, enable_resp_headers): - requests.get = request(requests.get, enable_req_headers, enable_resp_headers, "requests") - requests.post = request(requests.post, enable_req_headers, enable_resp_headers, "requests") - requests.put = request(requests.put, enable_req_headers, enable_resp_headers, "requests") - requests.patch = request(requests.patch, enable_req_headers, enable_resp_headers, "requests") - requests.delete = request(requests.delete, enable_req_headers, enable_resp_headers, "requests") - - ulib.urlopen = request(ulib.urlopen, enable_req_headers, enable_resp_headers, "urllib") + requests.get = request(requests.get, enable_req_headers, enable_resp_headers, 'requests') + requests.post = request(requests.post, enable_req_headers, enable_resp_headers, 'requests') + requests.put = request(requests.put, enable_req_headers, enable_resp_headers, 'requests') + requests.patch = request(requests.patch, enable_req_headers, enable_resp_headers, 'requests') + requests.delete = request(requests.delete, enable_req_headers, enable_resp_headers, 'requests') + + ulib.urlopen = request(ulib.urlopen, enable_req_headers, enable_resp_headers, 'urllib') + + try: + # 3.x + import httpx + httpx.get = request(httpx.get, enable_req_headers, enable_resp_headers, 'httpx') + httpx.post = request(httpx.post, enable_req_headers, enable_resp_headers, 'httpx') + httpx.put = request(httpx.put, enable_req_headers, enable_resp_headers, 'httpx') + httpx.patch = request(httpx.patch, enable_req_headers, enable_resp_headers, 'httpx') + httpx.delete = request(httpx.delete, enable_req_headers, enable_resp_headers, 'httpx') + except ImportError: + pass