From d3c8239c9003ef02fd8911ea0795c08c8d442e8b Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss <26120069+alejandroroiz@users.noreply.github.com> Date: Mon, 18 Dec 2023 14:38:37 -0600 Subject: [PATCH] Upgrade pynamodb alejandroroiz (#417) --- VERSION | 2 +- confidant/models/blind_credential.py | 7 +- confidant/models/connection_cls.py | 10 - confidant/models/credential.py | 12 +- confidant/models/service.py | 7 +- confidant/models/session_cls.py | 15 -- confidant/scripts/migrate_bool.py | 7 +- confidant/settings.py | 3 +- docs/root/configuration.md | 3 +- requirements.in | 4 +- requirements.txt | 219 +++++++++++++++++++ requirements3.txt | 7 +- tests/unit/confidant/scripts/archive_test.py | 11 - tests/unit/confidant/scripts/restore_test.py | 10 - 14 files changed, 243 insertions(+), 74 deletions(-) delete mode 100644 confidant/models/connection_cls.py delete mode 100644 confidant/models/session_cls.py create mode 100644 requirements.txt diff --git a/VERSION b/VERSION index a194c18e..5e3ec53e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.5.1 +6.5.2 diff --git a/confidant/models/blind_credential.py b/confidant/models/blind_credential.py index 5d832293..f51602c5 100644 --- a/confidant/models/blind_credential.py +++ b/confidant/models/blind_credential.py @@ -11,8 +11,6 @@ from pynamodb.indexes import GlobalSecondaryIndex, AllProjection from confidant import settings -from confidant.models.session_cls import DDBSession -from confidant.models.connection_cls import DDBConnection from confidant.models.non_null_unicode_set_attribute import ( NonNullUnicodeSetAttribute ) @@ -33,8 +31,9 @@ class Meta: if settings.DYNAMODB_URL: host = settings.DYNAMODB_URL region = settings.AWS_DEFAULT_REGION - connection_cls = DDBConnection - session_cls = DDBSession + connect_timeout_seconds = settings.PYNAMO_CONNECT_TIMEOUT_SECONDS + read_timeout_seconds = settings.PYNAMO_READ_TIMEOUT_SECONDS + max_pool_connection = settings.PYNAMO_CONNECTION_POOL_SIZE id = UnicodeAttribute(hash_key=True) revision = NumberAttribute() diff --git a/confidant/models/connection_cls.py b/confidant/models/connection_cls.py deleted file mode 100644 index d69b8673..00000000 --- a/confidant/models/connection_cls.py +++ /dev/null @@ -1,10 +0,0 @@ -import pynamodb.connection - -from confidant import settings - - -class DDBConnection(pynamodb.connection.Connection): - def __init__(self, *args, **kwargs): - super(DDBConnection, self).__init__(*args, **kwargs) - _timeout_secs = settings.PYNAMO_REQUEST_TIMEOUT_SECONDS - self._request_timeout_seconds = _timeout_secs diff --git a/confidant/models/credential.py b/confidant/models/credential.py index 421002c2..6da865c5 100644 --- a/confidant/models/credential.py +++ b/confidant/models/credential.py @@ -14,8 +14,6 @@ from pynamodb.indexes import GlobalSecondaryIndex, AllProjection from confidant import settings -from confidant.models.session_cls import DDBSession -from confidant.models.connection_cls import DDBConnection from confidant.services import keymanager from confidant.services.ciphermanager import CipherManager @@ -64,8 +62,9 @@ class Meta: if settings.DYNAMODB_URL: host = settings.DYNAMODB_URL region = settings.AWS_DEFAULT_REGION - connection_cls = DDBConnection - session_cls = DDBSession + connect_timeout_seconds = settings.PYNAMO_CONNECT_TIMEOUT_SECONDS + read_timeout_seconds = settings.PYNAMO_READ_TIMEOUT_SECONDS + max_pool_connection = settings.PYNAMO_CONNECTION_POOL_SIZE data_type_date_index = DataTypeDateIndex() @@ -237,8 +236,9 @@ class Meta: if settings.DYNAMODB_URL: host = settings.DYNAMODB_URL region = settings.AWS_DEFAULT_REGION - connection_cls = DDBConnection - session_cls = DDBSession + connect_timeout_seconds = settings.PYNAMO_CONNECT_TIMEOUT_SECONDS + read_timeout_seconds = settings.PYNAMO_READ_TIMEOUT_SECONDS + max_pool_connection = settings.PYNAMO_CONNECTION_POOL_SIZE archive_date = UTCDateTimeAttribute(default=datetime.now) data_type_date_index = ArchiveDataTypeDateIndex() diff --git a/confidant/models/service.py b/confidant/models/service.py index 396441d0..5f4d6605 100644 --- a/confidant/models/service.py +++ b/confidant/models/service.py @@ -10,8 +10,6 @@ from pynamodb.indexes import GlobalSecondaryIndex, AllProjection from confidant import settings -from confidant.models.session_cls import DDBSession -from confidant.models.connection_cls import DDBConnection from confidant.models.non_null_unicode_set_attribute import ( NonNullUnicodeSetAttribute ) @@ -32,8 +30,9 @@ class Meta: if settings.DYNAMODB_URL: host = settings.DYNAMODB_URL region = settings.AWS_DEFAULT_REGION - connection_cls = DDBConnection - session_cls = DDBSession + connect_timeout_seconds = settings.PYNAMO_CONNECT_TIMEOUT_SECONDS + read_timeout_seconds = settings.PYNAMO_READ_TIMEOUT_SECONDS + max_pool_connection = settings.PYNAMO_CONNECTION_POOL_SIZE id = UnicodeAttribute(hash_key=True) data_type = UnicodeAttribute() diff --git a/confidant/models/session_cls.py b/confidant/models/session_cls.py deleted file mode 100644 index ebb28655..00000000 --- a/confidant/models/session_cls.py +++ /dev/null @@ -1,15 +0,0 @@ -from botocore.vendored.requests import Session as BotoRequestsSession -from botocore.vendored.requests.adapters import HTTPAdapter as BotoHTTPAdapter - -from confidant import settings - - -class DDBSession(BotoRequestsSession): - def __init__(self): - super(DDBSession, self).__init__() - self.mount('https://', BotoHTTPAdapter( - pool_maxsize=settings.PYNAMO_CONNECTION_POOL_SIZE - )) - self.mount('http://', BotoHTTPAdapter( - pool_maxsize=settings.PYNAMO_CONNECTION_POOL_SIZE - )) diff --git a/confidant/scripts/migrate_bool.py b/confidant/scripts/migrate_bool.py index ef40c28b..c9ae6d82 100644 --- a/confidant/scripts/migrate_bool.py +++ b/confidant/scripts/migrate_bool.py @@ -13,8 +13,6 @@ from pynamodb.models import Model from confidant import settings -from confidant.models.session_cls import DDBSession -from confidant.models.connection_cls import DDBConnection logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) @@ -27,8 +25,9 @@ class Meta: if settings.DYNAMODB_URL: host = settings.DYNAMODB_URL region = settings.AWS_DEFAULT_REGION - connection_cls = DDBConnection - session_cls = DDBSession + connect_timeout_seconds = settings.PYNAMO_CONNECT_TIMEOUT_SECONDS + read_timeout_seconds = settings.PYNAMO_READ_TIMEOUT_SECONDS + max_pool_connection = settings.PYNAMO_CONNECTION_POOL_SIZE id = UnicodeAttribute(hash_key=True) enabled = BooleanAttribute(default=True) diff --git a/confidant/settings.py b/confidant/settings.py index 16634ebd..83a8d9c2 100644 --- a/confidant/settings.py +++ b/confidant/settings.py @@ -342,7 +342,8 @@ def str_env(var_name, default=''): DYNAMODB_CREATE_TABLE = bool_env('DYNAMODB_CREATE_TABLE', False) # Connection pool size for PynamoDB connections to DynamoDB PYNAMO_CONNECTION_POOL_SIZE = int_env('PYNAMO_CONNECTION_POOL_SIZE', 100) -PYNAMO_REQUEST_TIMEOUT_SECONDS = int_env('PYNAMO_REQUEST_TIMEOUT_SECONDS', 1) +PYNAMO_CONNECT_TIMEOUT_SECONDS = int_env('PYNAMO_CONNECT_TIMEOUT_SECONDS', 1) +PYNAMO_READ_TIMEOUT_SECONDS = int_env('PYNAMO_READ_TIMEOUT_SECONDS', 1) # page limit size for history API endpoints listing HISTORY_PAGE_LIMIT = int_env('HISTORY_PAGE_LIMIT') if HISTORY_PAGE_LIMIT == 0: diff --git a/docs/root/configuration.md b/docs/root/configuration.md index 65914dd0..2b514890 100644 --- a/docs/root/configuration.md +++ b/docs/root/configuration.md @@ -531,7 +531,8 @@ request that could be general networking failures, attempting to avoid request pileups. If this setting is too aggressive, you can adjust it via: ``` -export PYNAMO_REQUEST_TIMEOUT_SECONDS=1 +export PYNAMO_CONNECT_TIMEOUT_SECONDS=1 +export PYNAMO_READ_TIMEOUT_SECONDS=1 ``` To avoid recreating connections to dynamodb on each request, we open a larger diff --git a/requirements.in b/requirements.in index abdfd53e..10c798eb 100644 --- a/requirements.in +++ b/requirements.in @@ -35,7 +35,7 @@ boto3==1.9.227 # License: MIT # Upstream url: http://jlafon.io/pynamodb.html # Use: For DynamoDB storage -pynamodb>=3.2.1,<4 +pynamodb==5.5.1 # cryptography is a package which provides cryptographic recipes and # primitives to Python developers. @@ -143,7 +143,7 @@ statsd # License: Apache2 # Upstream url: https://github.com/lyft/python-kmsauth # Use: kmsauth support library -kmsauth==0.6.2 +kmsauth==0.6.3 # License: Apache2 # Upstream url: https://pypi.python.org/pypi/toasted-marshmallow diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..6345bff5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,219 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# cd ~/src/confidant-private/upstream && run-piptools +# +async-timeout==4.0.2 + # via redis +attrs==23.1.0 + # via toastedmarshmallow +authomatic==1.2.1 + # via -r requirements.in +blinker==1.6.2 + # via -r requirements.in +boto3==1.9.227 + # via + # -r requirements.in + # kmsauth +botocore==1.12.227 + # via + # -r requirements.in + # boto3 + # pynamodb + # s3transfer +cachetools==5.2.0 + # via -r requirements.in +cerberus==1.3.4 + # via -r requirements.in +certifi==2023.5.7 + # via requests +cffi==1.15.1 + # via + # -r requirements.in + # cryptography +charset-normalizer==3.1.0 + # via requests +click==7.1.2 + # via flask +coverage[toml]==7.2.7 + # via pytest-cov +cryptography==41.0.1 + # via + # -r requirements.in + # jwcrypto + # pyopenssl + # types-pyopenssl + # types-redis +deprecated==1.2.14 + # via jwcrypto +docutils==0.15.2 + # via botocore +exceptiongroup==1.1.1 + # via pytest +fakeredis==2.17.0 + # via -r requirements.in +flask==1.1.4 + # via + # -r requirements.in + # flask-script + # flask-session + # flask-sslify +flask-script==2.0.5 + # via -r requirements.in +flask-session==0.2.3 + # via -r requirements.in +flask-sslify==0.1.5 + # via -r requirements.in +gevent==22.10.2 + # via + # -r requirements.in + # pytest-gevent +greenlet==2.0.2 + # via + # -r requirements.in + # gevent +guard==1.0.1 + # via -r requirements.in +gunicorn==20.1.0 + # via -r requirements.in +idna==3.4 + # via requests +iniconfig==2.0.0 + # via pytest +isodate==0.6.1 + # via python3-saml +itsdangerous==1.1.0 + # via flask +jinja2==2.11.3 + # via flask +jmespath==0.10.0 + # via + # boto3 + # botocore +jwcrypto==1.5.0 + # via -r requirements.in +kmsauth==0.6.3 + # via -r requirements.in +lru-dict==1.2.0 + # via -r requirements.in +lxml==4.9.2 + # via + # python3-saml + # xmlsec +markupsafe==2.0.1 + # via jinja2 +mypy==1.4.1 + # via -r requirements.in +mypy-extensions==1.0.0 + # via mypy +ndg-httpsclient==0.5.1 + # via -r requirements.in +packaging==23.1 + # via pytest +pluggy==1.2.0 + # via pytest +pyasn1==0.5.0 + # via + # -r requirements.in + # ndg-httpsclient +pycparser==2.21 + # via cffi +pyjwt==2.7.0 + # via -r requirements.in +pynamodb==5.5.1 + # via -r requirements.in +pyopenssl==23.2.0 + # via + # -r requirements.in + # ndg-httpsclient +pytest==7.4.0 + # via + # -r requirements.in + # pytest-cov + # pytest-env + # pytest-gevent + # pytest-lazy-fixture + # pytest-mock +pytest-cov==4.1.0 + # via -r requirements.in +pytest-env==0.8.2 + # via -r requirements.in +pytest-gevent==1.1.0 + # via -r requirements.in +pytest-lazy-fixture==0.6.3 + # via -r requirements.in +pytest-mock==3.11.1 + # via -r requirements.in +python-dateutil==2.8.2 + # via botocore +python-json-logger==2.0.7 + # via -r requirements.in +python3-saml==1.15.0 + # via -r requirements.in +pytz==2023.3 + # via -r requirements.in +pyyaml==6.0 + # via -r requirements.in +redis==4.6.0 + # via + # -r requirements.in + # fakeredis +requests==2.31.0 + # via -r requirements.in +s3transfer==0.2.1 + # via boto3 +six==1.16.0 + # via + # isodate + # python-dateutil +sortedcontainers==2.4.0 + # via fakeredis +statsd==4.0.1 + # via -r requirements.in +toastedmarshmallow==2.15.2.post1 + # via -r requirements.in +tomli==2.0.1 + # via + # coverage + # mypy + # pytest +types-pyopenssl==23.2.0.1 + # via types-redis +types-pytz==2023.3.0.0 + # via -r requirements.in +types-pyyaml==6.0.12.10 + # via -r requirements.in +types-redis==4.6.0.0 + # via -r requirements.in +types-requests==2.31.0.1 + # via -r requirements.in +types-urllib3==1.26.25.13 + # via types-requests +typing-extensions==4.7.0 + # via mypy +urllib3==1.25.11 + # via + # botocore + # requests +werkzeug==1.0.1 + # via flask +wrapt==1.15.0 + # via deprecated +xmlsec==1.3.13 + # via python3-saml +zope-event==5.0 + # via gevent +zope-interface==6.0 + # via gevent + +pip==23.1.2 + # via -r piptools_requirements.txt +setuptools==68.0.0 + # via + # -r piptools_requirements.txt + # cerberus + # gevent + # gunicorn + # zope-event + # zope-interface diff --git a/requirements3.txt b/requirements3.txt index 8f924b4c..bbbddd58 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -121,7 +121,7 @@ pycparser==2.21 # via cffi pyjwt==2.7.0 # via -r requirements.in -pynamodb==3.4.1 +pynamodb==5.5.1 # via -r requirements.in pyopenssl==23.2.0 # via @@ -146,9 +146,7 @@ pytest-lazy-fixture==0.6.3 pytest-mock==3.11.1 # via -r requirements.in python-dateutil==2.8.2 - # via - # botocore - # pynamodb + # via botocore python-json-logger==2.0.7 # via -r requirements.in python3-saml==1.15.0 @@ -168,7 +166,6 @@ s3transfer==0.2.1 six==1.16.0 # via # isodate - # pynamodb # python-dateutil sortedcontainers==2.4.0 # via fakeredis diff --git a/tests/unit/confidant/scripts/archive_test.py b/tests/unit/confidant/scripts/archive_test.py index 3d3d708f..23cb7ed5 100644 --- a/tests/unit/confidant/scripts/archive_test.py +++ b/tests/unit/confidant/scripts/archive_test.py @@ -43,17 +43,6 @@ def credentials( mocker: MockerFixture, now: datetime ) -> Dict[str, List[Credential]]: - gmd_mock = mocker.Mock(return_value='test') - gmd_mock.range_keyname = 'test' - mocker.patch( - 'confidant.services.credentialmanager.Credential._get_meta_data', - return_value=gmd_mock, - ) - mocker.patch( - 'confidant.services.credentialmanager.CredentialArchive.' - '_get_meta_data', - return_value=gmd_mock, - ) credential = Credential( id='1234', name='test', diff --git a/tests/unit/confidant/scripts/restore_test.py b/tests/unit/confidant/scripts/restore_test.py index 1208ba19..40dffcbe 100644 --- a/tests/unit/confidant/scripts/restore_test.py +++ b/tests/unit/confidant/scripts/restore_test.py @@ -33,16 +33,6 @@ def credentials( mocker: MockerFixture, now: datetime ) -> Dict[str, List[Credential]]: - gmd_mock = mocker.Mock(return_value='test') - gmd_mock.range_keyname = 'test' - mocker.patch( - 'confidant.scripts.restore.Credential._get_meta_data', - return_value=gmd_mock, - ) - mocker.patch( - 'confidant.scripts.restore.CredentialArchive._get_meta_data', - return_value=gmd_mock, - ) archive_credential = CredentialArchive( id='1234', name='test',