From 0a8039abd291c2353f750653879d21bfbf341585 Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Sat, 14 Jan 2023 13:57:08 +0530 Subject: [PATCH 01/18] Added support to default endpoint protocol and blob endpoint --- AUTHORS.rst | 1 + storefact/_store_creation.py | 12 ++++++++---- storefact/_urls.py | 6 ++++++ tests/test_urls.py | 14 ++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index dfb4b45..4022e1c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -4,3 +4,4 @@ Authors * Felix Marczinowski - felix.marczinowski@blue-yonder.com * Cornelius Riemenschneider - cornelius.riemenschneider@blue-yonder.com +* Aditya Jaroli - aditya.jaroli@blueyonder.com diff --git a/storefact/_store_creation.py b/storefact/_store_creation.py index b7211aa..0e038ef 100644 --- a/storefact/_store_creation.py +++ b/storefact/_store_creation.py @@ -111,9 +111,13 @@ def _build_azure_url( use_sas=False, **kwargs): protocol = default_endpoints_protocol or 'https' if use_sas: - return ('DefaultEndpointsProtocol={protocol};AccountName={account_name};' - 'SharedAccessSignature={shared_access_signature}'.format( - protocol=protocol, account_name=account_name, shared_access_signature=account_key)) + conn_string = ('DefaultEndpointsProtocol={protocol};AccountName={account_name};' + 'SharedAccessSignature={shared_access_signature}'.format( + protocol=protocol, account_name=account_name, shared_access_signature=account_key)) else: - return 'DefaultEndpointsProtocol={protocol};AccountName={account_name};AccountKey={account_key}'.format( + conn_string = 'DefaultEndpointsProtocol={protocol};AccountName={account_name};AccountKey={account_key}'.format( protocol=protocol, account_name=account_name, account_key=account_key) + + if blob_endpoint: + conn_string += ';BlobEndpoint={blob_endpoint}'.format(blob_endpoint=blob_endpoint) + return conn_string diff --git a/storefact/_urls.py b/storefact/_urls.py index 2b4b6c2..8a2375a 100644 --- a/storefact/_urls.py +++ b/storefact/_urls.py @@ -44,6 +44,12 @@ def url2dict(url, raise_on_extra_params=False): if u'create_if_missing' in parsed['query']: create_if_missing = parsed['query'].pop(u'create_if_missing')[-1] # use last appearance of key params['create_if_missing'] = create_if_missing in TRUEVALUES + if u'blob_endpoint' in parsed['query']: + blob_endpoint = parsed['query'].pop(u'blob_endpoint')[-1] + params['blob_endpoint'] = blob_endpoint + if u'default_endpoints_protocol' in parsed['query']: + default_endpoints_protocol = parsed['query'].pop(u'default_endpoints_protocol')[-1] + params['default_endpoints_protocol'] = default_endpoints_protocol # get store-specific parameters: store_params = extract_params(**parsed) diff --git a/tests/test_urls.py b/tests/test_urls.py index 78d916e..b2b773c 100644 --- a/tests/test_urls.py +++ b/tests/test_urls.py @@ -67,5 +67,19 @@ def test_bad_url2dict(url, raises): storefact.url2dict(url) +def test_url2dict_with_protocol_and_point_query_params(): + url = u'azure://MYACCOUNT:dead%2Fbeef@1buc-ket1?create_if_missing=true&default_endpoints_protocol=http&blob_endpoint=http://network:888/devAcc' + expected = dict( + type='azure', + account_name='MYACCOUNT', + account_key='dead/beef', + container='1buc-ket1', + create_if_missing=True, + default_endpoints_protocol="http", + blob_endpoint="http://network:888/devAcc" + ) + assert storefact.url2dict(url) == expected + + def test_roundtrip(): assert isinstance(storefact.get_store_from_url(u'memory://#wrap:readonly'), simplekv.decorator.ReadOnlyDecorator) From 6d8a9b195894d47386c626bd1a436f41c162ddb7 Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 10:20:39 +0530 Subject: [PATCH 02/18] Added new environment support, supporting test cases and updated readme --- .github/workflows/test_execution_py27.yml | 21 ++++++++++++++++++ README.rst | 2 +- setup.py | 1 + tests/test_store_creation.py | 26 +++++++++++++++++++++++ tests/test_urls.py | 8 +++++++ tox.ini | 2 +- 6 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test_execution_py27.yml diff --git a/.github/workflows/test_execution_py27.yml b/.github/workflows/test_execution_py27.yml new file mode 100644 index 0000000..4cad4ce --- /dev/null +++ b/.github/workflows/test_execution_py27.yml @@ -0,0 +1,21 @@ +--- +name: Run unit and component test cases + +on: + push: + branches: + - "**" + +jobs: + tests: + name: "Test Execution" + runs-on: "ubuntu-20.04" + steps: + - name: "Set up Python 2.7" + uses: "actions/setup-python@v3" + with: + python-version: "2.7" + cache: 'pip' + - name: "Install dependencies" + run: | + tox diff --git a/README.rst b/README.rst index 01cc683..e932722 100644 --- a/README.rst +++ b/README.rst @@ -72,7 +72,7 @@ URL and store types: * Azure Blob Storage (:code:`azure://` and :code:`hazure://`): * with storage account key: :code:`azure://account_name:account_key@container[?create_if_missing=true][?max_connections=2]` * with SAS token: :code:`azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)]` - * with SAS and additional parameters: :code:`azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)][?max_block_size=4*1024*1024&max_single_put_size=64*1024*1024]` + * with SAS and additional parameters: :code:`azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)][?max_block_size=4*1024*1024&max_single_put_size=64*1024*1024][?default_endpoints_protocol=http&blob_endpoint=http://localhost:2121]` Storage URLs starting with a :code:`h` indicate extended allowed characters. This allows the usage of slashes and spaces in blob names. URL options with :code:`[]` are optional and the :code:`[]` need to be removed. diff --git a/setup.py b/setup.py index b49c117..f2bf0d8 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ def read(*names, **kwargs): "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Utilities", ], diff --git a/tests/test_store_creation.py b/tests/test_store_creation.py index 8ec5afe..76366bb 100644 --- a/tests/test_store_creation.py +++ b/tests/test_store_creation.py @@ -54,6 +54,32 @@ def test_create_store_hazure(mocker): ) +def test_create_store_hazure_with_protocol_and_blob_endpoint(mocker): + mock_hazure = mocker.patch("storefact._hstores.HAzureBlockBlobStore") + create_store( + "hazure", + { + "account_name": "ACCOUNT", + "account_key": "KEY", + "container": "cont_name", + "create_if_missing": True, + "blob_endpoint": "http://local:2121", + "default_endpoints_protocol": "http" + }, + ) + mock_hazure.assert_called_once_with( + checksum=True, + conn_string="DefaultEndpointsProtocol=http;AccountName=ACCOUNT;AccountKey=KEY;BlobEndpoint=http://local:2121", + container="cont_name", + create_if_missing=True, + max_connections=2, + public=False, + socket_timeout=(20, 100), + max_block_size=4194304, + max_single_put_size=67108864, + ) + + def test_create_store_azure_inconsistent_params(): with pytest.raises( Exception, match="create_if_missing is incompatible with the use of SAS tokens" diff --git a/tests/test_urls.py b/tests/test_urls.py index b2b773c..bbf1436 100644 --- a/tests/test_urls.py +++ b/tests/test_urls.py @@ -5,6 +5,14 @@ import simplekv.decorator good_urls = [ + (u'azure://MYACCOUNT:dead%2Fbeef@1buc-ket1?param1=foo&default_endpoints_protocol=http&blob_endpoint=http://host:port&create_if_missing=true', dict( + type='azure', + account_name='MYACCOUNT', + account_key='dead/beef', + container='1buc-ket1', + create_if_missing=True, + default_endpoints_protocol='http', + blob_endpoint='http://host:port')), (u'azure://MYACCOUNT:dead%2Fbeef@1buc-ket1?param1=foo¶m2=🍺&eat_more_🍎=1&create_if_missing=true', dict( type='azure', account_name='MYACCOUNT', diff --git a/tox.ini b/tox.ini index 0444dd5..ea21766 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,35,36} +envlist = py{27,35,36,39} [testenv] deps = From cffebd7fd34c570e7693f89554e2176a5db8ed87 Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 10:23:43 +0530 Subject: [PATCH 03/18] Added command to checkout repo --- .github/workflows/test_execution_py27.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_execution_py27.yml b/.github/workflows/test_execution_py27.yml index 4cad4ce..72944c6 100644 --- a/.github/workflows/test_execution_py27.yml +++ b/.github/workflows/test_execution_py27.yml @@ -11,6 +11,11 @@ jobs: name: "Test Execution" runs-on: "ubuntu-20.04" steps: + - name: "SETUP - Checkout ${{ github.repository }}" + uses: "actions/checkout@v2" + with: + ref: "${{ github.event.pull_request.head.sha }}" + fetch-depth: 0 - name: "Set up Python 2.7" uses: "actions/setup-python@v3" with: From 0dd229c31d44f73f058fc94ab07924d777a4b48b Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 10:55:08 +0530 Subject: [PATCH 04/18] Updated the workflow file --- .github/workflows/test_execution_py27.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_execution_py27.yml b/.github/workflows/test_execution_py27.yml index 72944c6..5360358 100644 --- a/.github/workflows/test_execution_py27.yml +++ b/.github/workflows/test_execution_py27.yml @@ -23,4 +23,8 @@ jobs: cache: 'pip' - name: "Install dependencies" run: | - tox + python -m pip install --upgrade pip + pip install setuptools_scm wheel + pip install -r requirements.in + pip install tox + tox -e 27 From 3e475edc373316d0a817d6efbe538a251ce31b06 Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 10:56:52 +0530 Subject: [PATCH 05/18] Updated the workflow file --- .github/workflows/test_execution_py27.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_execution_py27.yml b/.github/workflows/test_execution_py27.yml index 5360358..d436d66 100644 --- a/.github/workflows/test_execution_py27.yml +++ b/.github/workflows/test_execution_py27.yml @@ -25,6 +25,6 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools_scm wheel - pip install -r requirements.in + pip install -r requirements.txt pip install tox tox -e 27 From eb10a944e8d975bf199c5b8c5495c4f26061b99b Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 10:59:26 +0530 Subject: [PATCH 06/18] Updated the workflow file --- .github/workflows/test_execution_py27.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_execution_py27.yml b/.github/workflows/test_execution_py27.yml index d436d66..034a36b 100644 --- a/.github/workflows/test_execution_py27.yml +++ b/.github/workflows/test_execution_py27.yml @@ -27,4 +27,4 @@ jobs: pip install setuptools_scm wheel pip install -r requirements.txt pip install tox - tox -e 27 + tox From 931b330649cca3a673cc0dba28c362498fc4ac9a Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 12:34:04 +0530 Subject: [PATCH 07/18] Mentioned the correct python 2.7 version name --- .github/workflows/test_execution_py27.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_execution_py27.yml b/.github/workflows/test_execution_py27.yml index 034a36b..614e17d 100644 --- a/.github/workflows/test_execution_py27.yml +++ b/.github/workflows/test_execution_py27.yml @@ -27,4 +27,4 @@ jobs: pip install setuptools_scm wheel pip install -r requirements.txt pip install tox - tox + tox -e py27 From 1f66fc4937347436d11dba5715ead1dc233aaa3a Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 12:37:33 +0530 Subject: [PATCH 08/18] Added test github workflow for different python envs --- .github/workflows/test_execution_py27.yml | 2 +- .github/workflows/test_execution_py35.yml | 30 +++++++++++++++++++++++ .github/workflows/test_execution_py36.yml | 30 +++++++++++++++++++++++ .github/workflows/test_execution_py39.yml | 30 +++++++++++++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test_execution_py35.yml create mode 100644 .github/workflows/test_execution_py36.yml create mode 100644 .github/workflows/test_execution_py39.yml diff --git a/.github/workflows/test_execution_py27.yml b/.github/workflows/test_execution_py27.yml index 614e17d..f984513 100644 --- a/.github/workflows/test_execution_py27.yml +++ b/.github/workflows/test_execution_py27.yml @@ -21,7 +21,7 @@ jobs: with: python-version: "2.7" cache: 'pip' - - name: "Install dependencies" + - name: "Install dependencies and Run tests" run: | python -m pip install --upgrade pip pip install setuptools_scm wheel diff --git a/.github/workflows/test_execution_py35.yml b/.github/workflows/test_execution_py35.yml new file mode 100644 index 0000000..48c5f00 --- /dev/null +++ b/.github/workflows/test_execution_py35.yml @@ -0,0 +1,30 @@ +--- +name: Run unit and component test cases + +on: + push: + branches: + - "**" + +jobs: + tests: + name: "Test Execution" + runs-on: "ubuntu-20.04" + steps: + - name: "SETUP - Checkout ${{ github.repository }}" + uses: "actions/checkout@v2" + with: + ref: "${{ github.event.pull_request.head.sha }}" + fetch-depth: 0 + - name: "Set up Python 3.5" + uses: "actions/setup-python@v3" + with: + python-version: "3.5" + cache: 'pip' + - name: "Install dependencies and Run tests" + run: | + python -m pip install --upgrade pip + pip install setuptools_scm wheel + pip install -r requirements.txt + pip install tox + tox -e 35 diff --git a/.github/workflows/test_execution_py36.yml b/.github/workflows/test_execution_py36.yml new file mode 100644 index 0000000..7da2682 --- /dev/null +++ b/.github/workflows/test_execution_py36.yml @@ -0,0 +1,30 @@ +--- +name: Run unit and component test cases + +on: + push: + branches: + - "**" + +jobs: + tests: + name: "Test Execution" + runs-on: "ubuntu-20.04" + steps: + - name: "SETUP - Checkout ${{ github.repository }}" + uses: "actions/checkout@v2" + with: + ref: "${{ github.event.pull_request.head.sha }}" + fetch-depth: 0 + - name: "Set up Python 3.6" + uses: "actions/setup-python@v3" + with: + python-version: "3.6" + cache: 'pip' + - name: "Install dependencies and Run tests" + run: | + python -m pip install --upgrade pip + pip install setuptools_scm wheel + pip install -r requirements.txt + pip install tox + tox -e 36 diff --git a/.github/workflows/test_execution_py39.yml b/.github/workflows/test_execution_py39.yml new file mode 100644 index 0000000..3870b1c --- /dev/null +++ b/.github/workflows/test_execution_py39.yml @@ -0,0 +1,30 @@ +--- +name: Run unit and component test cases + +on: + push: + branches: + - "**" + +jobs: + tests: + name: "Test Execution" + runs-on: "ubuntu-20.04" + steps: + - name: "SETUP - Checkout ${{ github.repository }}" + uses: "actions/checkout@v2" + with: + ref: "${{ github.event.pull_request.head.sha }}" + fetch-depth: 0 + - name: "Set up Python 3.9" + uses: "actions/setup-python@v3" + with: + python-version: "3.9" + cache: 'pip' + - name: "Install dependencies and Run tests" + run: | + python -m pip install --upgrade pip + pip install setuptools_scm wheel + pip install -r requirements.txt + pip install tox + tox -e 39 From 593203469aa147134973a736ea0a27d28337206f Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 12:40:29 +0530 Subject: [PATCH 09/18] Updated the workflow files --- .github/workflows/test_execution_py35.yml | 2 +- .github/workflows/test_execution_py36.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_execution_py35.yml b/.github/workflows/test_execution_py35.yml index 48c5f00..8963013 100644 --- a/.github/workflows/test_execution_py35.yml +++ b/.github/workflows/test_execution_py35.yml @@ -27,4 +27,4 @@ jobs: pip install setuptools_scm wheel pip install -r requirements.txt pip install tox - tox -e 35 + tox -e py35 diff --git a/.github/workflows/test_execution_py36.yml b/.github/workflows/test_execution_py36.yml index 7da2682..aa9c5c8 100644 --- a/.github/workflows/test_execution_py36.yml +++ b/.github/workflows/test_execution_py36.yml @@ -27,4 +27,4 @@ jobs: pip install setuptools_scm wheel pip install -r requirements.txt pip install tox - tox -e 36 + tox -e py36 From c08bcf1cd490dee0cb0bc6aeaaefb9a1515eff07 Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 12:48:57 +0530 Subject: [PATCH 10/18] Updated the workflow title --- .github/workflows/test_execution_py27.yml | 2 +- .github/workflows/test_execution_py35.yml | 2 +- .github/workflows/test_execution_py36.yml | 2 +- .github/workflows/test_execution_py39.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_execution_py27.yml b/.github/workflows/test_execution_py27.yml index f984513..31023fd 100644 --- a/.github/workflows/test_execution_py27.yml +++ b/.github/workflows/test_execution_py27.yml @@ -1,5 +1,5 @@ --- -name: Run unit and component test cases +name: Run unit tests for python 2.7 on: push: diff --git a/.github/workflows/test_execution_py35.yml b/.github/workflows/test_execution_py35.yml index 8963013..cf71557 100644 --- a/.github/workflows/test_execution_py35.yml +++ b/.github/workflows/test_execution_py35.yml @@ -1,5 +1,5 @@ --- -name: Run unit and component test cases +name: Run unit tests for python 3.5 on: push: diff --git a/.github/workflows/test_execution_py36.yml b/.github/workflows/test_execution_py36.yml index aa9c5c8..7c03f54 100644 --- a/.github/workflows/test_execution_py36.yml +++ b/.github/workflows/test_execution_py36.yml @@ -1,5 +1,5 @@ --- -name: Run unit and component test cases +name: Run unit tests for python 3.6 on: push: diff --git a/.github/workflows/test_execution_py39.yml b/.github/workflows/test_execution_py39.yml index 3870b1c..04e6fe5 100644 --- a/.github/workflows/test_execution_py39.yml +++ b/.github/workflows/test_execution_py39.yml @@ -1,5 +1,5 @@ --- -name: Run unit and component test cases +name: Run unit tests for python 3.9 on: push: From ee3e5fdc086b50ef53ffb21581c5b29eed0263c8 Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 14:40:20 +0530 Subject: [PATCH 11/18] Added github workflows --- .github/workflows/publish.yml | 26 +++++++++++++++++++++++ .github/workflows/unit_test_execution.yml | 26 +++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/unit_test_execution.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..c93673b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,26 @@ +name: Upload Release ot PyPI + +on: + release: + types: [published] + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.STOREFACT_PYPI }} \ No newline at end of file diff --git a/.github/workflows/unit_test_execution.yml b/.github/workflows/unit_test_execution.yml new file mode 100644 index 0000000..35998b8 --- /dev/null +++ b/.github/workflows/unit_test_execution.yml @@ -0,0 +1,26 @@ +name: Python package + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python: ["2.7", "3.5", "3.6", "3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: Install tox and any other packages + run: | + python -m pip install --upgrade pip + pip install setuptools_scm wheel + pip install -r requirements.txt + pip install tox + - name: Run tox + run: tox \ No newline at end of file From ffcecd164ca6ac3e1c9452a57c24754846c28e8c Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 14:45:42 +0530 Subject: [PATCH 12/18] Updated the github workflow --- .github/workflows/unit_test_execution.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/unit_test_execution.yml b/.github/workflows/unit_test_execution.yml index 35998b8..2f09ef1 100644 --- a/.github/workflows/unit_test_execution.yml +++ b/.github/workflows/unit_test_execution.yml @@ -4,23 +4,22 @@ on: [push] jobs: build: - - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: - python: ["2.7", "3.5", "3.6", "3.9"] - + python-version: ["2.7", "3.5", "3.6", "3.9"] + name: Python ${{ matrix.python-version }} steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 + - uses: actions/checkout@v2 + - name: Setup python + uses: actions/setup-python@v1 with: - python-version: ${{ matrix.python }} - - name: Install tox and any other packages - run: | + python-version: ${{ matrix.python-version }} + architecture: x64 + - run: | python -m pip install --upgrade pip pip install setuptools_scm wheel pip install -r requirements.txt pip install tox - name: Run tox - run: tox \ No newline at end of file + run: tox \ No newline at end of file From d4c6ed3d8d5851e69ad6e81115aa8fc5bf343518 Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 14:46:27 +0530 Subject: [PATCH 13/18] Updated the github workflow --- .github/workflows/unit_test_execution.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test_execution.yml b/.github/workflows/unit_test_execution.yml index 2f09ef1..3d0b708 100644 --- a/.github/workflows/unit_test_execution.yml +++ b/.github/workflows/unit_test_execution.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Setup python - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} architecture: x64 From 9dd9b8d817e1d1c08cfdd6e9398c8d12ab337f51 Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 14:47:16 +0530 Subject: [PATCH 14/18] Updated the github workflow --- .github/workflows/unit_test_execution.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test_execution.yml b/.github/workflows/unit_test_execution.yml index 3d0b708..65c51bf 100644 --- a/.github/workflows/unit_test_execution.yml +++ b/.github/workflows/unit_test_execution.yml @@ -22,4 +22,4 @@ jobs: pip install -r requirements.txt pip install tox - name: Run tox - run: tox \ No newline at end of file + run: tox \ No newline at end of file From 38c9e05fa8024e610d8db00f577b1685d20042cd Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 14:50:36 +0530 Subject: [PATCH 15/18] Updated the github workflow --- .github/workflows/unit_test_execution.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test_execution.yml b/.github/workflows/unit_test_execution.yml index 65c51bf..32e6122 100644 --- a/.github/workflows/unit_test_execution.yml +++ b/.github/workflows/unit_test_execution.yml @@ -22,4 +22,4 @@ jobs: pip install -r requirements.txt pip install tox - name: Run tox - run: tox \ No newline at end of file + run: tox -e py \ No newline at end of file From 31b019b397768602f6d73c8444b76231e8cc95ed Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 14:52:13 +0530 Subject: [PATCH 16/18] Removed individual workflow files --- .github/workflows/test_execution_py27.yml | 30 ----------------------- .github/workflows/test_execution_py35.yml | 30 ----------------------- .github/workflows/test_execution_py36.yml | 30 ----------------------- .github/workflows/test_execution_py39.yml | 30 ----------------------- .github/workflows/unit_test_execution.yml | 2 +- 5 files changed, 1 insertion(+), 121 deletions(-) delete mode 100644 .github/workflows/test_execution_py27.yml delete mode 100644 .github/workflows/test_execution_py35.yml delete mode 100644 .github/workflows/test_execution_py36.yml delete mode 100644 .github/workflows/test_execution_py39.yml diff --git a/.github/workflows/test_execution_py27.yml b/.github/workflows/test_execution_py27.yml deleted file mode 100644 index 31023fd..0000000 --- a/.github/workflows/test_execution_py27.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -name: Run unit tests for python 2.7 - -on: - push: - branches: - - "**" - -jobs: - tests: - name: "Test Execution" - runs-on: "ubuntu-20.04" - steps: - - name: "SETUP - Checkout ${{ github.repository }}" - uses: "actions/checkout@v2" - with: - ref: "${{ github.event.pull_request.head.sha }}" - fetch-depth: 0 - - name: "Set up Python 2.7" - uses: "actions/setup-python@v3" - with: - python-version: "2.7" - cache: 'pip' - - name: "Install dependencies and Run tests" - run: | - python -m pip install --upgrade pip - pip install setuptools_scm wheel - pip install -r requirements.txt - pip install tox - tox -e py27 diff --git a/.github/workflows/test_execution_py35.yml b/.github/workflows/test_execution_py35.yml deleted file mode 100644 index cf71557..0000000 --- a/.github/workflows/test_execution_py35.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -name: Run unit tests for python 3.5 - -on: - push: - branches: - - "**" - -jobs: - tests: - name: "Test Execution" - runs-on: "ubuntu-20.04" - steps: - - name: "SETUP - Checkout ${{ github.repository }}" - uses: "actions/checkout@v2" - with: - ref: "${{ github.event.pull_request.head.sha }}" - fetch-depth: 0 - - name: "Set up Python 3.5" - uses: "actions/setup-python@v3" - with: - python-version: "3.5" - cache: 'pip' - - name: "Install dependencies and Run tests" - run: | - python -m pip install --upgrade pip - pip install setuptools_scm wheel - pip install -r requirements.txt - pip install tox - tox -e py35 diff --git a/.github/workflows/test_execution_py36.yml b/.github/workflows/test_execution_py36.yml deleted file mode 100644 index 7c03f54..0000000 --- a/.github/workflows/test_execution_py36.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -name: Run unit tests for python 3.6 - -on: - push: - branches: - - "**" - -jobs: - tests: - name: "Test Execution" - runs-on: "ubuntu-20.04" - steps: - - name: "SETUP - Checkout ${{ github.repository }}" - uses: "actions/checkout@v2" - with: - ref: "${{ github.event.pull_request.head.sha }}" - fetch-depth: 0 - - name: "Set up Python 3.6" - uses: "actions/setup-python@v3" - with: - python-version: "3.6" - cache: 'pip' - - name: "Install dependencies and Run tests" - run: | - python -m pip install --upgrade pip - pip install setuptools_scm wheel - pip install -r requirements.txt - pip install tox - tox -e py36 diff --git a/.github/workflows/test_execution_py39.yml b/.github/workflows/test_execution_py39.yml deleted file mode 100644 index 04e6fe5..0000000 --- a/.github/workflows/test_execution_py39.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -name: Run unit tests for python 3.9 - -on: - push: - branches: - - "**" - -jobs: - tests: - name: "Test Execution" - runs-on: "ubuntu-20.04" - steps: - - name: "SETUP - Checkout ${{ github.repository }}" - uses: "actions/checkout@v2" - with: - ref: "${{ github.event.pull_request.head.sha }}" - fetch-depth: 0 - - name: "Set up Python 3.9" - uses: "actions/setup-python@v3" - with: - python-version: "3.9" - cache: 'pip' - - name: "Install dependencies and Run tests" - run: | - python -m pip install --upgrade pip - pip install setuptools_scm wheel - pip install -r requirements.txt - pip install tox - tox -e 39 diff --git a/.github/workflows/unit_test_execution.yml b/.github/workflows/unit_test_execution.yml index 32e6122..3c44567 100644 --- a/.github/workflows/unit_test_execution.yml +++ b/.github/workflows/unit_test_execution.yml @@ -1,4 +1,4 @@ -name: Python package +name: Unit test execution on: [push] From 5ecd56b62b433b9baccfad590de95454118082a3 Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 16:12:02 +0530 Subject: [PATCH 17/18] Implemented review comments --- storefact/_urls.py | 34 ++++++++++---- tests/test_urls.py | 115 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 9 deletions(-) diff --git a/storefact/_urls.py b/storefact/_urls.py index 8a2375a..acb127e 100644 --- a/storefact/_urls.py +++ b/storefact/_urls.py @@ -41,15 +41,9 @@ def url2dict(url, raise_on_extra_params=False): wrappers = wrap_spec[-1].partition('wrap:')[2] # remove the 'wrap:' part params['wrap'] = wrappers - if u'create_if_missing' in parsed['query']: - create_if_missing = parsed['query'].pop(u'create_if_missing')[-1] # use last appearance of key - params['create_if_missing'] = create_if_missing in TRUEVALUES - if u'blob_endpoint' in parsed['query']: - blob_endpoint = parsed['query'].pop(u'blob_endpoint')[-1] - params['blob_endpoint'] = blob_endpoint - if u'default_endpoints_protocol' in parsed['query']: - default_endpoints_protocol = parsed['query'].pop(u'default_endpoints_protocol')[-1] - params['default_endpoints_protocol'] = default_endpoints_protocol + extract_from_query_params(parsed['query'], params, "create_if_missing", is_boolean_type=True) + extract_from_query_params(parsed['query'], params, "blob_endpoint") + extract_from_query_params(parsed['query'], params, "default_endpoints_protocol") # get store-specific parameters: store_params = extract_params(**parsed) @@ -57,6 +51,28 @@ def url2dict(url, raise_on_extra_params=False): return params +def extract_from_query_params(query_params: dict, params: dict, key: str, is_boolean_type: bool = False): + """ + + Args: + query_params: dictionary to extract from + params: dictionary to extract to + key: key to look for in query_params + is_boolean_type: if expected value is in Boolean True, False then pass the value as True. + Default is False. + + Returns: + None + note: Updates the params dictionary as objects are pass by reference + + """ + if query_params and params is not None and key in query_params: + value = query_params.pop(key)[-1] + if is_boolean_type: + value = value in TRUEVALUES + params[key] = value + + def extract_params(scheme, host, port, path, query, userinfo): if scheme in ('memory', 'hmemory'): return {} diff --git a/tests/test_urls.py b/tests/test_urls.py index bbf1436..b038716 100644 --- a/tests/test_urls.py +++ b/tests/test_urls.py @@ -2,6 +2,7 @@ import pytest import storefact +from storefact._urls import extract_from_query_params import simplekv.decorator good_urls = [ @@ -89,5 +90,119 @@ def test_url2dict_with_protocol_and_point_query_params(): assert storefact.url2dict(url) == expected +def test_extract_query_param_when_query_params_is_null(): + query_params = None + params = { + 'Hello': 'test' + } + key = "test" + extract_from_query_params(query_params, params, key) + assert params is not None and len(params) == 1 + + +def test_extract_query_param_when_query_params_is_empty(): + query_params = {} + params = { + 'Hello': 'test' + } + key = "test" + extract_from_query_params(query_params, params, key) + assert params is not None and len(params) == 1 + + +def test_extract_query_param_when_params_is_null(): + query_params = { + 'create_if_missing': ['true'] + } + params = None + key = "test" + extract_from_query_params(query_params, params, key) + assert params is None + + +def test_extract_query_param_when_params_is_empty(): + query_params = { + 'create_if_missing': ['true'] + } + params = {} + key = "create_if_missing" + extract_from_query_params(query_params, params, key, is_boolean_type=True) + assert params is not None and len(params) == 1 + assert params[key] is True + + +def test_extract_query_param_when_key_is_null(): + query_params = { + 'create_if_missing': ['true'] + } + params = {} + key = None + extract_from_query_params(query_params, params, key) + assert params is not None and len(params) == 0 + + +def test_extract_query_param_when_key_is_empty_string(): + query_params = { + 'create_if_missing': ['true'] + } + params = {} + key = '' + extract_from_query_params(query_params, params, key) + assert params is not None and len(params) == 0 + + +def test_extract_query_param_when_value_is_not_boolean(): + query_params = { + 'create_if_missing': ['true'] + } + params = {} + key = "create_if_missing" + extract_from_query_params(query_params, params, key, is_boolean_type=False) + assert params is not None and len(params) == 1 + assert params[key] == 'true' + + +def test_extract_query_param_when_key_is_not_found_in_query_params(): + query_params = { + 'create_if_missing': ['true'] + } + params = {} + key = "key_not_found" + extract_from_query_params(query_params, params, key) + assert params is not None and len(params) == 0 + + +def test_extract_query_param_when_key_is_extracted(): + query_params = { + 'create_if_missing': ['true'], + 'deploy': ['now'] + } + + params = { + 'use_sas': True + } + key = "deploy" + extract_from_query_params(query_params, params, key) + assert params is not None and len(params) == 2 + assert params['use_sas'] is True + assert params['deploy'] == 'now' + + +def test_extract_query_param_when_key_already_exit(): + query_params = { + 'create_if_missing': ['true'], + 'deploy': ['now'] + } + + params = { + 'deploy': 'later' + } + + key = "deploy" + extract_from_query_params(query_params, params, key) + assert params is not None and len(params) == 1 + assert params['deploy'] == 'now' + + def test_roundtrip(): assert isinstance(storefact.get_store_from_url(u'memory://#wrap:readonly'), simplekv.decorator.ReadOnlyDecorator) From 64f2b6259ba4f5e2b76dde7d447e92f5d1e2055a Mon Sep 17 00:00:00 2001 From: aditya-jaroli-by Date: Tue, 24 Jan 2023 16:16:26 +0530 Subject: [PATCH 18/18] Fixed code for python 2.7 env --- storefact/_urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storefact/_urls.py b/storefact/_urls.py index acb127e..92d7a04 100644 --- a/storefact/_urls.py +++ b/storefact/_urls.py @@ -51,7 +51,7 @@ def url2dict(url, raise_on_extra_params=False): return params -def extract_from_query_params(query_params: dict, params: dict, key: str, is_boolean_type: bool = False): +def extract_from_query_params(query_params, params, key, is_boolean_type=False): """ Args: