Merge pull request #23 from Blue-Yonder-OSS/new-query-params-support_…

…master

urls don't have option to pass default protocol and blob endpoint url

.github/workflows/publish.yml

@@ -0,0 +1,26 @@
+name: Upload Release ot PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.STOREFACT_PYPI }}

.github/workflows/unit_test_execution.yml

@@ -0,0 +1,25 @@
+name: Unit test execution
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ["2.7", "3.5", "3.6", "3.9"]
+    name: Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install setuptools_scm wheel
+          pip install -r requirements.txt
+          pip install tox
+      - name: Run tox
+        run: tox -e py

AUTHORS.rst

@@ -4,3 +4,4 @@ Authors
 
 * Felix Marczinowski - [email protected]
 * Cornelius Riemenschneider - [email protected]
+* Aditya Jaroli - [email protected]

README.rst

@@ -72,7 +72,7 @@ URL and store types:
 * Azure Blob Storage (:code:`azure://` and :code:`hazure://`):
     * with storage account key: :code:`azure://account_name:account_key@container[?create_if_missing=true][?max_connections=2]`
     * with SAS token: :code:`azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)]`
-    * with SAS and additional parameters: :code:`azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)][?max_block_size=4*1024*1024&max_single_put_size=64*1024*1024]`
+    * with SAS and additional parameters: :code:`azure://account_name:shared_access_signature@container?use_sas&create_if_missing=false[?max_connections=2&socket_timeout=(20,100)][?max_block_size=4*1024*1024&max_single_put_size=64*1024*1024][?default_endpoints_protocol=http&blob_endpoint=http://localhost:2121]`
 
 Storage URLs starting with a :code:`h` indicate extended allowed characters. This allows the usage of slashes and spaces in blob names.
 URL options with :code:`[]` are optional and the :code:`[]` need to be removed.

setup.py

@@ -37,6 +37,7 @@ def read(*names, **kwargs):
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: Implementation :: CPython",
         "Topic :: Utilities",
     ],

storefact/_store_creation.py

@@ -111,9 +111,13 @@ def _build_azure_url(
         use_sas=False, **kwargs):
     protocol = default_endpoints_protocol or 'https'
     if use_sas:
-        return ('DefaultEndpointsProtocol={protocol};AccountName={account_name};'
-                'SharedAccessSignature={shared_access_signature}'.format(
-                    protocol=protocol, account_name=account_name, shared_access_signature=account_key))
+        conn_string = ('DefaultEndpointsProtocol={protocol};AccountName={account_name};'
+                       'SharedAccessSignature={shared_access_signature}'.format(
+            protocol=protocol, account_name=account_name, shared_access_signature=account_key))
     else:
-        return 'DefaultEndpointsProtocol={protocol};AccountName={account_name};AccountKey={account_key}'.format(
+        conn_string = 'DefaultEndpointsProtocol={protocol};AccountName={account_name};AccountKey={account_key}'.format(
             protocol=protocol, account_name=account_name, account_key=account_key)
+
+    if blob_endpoint:
+        conn_string += ';BlobEndpoint={blob_endpoint}'.format(blob_endpoint=blob_endpoint)
+    return conn_string

storefact/_urls.py

@@ -41,16 +41,38 @@ def url2dict(url, raise_on_extra_params=False):
         wrappers = wrap_spec[-1].partition('wrap:')[2]  # remove the 'wrap:' part
         params['wrap'] = wrappers
 
-    if u'create_if_missing' in parsed['query']:
-        create_if_missing = parsed['query'].pop(u'create_if_missing')[-1]  # use last appearance of key
-        params['create_if_missing'] = create_if_missing in TRUEVALUES
+    extract_from_query_params(parsed['query'], params, "create_if_missing", is_boolean_type=True)
+    extract_from_query_params(parsed['query'], params, "blob_endpoint")
+    extract_from_query_params(parsed['query'], params, "default_endpoints_protocol")
 
     # get store-specific parameters:
     store_params = extract_params(**parsed)
     params.update(store_params)
     return params
 
 
+def extract_from_query_params(query_params, params, key, is_boolean_type=False):
+    """
+
+    Args:
+        query_params: dictionary to extract from
+        params: dictionary to extract to
+        key: key to look for in query_params
+        is_boolean_type: if expected value is in Boolean True, False then pass the value as True.
+        Default is False.
+
+    Returns:
+        None
+    note: Updates the params dictionary as objects are pass by reference
+
+    """
+    if query_params and params is not None and key in query_params:
+        value = query_params.pop(key)[-1]
+        if is_boolean_type:
+            value = value in TRUEVALUES
+        params[key] = value
+
+
 def extract_params(scheme, host, port, path, query, userinfo):
     if scheme in ('memory', 'hmemory'):
         return {}

tests/test_store_creation.py

@@ -54,6 +54,32 @@ def test_create_store_hazure(mocker):
     )
 
 
+def test_create_store_hazure_with_protocol_and_blob_endpoint(mocker):
+    mock_hazure = mocker.patch("storefact._hstores.HAzureBlockBlobStore")
+    create_store(
+        "hazure",
+        {
+            "account_name": "ACCOUNT",
+            "account_key": "KEY",
+            "container": "cont_name",
+            "create_if_missing": True,
+            "blob_endpoint": "http://local:2121",
+            "default_endpoints_protocol": "http"
+        },
+    )
+    mock_hazure.assert_called_once_with(
+        checksum=True,
+        conn_string="DefaultEndpointsProtocol=http;AccountName=ACCOUNT;AccountKey=KEY;BlobEndpoint=http://local:2121",
+        container="cont_name",
+        create_if_missing=True,
+        max_connections=2,
+        public=False,
+        socket_timeout=(20, 100),
+        max_block_size=4194304,
+        max_single_put_size=67108864,
+    )
+
+
 def test_create_store_azure_inconsistent_params():
     with pytest.raises(
             Exception, match="create_if_missing is incompatible with the use of SAS tokens"

tests/test_urls.py

@@ -2,9 +2,18 @@
 
 import pytest
 import storefact
+from storefact._urls import extract_from_query_params
 import simplekv.decorator
 
 good_urls = [
+    (u'azure://MYACCOUNT:dead%2Fbeef@1buc-ket1?param1=foo&default_endpoints_protocol=http&blob_endpoint=http://host:port&create_if_missing=true', dict(
+        type='azure',
+        account_name='MYACCOUNT',
+        account_key='dead/beef',
+        container='1buc-ket1',
+        create_if_missing=True,
+        default_endpoints_protocol='http',
+        blob_endpoint='http://host:port')),
     (u'azure://MYACCOUNT:dead%2Fbeef@1buc-ket1?param1=foo&param2=🍺&eat_more_🍎=1&create_if_missing=true', dict(
         type='azure',
         account_name='MYACCOUNT',
@@ -67,5 +76,133 @@ def test_bad_url2dict(url, raises):
         storefact.url2dict(url)
 
 
+def test_url2dict_with_protocol_and_point_query_params():
+    url = u'azure://MYACCOUNT:dead%2Fbeef@1buc-ket1?create_if_missing=true&default_endpoints_protocol=http&blob_endpoint=http://network:888/devAcc'
+    expected = dict(
+        type='azure',
+        account_name='MYACCOUNT',
+        account_key='dead/beef',
+        container='1buc-ket1',
+        create_if_missing=True,
+        default_endpoints_protocol="http",
+        blob_endpoint="http://network:888/devAcc"
+    )
+    assert storefact.url2dict(url) == expected
+
+
+def test_extract_query_param_when_query_params_is_null():
+    query_params = None
+    params = {
+        'Hello': 'test'
+    }
+    key = "test"
+    extract_from_query_params(query_params, params, key)
+    assert params is not None and len(params) == 1
+
+
+def test_extract_query_param_when_query_params_is_empty():
+    query_params = {}
+    params = {
+        'Hello': 'test'
+    }
+    key = "test"
+    extract_from_query_params(query_params, params, key)
+    assert params is not None and len(params) == 1
+
+
+def test_extract_query_param_when_params_is_null():
+    query_params = {
+        'create_if_missing': ['true']
+    }
+    params = None
+    key = "test"
+    extract_from_query_params(query_params, params, key)
+    assert params is None
+
+
+def test_extract_query_param_when_params_is_empty():
+    query_params = {
+        'create_if_missing': ['true']
+    }
+    params = {}
+    key = "create_if_missing"
+    extract_from_query_params(query_params, params, key, is_boolean_type=True)
+    assert params is not None and len(params) == 1
+    assert params[key] is True
+
+
+def test_extract_query_param_when_key_is_null():
+    query_params = {
+        'create_if_missing': ['true']
+    }
+    params = {}
+    key = None
+    extract_from_query_params(query_params, params, key)
+    assert params is not None and len(params) == 0
+
+
+def test_extract_query_param_when_key_is_empty_string():
+    query_params = {
+        'create_if_missing': ['true']
+    }
+    params = {}
+    key = ''
+    extract_from_query_params(query_params, params, key)
+    assert params is not None and len(params) == 0
+
+
+def test_extract_query_param_when_value_is_not_boolean():
+    query_params = {
+        'create_if_missing': ['true']
+    }
+    params = {}
+    key = "create_if_missing"
+    extract_from_query_params(query_params, params, key, is_boolean_type=False)
+    assert params is not None and len(params) == 1
+    assert params[key] == 'true'
+
+
+def test_extract_query_param_when_key_is_not_found_in_query_params():
+    query_params = {
+        'create_if_missing': ['true']
+    }
+    params = {}
+    key = "key_not_found"
+    extract_from_query_params(query_params, params, key)
+    assert params is not None and len(params) == 0
+
+
+def test_extract_query_param_when_key_is_extracted():
+    query_params = {
+        'create_if_missing': ['true'],
+        'deploy': ['now']
+    }
+
+    params = {
+        'use_sas': True
+    }
+    key = "deploy"
+    extract_from_query_params(query_params, params, key)
+    assert params is not None and len(params) == 2
+    assert params['use_sas'] is True
+    assert params['deploy'] == 'now'
+
+
+def test_extract_query_param_when_key_already_exit():
+    query_params = {
+        'create_if_missing': ['true'],
+        'deploy': ['now']
+    }
+
+    params = {
+        'deploy': 'later'
+    }
+
+    key = "deploy"
+    extract_from_query_params(query_params, params, key)
+    assert params is not None and len(params) == 1
+    assert params['deploy'] == 'now'
+
+
 def test_roundtrip():
     assert isinstance(storefact.get_store_from_url(u'memory://#wrap:readonly'), simplekv.decorator.ReadOnlyDecorator)

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{27,35,36}
+envlist = py{27,35,36,39}
 
 [testenv]
 deps =