1.5.0 ipfs upload, get, we3_gw auth

docs/source/modules.rst

@@ -13,6 +13,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 BaseClass
 ___________________
@@ -21,6 +22,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 ChainUtils
 ___________________
@@ -29,6 +31,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 CommonFunctions
 ___________________
@@ -37,6 +40,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 Datalog
 ___________________
@@ -45,6 +49,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 Launch
 ___________________
@@ -53,6 +58,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 Liability
 ___________________
@@ -61,6 +67,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 PubSub
 ___________________
@@ -69,6 +76,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 ReqRes
 ___________________
@@ -77,6 +85,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 RWS
 ___________________
@@ -85,6 +94,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 ServiceFunctions
 ___________________
@@ -93,6 +103,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 
 Subscriber
@@ -102,6 +113,7 @@ ___________________
     :members:
     :private-members:
     :inherited-members:
+    :special-members: __init__
 
 
 
@@ -124,4 +136,11 @@ Decorators
 Utils
 ----------
 .. automodule:: robonomicsinterface.utils
-    :members:
+    :members:
+
+IPFS Utils
+----------
+.. automodule:: robonomicsinterface.ipfs_utils
+    :members: web_3_auth, ipfs_upload_content, ipfs_get_content
+
+

docs/source/usage.rst

@@ -435,28 +435,39 @@ string and vice-versa.
     ipfs_hash_decoded = ipfs_32_bytes_to_qm_hash("0xcc2d976220820d023b7170f520d3490e811ed988ae3d6221474ee97e559b0361")
     # >>> 'Qmc5gCcjYypU7y28oCALwfSvxCBskLuPKWpK4qpterKC7z'
 
-There is one more useful functionality: Content upload to IPFS via Web3-auth gateway (and IPFS content fetch). The only
-thing needed - account seed to sign an authentication message.
+
+IPFS Utils
+++++++++++
+
+There is one more useful functionality: Content upload to IPFS via local or remote gateway (and IPFS content fetch). To use
+the local gateway no additional parameters needed:
 
 .. code-block:: python
 
     from robonomicsinterface.utils import ipfs_get_content, ipfs_upload_content
 
-    seed = "seed"
-
-    content = "Hello, World!"
-    cid, size = ipfs_upload_content(tester_tokens_seed, content)
-    print(cid, size)
+    content = "heeelo"
+    cid, size = ipfs_upload_content(content=content)
+    print(cid)
+    >>> QmeWzphuZbSqVKaxeYQ45VUeaHv18qSgPX4wpQAD44uuMt
 
     content_ = ipfs_get_content(cid)
     print(content_)
+    >>> b'heeelo'
 
+One may also pass a gateway address and use Web3-authenticate gateways! See more info
+`on Crust Wiki <https://wiki.crust.network/docs/en/buildIPFSWeb3AuthGW>`__.
 
-    with open("path_to_file", 'rb') as f:
-        content = f.read()
-    cid, size = ipfs_upload_content(tester_tokens_seed, content)
-    print(cid, size)
+.. code-block:: python
 
-    content_ = ipfs_get_content(cid)
-    with open("path_to_the_fetched_file", 'wb') as f:
-        f.write(content_)
+    from robonomicsinterface.utils import ipfs_get_content, ipfs_upload_content, web_3_auth
+
+    content = "heeelo"
+    auth = web_3_auth(tester_tokens_seed)
+    cid, size = ipfs_upload_content(content=content, gateway="web3_gateway_url", auth=auth)
+    print(cid)
+    >>> QmeWzphuZbSqVKaxeYQ45VUeaHv18qSgPX4wpQAD44uuMt
+
+    content_ = ipfs_get_content(cid, gateway="web3_gateway_url")
+    print(content_)
+    >>> b'heeelo'

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "robonomics-interface"
-version = "1.4.0"
+version = "1.5.0"
 description = "Robonomics wrapper over https://github.com/polkascan/py-substrate-interface created to facilitate programming with Robonomics"
 authors = ["Pavel Tarasov <[email protected]>"]
 license = "Apache-2.0"

robonomicsinterface/__init__.py

@@ -18,3 +18,4 @@
     SubEvent,
     Subscriber,
 )
+from .ipfs_utils import web_3_auth, ipfs_upload_content, ipfs_get_content

robonomicsinterface/constants.py

@@ -7,6 +7,3 @@
         "RingBufferIndex": {"type": "struct", "type_mapping": [["start", "Compact<u64>"], ["end", "Compact<u64>"]]},
     }
 }
-
-W3GW = "https://crustwebsites.net"
-W3PS = "https://pin.crustcode.com"

robonomicsinterface/exceptions.py

@@ -31,11 +31,3 @@ class FailedToUploadFile(Exception):
     """
 
     pass
-
-
-class FailedToPinFile(Exception):
-    """
-    Failed to upload a file to Crust Network.
-    """
-
-    pass

robonomicsinterface/ipfs_utils/__init__.py

@@ -0,0 +1 @@
+from .ipfs_gateway_interaction import web_3_auth, ipfs_upload_content, ipfs_get_content

robonomicsinterface/ipfs_utils/ipfs_gateway_interaction.py

@@ -0,0 +1,74 @@
+"""
+A simple utility to upload and download content through ipfs gateways, local or public, with authentication or not.
+"""
+import logging
+import requests
+import typing as tp
+
+from ast import literal_eval
+from substrateinterface import Keypair
+
+from ..exceptions import FailedToUploadFile
+from ..utils import create_keypair
+
+logger = logging.getLogger(__name__)
+
+
+def web_3_auth(seed: str) -> tp.Tuple[str, str]:
+    """
+    Get authentication header for a Web3-auth IPFS gateway.
+
+    :param seed: Substrate account seed in any, mnemonic or raw form.
+
+    :return: Authentication header.
+
+    """
+
+    keypair: Keypair = create_keypair(seed)
+    return f"sub-{keypair.ss58_address}", f"0x{keypair.sign(keypair.ss58_address).hex()}"
+
+
+def ipfs_upload_content(
+    content: tp.Any, gateway: str = "http://127.0.0.1:5001", auth: tp.Optional[tp.Tuple[str, str]] = None
+) -> tp.Tuple[str, int]:
+    """
+    Upload content to IPFS and pin the CID for some time via IPFS Web3 Gateway with private-key-signed message.
+        The signed message is user's pubkey. https://wiki.crust.network/docs/en/buildIPFSWeb3AuthGW#usage.
+
+    :param content: Content to upload to IPFS. To upload media use open(.., "rb") and read().
+    :param gateway: Gateway to upload content through. Defaults to local IPFS gateway.
+    :param auth: Gateway authentication header if needed. Should be of form (login, password) Defaults to empty.
+
+    :return: IPFS cid and file size.
+
+    """
+
+    response = requests.post(
+        f"{gateway}/api/v0/add",
+        auth=auth,
+        files={"file@": (None, content)},
+    )
+
+    if response.status_code == 200:
+        resp = literal_eval(response.content.decode("utf-8"))
+        cid: str = resp["Hash"]
+        size: int = int(resp["Size"])
+    else:
+        raise FailedToUploadFile(response.status_code)
+
+    return cid, size
+
+
+def ipfs_get_content(cid: str, gateway: str = "http://127.0.0.1:8080") -> tp.Any:
+    """
+    Get content file in IPFS network
+
+    :param cid: IPFS cid.
+    :param gateway: Gateway to get content through. Defaults to local IPFS gateway.
+
+
+    :return: Content of a file stored.
+
+    """
+
+    return requests.get(f"{gateway}/ipfs/{cid}").content

robonomicsinterface/utils.py

@@ -1,16 +1,11 @@
 import hashlib
 import logging
-import requests
 import typing as tp
 
-from ast import literal_eval
 from base58 import b58decode, b58encode
 from scalecodec.base import RuntimeConfiguration, ScaleBytes, ScaleType
 from substrateinterface import Keypair, KeypairType
 
-from .constants import W3GW, W3PS
-from .exceptions import FailedToPinFile, FailedToUploadFile
-
 logger = logging.getLogger(__name__)
 
 
@@ -87,74 +82,3 @@ def str_to_scalebytes(data: tp.Union[int, str], type_str: str) -> ScaleBytes:
 
     scale_obj: ScaleType = RuntimeConfiguration().create_scale_object(type_str)
     return scale_obj.encode(data)
-
-
-def ipfs_upload_content(seed: str, content: tp.Any, pin: bool = False) -> tp.Tuple[str, int]:
-    """
-    Upload content to IPFS and pin the CID for some time via IPFS Web3 Gateway with private-key-signed message.
-        The signed message is user's pubkey. https://wiki.crust.network/docs/en/buildIPFSWeb3AuthGW#usage.
-
-    :param seed: Account seed in raw/mnemonic form.
-    :param content: Content to upload to IPFS. To upload media use open(.., "rb") and read().
-    :param pin: Whether pin file or not.
-
-    :return: IPFS cid and file size.
-
-    """
-
-    keypair: Keypair = create_keypair(seed)
-
-    response = requests.post(
-        W3GW + "/api/v0/add",
-        auth=(f"sub-{keypair.ss58_address}", f"0x{keypair.sign(keypair.ss58_address).hex()}"),
-        files={"file@": (None, content)},
-    )
-
-    if response.status_code == 200:
-        resp = literal_eval(response.content.decode("utf-8"))
-        cid: str = resp["Hash"]
-        size: int = int(resp["Size"])
-    else:
-        raise FailedToUploadFile(response.status_code)
-
-    if pin:
-        _pin_ipfs_cid(keypair, cid)
-
-    return cid, size
-
-
-def _pin_ipfs_cid(keypair: Keypair, ipfs_cid: str) -> bool:
-    """
-    Pin file for some time via Web3 IPFS pinning service. This may help to spread the file wider across IPFS.
-
-    :param keypair: Account keypair.
-    :param ipfs_cid: Uploaded file cid.
-
-    :return: Server response flag.
-    """
-
-    body = {"cid": ipfs_cid}
-
-    response = requests.post(
-        W3PS + "/psa/pins",
-        auth=(f"sub-{keypair.ss58_address}", f"0x{keypair.sign(keypair.ss58_address).hex()}"),
-        json=body,
-    )
-
-    if response.status_code == 200:
-        return True
-    else:
-        raise FailedToPinFile(response.status_code)
-
-
-def ipfs_get_content(cid: str) -> tp.Any:
-    """
-    Get content file in IPFS network
-
-    :param cid: IPFS cid.
-
-    :return: Content of a file stored.
-
-    """
-
-    return requests.get(W3GW + "/ipfs/" + cid).content