From 5f91154c2e39fbb86d96273616f8de75971353ab Mon Sep 17 00:00:00 2001 From: PaTara Date: Thu, 29 Dec 2022 01:24:11 +0300 Subject: [PATCH] 1.5.0 ipfs upload, get, we3_gw auth --- docs/source/modules.rst | 21 ++++- docs/source/usage.rst | 39 ++++++---- pyproject.toml | 2 +- robonomicsinterface/__init__.py | 1 + robonomicsinterface/constants.py | 3 - robonomicsinterface/exceptions.py | 8 -- robonomicsinterface/ipfs_utils/__init__.py | 1 + .../ipfs_utils/ipfs_gateway_interaction.py | 74 ++++++++++++++++++ robonomicsinterface/utils.py | 76 ------------------- 9 files changed, 122 insertions(+), 103 deletions(-) create mode 100644 robonomicsinterface/ipfs_utils/__init__.py create mode 100644 robonomicsinterface/ipfs_utils/ipfs_gateway_interaction.py diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 1b656a9..9d5fa26 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -13,6 +13,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ BaseClass ___________________ @@ -21,6 +22,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ ChainUtils ___________________ @@ -29,6 +31,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ CommonFunctions ___________________ @@ -37,6 +40,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ Datalog ___________________ @@ -45,6 +49,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ Launch ___________________ @@ -53,6 +58,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ Liability ___________________ @@ -61,6 +67,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ PubSub ___________________ @@ -69,6 +76,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ ReqRes ___________________ @@ -77,6 +85,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ RWS ___________________ @@ -85,6 +94,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ ServiceFunctions ___________________ @@ -93,6 +103,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ Subscriber @@ -102,6 +113,7 @@ ___________________ :members: :private-members: :inherited-members: + :special-members: __init__ @@ -124,4 +136,11 @@ Decorators Utils ---------- .. automodule:: robonomicsinterface.utils - :members: \ No newline at end of file + :members: + +IPFS Utils +---------- +.. automodule:: robonomicsinterface.ipfs_utils + :members: web_3_auth, ipfs_upload_content, ipfs_get_content + + diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 8e8f31d..3f34053 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -435,28 +435,39 @@ string and vice-versa. ipfs_hash_decoded = ipfs_32_bytes_to_qm_hash("0xcc2d976220820d023b7170f520d3490e811ed988ae3d6221474ee97e559b0361") # >>> 'Qmc5gCcjYypU7y28oCALwfSvxCBskLuPKWpK4qpterKC7z' -There is one more useful functionality: Content upload to IPFS via Web3-auth gateway (and IPFS content fetch). The only -thing needed - account seed to sign an authentication message. + +IPFS Utils +++++++++++ + +There is one more useful functionality: Content upload to IPFS via local or remote gateway (and IPFS content fetch). To use +the local gateway no additional parameters needed: .. code-block:: python from robonomicsinterface.utils import ipfs_get_content, ipfs_upload_content - seed = "seed" - - content = "Hello, World!" - cid, size = ipfs_upload_content(tester_tokens_seed, content) - print(cid, size) + content = "heeelo" + cid, size = ipfs_upload_content(content=content) + print(cid) + >>> QmeWzphuZbSqVKaxeYQ45VUeaHv18qSgPX4wpQAD44uuMt content_ = ipfs_get_content(cid) print(content_) + >>> b'heeelo' +One may also pass a gateway address and use Web3-authenticate gateways! See more info +`on Crust Wiki `__. - with open("path_to_file", 'rb') as f: - content = f.read() - cid, size = ipfs_upload_content(tester_tokens_seed, content) - print(cid, size) +.. code-block:: python - content_ = ipfs_get_content(cid) - with open("path_to_the_fetched_file", 'wb') as f: - f.write(content_) + from robonomicsinterface.utils import ipfs_get_content, ipfs_upload_content, web_3_auth + + content = "heeelo" + auth = web_3_auth(tester_tokens_seed) + cid, size = ipfs_upload_content(content=content, gateway="web3_gateway_url", auth=auth) + print(cid) + >>> QmeWzphuZbSqVKaxeYQ45VUeaHv18qSgPX4wpQAD44uuMt + + content_ = ipfs_get_content(cid, gateway="web3_gateway_url") + print(content_) + >>> b'heeelo' diff --git a/pyproject.toml b/pyproject.toml index 3c849e4..406a935 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "robonomics-interface" -version = "1.4.0" +version = "1.5.0" description = "Robonomics wrapper over https://github.com/polkascan/py-substrate-interface created to facilitate programming with Robonomics" authors = ["Pavel Tarasov "] license = "Apache-2.0" diff --git a/robonomicsinterface/__init__.py b/robonomicsinterface/__init__.py index ebd8e3b..cbe6160 100644 --- a/robonomicsinterface/__init__.py +++ b/robonomicsinterface/__init__.py @@ -18,3 +18,4 @@ SubEvent, Subscriber, ) +from .ipfs_utils import web_3_auth, ipfs_upload_content, ipfs_get_content diff --git a/robonomicsinterface/constants.py b/robonomicsinterface/constants.py index 3eb4bcf..8bcbd67 100644 --- a/robonomicsinterface/constants.py +++ b/robonomicsinterface/constants.py @@ -7,6 +7,3 @@ "RingBufferIndex": {"type": "struct", "type_mapping": [["start", "Compact"], ["end", "Compact"]]}, } } - -W3GW = "https://crustwebsites.net" -W3PS = "https://pin.crustcode.com" diff --git a/robonomicsinterface/exceptions.py b/robonomicsinterface/exceptions.py index a9b93c2..4f793df 100644 --- a/robonomicsinterface/exceptions.py +++ b/robonomicsinterface/exceptions.py @@ -31,11 +31,3 @@ class FailedToUploadFile(Exception): """ pass - - -class FailedToPinFile(Exception): - """ - Failed to upload a file to Crust Network. - """ - - pass \ No newline at end of file diff --git a/robonomicsinterface/ipfs_utils/__init__.py b/robonomicsinterface/ipfs_utils/__init__.py new file mode 100644 index 0000000..ac8e5e8 --- /dev/null +++ b/robonomicsinterface/ipfs_utils/__init__.py @@ -0,0 +1 @@ +from .ipfs_gateway_interaction import web_3_auth, ipfs_upload_content, ipfs_get_content diff --git a/robonomicsinterface/ipfs_utils/ipfs_gateway_interaction.py b/robonomicsinterface/ipfs_utils/ipfs_gateway_interaction.py new file mode 100644 index 0000000..ec8b1da --- /dev/null +++ b/robonomicsinterface/ipfs_utils/ipfs_gateway_interaction.py @@ -0,0 +1,74 @@ +""" +A simple utility to upload and download content through ipfs gateways, local or public, with authentication or not. +""" +import logging +import requests +import typing as tp + +from ast import literal_eval +from substrateinterface import Keypair + +from ..exceptions import FailedToUploadFile +from ..utils import create_keypair + +logger = logging.getLogger(__name__) + + +def web_3_auth(seed: str) -> tp.Tuple[str, str]: + """ + Get authentication header for a Web3-auth IPFS gateway. + + :param seed: Substrate account seed in any, mnemonic or raw form. + + :return: Authentication header. + + """ + + keypair: Keypair = create_keypair(seed) + return f"sub-{keypair.ss58_address}", f"0x{keypair.sign(keypair.ss58_address).hex()}" + + +def ipfs_upload_content( + content: tp.Any, gateway: str = "http://127.0.0.1:5001", auth: tp.Optional[tp.Tuple[str, str]] = None +) -> tp.Tuple[str, int]: + """ + Upload content to IPFS and pin the CID for some time via IPFS Web3 Gateway with private-key-signed message. + The signed message is user's pubkey. https://wiki.crust.network/docs/en/buildIPFSWeb3AuthGW#usage. + + :param content: Content to upload to IPFS. To upload media use open(.., "rb") and read(). + :param gateway: Gateway to upload content through. Defaults to local IPFS gateway. + :param auth: Gateway authentication header if needed. Should be of form (login, password) Defaults to empty. + + :return: IPFS cid and file size. + + """ + + response = requests.post( + f"{gateway}/api/v0/add", + auth=auth, + files={"file@": (None, content)}, + ) + + if response.status_code == 200: + resp = literal_eval(response.content.decode("utf-8")) + cid: str = resp["Hash"] + size: int = int(resp["Size"]) + else: + raise FailedToUploadFile(response.status_code) + + return cid, size + + +def ipfs_get_content(cid: str, gateway: str = "http://127.0.0.1:8080") -> tp.Any: + """ + Get content file in IPFS network + + :param cid: IPFS cid. + :param gateway: Gateway to get content through. Defaults to local IPFS gateway. + + + :return: Content of a file stored. + + """ + + return requests.get(f"{gateway}/ipfs/{cid}").content diff --git a/robonomicsinterface/utils.py b/robonomicsinterface/utils.py index 7daa5c7..480d196 100644 --- a/robonomicsinterface/utils.py +++ b/robonomicsinterface/utils.py @@ -1,16 +1,11 @@ import hashlib import logging -import requests import typing as tp -from ast import literal_eval from base58 import b58decode, b58encode from scalecodec.base import RuntimeConfiguration, ScaleBytes, ScaleType from substrateinterface import Keypair, KeypairType -from .constants import W3GW, W3PS -from .exceptions import FailedToPinFile, FailedToUploadFile - logger = logging.getLogger(__name__) @@ -87,74 +82,3 @@ def str_to_scalebytes(data: tp.Union[int, str], type_str: str) -> ScaleBytes: scale_obj: ScaleType = RuntimeConfiguration().create_scale_object(type_str) return scale_obj.encode(data) - - -def ipfs_upload_content(seed: str, content: tp.Any, pin: bool = False) -> tp.Tuple[str, int]: - """ - Upload content to IPFS and pin the CID for some time via IPFS Web3 Gateway with private-key-signed message. - The signed message is user's pubkey. https://wiki.crust.network/docs/en/buildIPFSWeb3AuthGW#usage. - - :param seed: Account seed in raw/mnemonic form. - :param content: Content to upload to IPFS. To upload media use open(.., "rb") and read(). - :param pin: Whether pin file or not. - - :return: IPFS cid and file size. - - """ - - keypair: Keypair = create_keypair(seed) - - response = requests.post( - W3GW + "/api/v0/add", - auth=(f"sub-{keypair.ss58_address}", f"0x{keypair.sign(keypair.ss58_address).hex()}"), - files={"file@": (None, content)}, - ) - - if response.status_code == 200: - resp = literal_eval(response.content.decode("utf-8")) - cid: str = resp["Hash"] - size: int = int(resp["Size"]) - else: - raise FailedToUploadFile(response.status_code) - - if pin: - _pin_ipfs_cid(keypair, cid) - - return cid, size - - -def _pin_ipfs_cid(keypair: Keypair, ipfs_cid: str) -> bool: - """ - Pin file for some time via Web3 IPFS pinning service. This may help to spread the file wider across IPFS. - - :param keypair: Account keypair. - :param ipfs_cid: Uploaded file cid. - - :return: Server response flag. - """ - - body = {"cid": ipfs_cid} - - response = requests.post( - W3PS + "/psa/pins", - auth=(f"sub-{keypair.ss58_address}", f"0x{keypair.sign(keypair.ss58_address).hex()}"), - json=body, - ) - - if response.status_code == 200: - return True - else: - raise FailedToPinFile(response.status_code) - - -def ipfs_get_content(cid: str) -> tp.Any: - """ - Get content file in IPFS network - - :param cid: IPFS cid. - - :return: Content of a file stored. - - """ - - return requests.get(W3GW + "/ipfs/" + cid).content