Merged in feature/solana-decoder (pull request #1)

Solana decoder

.gitignore

@@ -159,4 +159,5 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
-**.csv
+**.csv
+test.py

cli/setup.py

@@ -39,6 +39,7 @@ def read(fname):
         "blockchain-etl-common==1.6.1",
         "click==8.1.3",
         "web3==5.29.2",
+        "based58==0.1.1",
         "requests",
     ],
     extras_require={

cli/solanaetl/decoder/system_program.py

@@ -0,0 +1,38 @@
+# The MIT License (MIT)
+# Copyright (c) 2022 Gamejam.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+# and associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+from typing import Dict
+
+from based58 import b58decode
+from solanaetl.decoder.utils import decode_params, ns64, public_key, u32
+
+
+def decode(data: str) -> Dict[str, object]:
+    data_decoded = b58decode(data.encode())
+    program_func_index, _ = u32(data_decoded)
+    decoding_params = {
+        # Create
+        0: {
+            "instruction": u32,
+            "lamports": ns64,
+            "space": ns64,
+            "program_id": public_key,
+        }
+    }
+
+    return decode_params(data_decoded, decoding_params, program_func_index)

cli/solanaetl/decoder/token_program.py

@@ -0,0 +1,38 @@
+# The MIT License (MIT)
+# Copyright (c) 2022 Gamejam.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+# and associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+from typing import Dict
+
+from based58 import b58decode
+from solanaetl.decoder.utils import decode_params, u4, u64
+
+# See: https://github.com/solana-labs/solana-program-library/blob/master/token/program/src/instruction.rs
+
+
+def decode(data: str) -> Dict[str, object]:
+    data_decoded = b58decode(data.encode())
+    program_func_index, _ = u4(data_decoded)
+    decoding_params = {
+        # Transfer
+        3: {
+            "instruction": u4,
+            "amount": u64,
+        }
+    }
+
+    return decode_params(data_decoded, decoding_params, program_func_index)

cli/solanaetl/decoder/utils.py

@@ -0,0 +1,68 @@
+# The MIT License (MIT)
+# Copyright (c) 2022 Gamejam.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+# and associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+from typing import Callable, Dict
+
+from based58 import b58encode
+
+V2E32 = pow(2, 32)
+
+
+def rounded_int64(hi32, lo32):
+    return hi32 * V2E32 + lo32
+
+
+def uint(data: bytes, n_bytes: int, offset: int = 0) -> tuple[int, int]:
+    return int.from_bytes(data[offset:n_bytes], byteorder="little"), offset+n_bytes
+
+
+def u4(data: bytes, offset: int = 0) -> tuple[int, int]:
+    return uint(data, 1, offset)
+
+
+def u32(data: bytes, offset: int = 0) -> tuple[int, int]:
+    return uint(data, 4, offset)
+
+
+def u64(data: bytes, offset: int = 0) -> tuple[int, int]:
+    return uint(data, 8, offset)
+
+
+def ns64(data: bytes, offset: int = 0) -> tuple[int, int]:
+    lo32, next_offset = u32(data, offset)
+    hi32, next_offset = u32(data, next_offset)
+    return rounded_int64(hi32, lo32), next_offset
+
+
+def blob(data: bytes, n_bytes: int, offset: int = 0) -> tuple[bytes, int]:
+    return data[offset:offset+n_bytes], offset+n_bytes
+
+
+def public_key(data: bytes, offset: int = 0) -> tuple[str, int]:
+    public_key_bytes, next_offset = blob(data, 32, offset)
+    return b58encode(public_key_bytes).decode("utf-8"), next_offset
+
+
+def decode_params(data: bytes, decoding_params: Dict[int, Dict[str, Callable]], func_index: int) -> Dict[str, object]:
+    offset = 0
+    decoded_params = {}
+
+    for property, hanlde_func in decoding_params.get(func_index).items():
+        decoded_params[property], offset = hanlde_func(data, offset)
+
+    return decoded_params

cli/solanaetl/domain/token.py

@@ -17,5 +17,4 @@
 
 class Token(object):
     def __init__(self) -> None:
-        # TODO: Implement me
-        pass
+        self.block_number = None

cli/solanaetl/domain/transaction.py

@@ -24,3 +24,4 @@ def __init__(self) -> None:
         self.block_number = None
         self.block_timestamp = None
         self.fee = None
+        self.status = None

cli/solanaetl/jobs/export_tokens_job.py

@@ -0,0 +1,68 @@
+# The MIT License (MIT)
+# Copyright (c) 2022 Gamejam.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+# and associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+from blockchainetl_common.jobs.exporters.composite_item_exporter import \
+    CompositeItemExporter
+from blockchainetl_common.jobs.base_job import BaseJob
+from solanaetl.executors.batch_work_executor import BatchWorkExecutor
+from solanaetl.mappers.token_mapper import TokenMapper
+from solanaetl.services.token_service import TokenService
+
+
+class ExportTokensJob(BaseJob):
+    def __init__(self, web3, item_exporter: CompositeItemExporter, token_addresses_iterable, max_workers) -> None:
+        self.item_exporter = item_exporter
+        self.token_addresses_iterable = token_addresses_iterable
+        self.batch_work_executor = BatchWorkExecutor(1, max_workers)
+
+        self.token_service = TokenService(web3, clean_user_provided_content)
+        self.token_mapper = TokenMapper()
+
+    def _start(self):
+        self.item_exporter.open()
+
+    def _export(self):
+        self.batch_work_executor.execute(
+            self.token_addresses_iterable, self._export_tokens)
+
+    def _export_tokens(self, token_addresses):
+        for token_address in token_addresses:
+            self._export_token(token_address)
+
+    def _export_token(self, token_address, block_number=None):
+        token = self.token_service.get_token(token_address)
+        token.block_number = block_number
+        token_dict = self.token_mapper.token_to_dict(token)
+        self.item_exporter.export_item(token_dict)
+
+    def _end(self):
+        self.batch_work_executor.shutdown()
+        self.item_exporter.close()
+
+
+ASCII_0 = 0
+
+
+def clean_user_provided_content(content):
+    if isinstance(content, str):
+        # This prevents this error in BigQuery
+        # Error while reading data, error message: Error detected while parsing row starting at position: 9999.
+        # Error: Bad character (ASCII 0) encountered.
+        return content.translate({ASCII_0: None})
+    else:
+        return content

cli/solanaetl/jobs/exporters/blocks_and_transactions_item_exporter.py

@@ -37,6 +37,7 @@
     'block_number',
     'block_timestamp',
     'fee',
+    'status',
 ]
 
 

cli/solanaetl/mappers/token_mapper.py

@@ -0,0 +1,32 @@
+# The MIT License (MIT)
+# Copyright (c) 2022 Gamejam.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+# and associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+from solanaetl.domain.token import Token
+
+
+class TokenMapper(object):
+    def token_to_dict(self, token: Token):
+        return {
+            'type': 'token',
+            # 'address': token.address,
+            # 'symbol': token.symbol,
+            # 'name': token.name,
+            # 'decimals': token.decimals,
+            # 'total_supply': token.total_supply,
+            # 'block_number': token.block_number
+        }

cli/solanaetl/mappers/transaction_mapper.py

@@ -38,6 +38,8 @@ def json_dict_to_transaction(self, json_dict, **kwargs):
         tx_meta_json = json_dict.get('meta')
         if tx_meta_json is not None:
             transaction.fee = tx_meta_json.get('fee')
+            tx_err = tx_meta_json.get('err')
+            transaction.status = "Success" if tx_err is None else "Fail"
 
         return transaction
 
@@ -51,4 +53,5 @@ def transaction_to_dict(self, transaction: Transaction):
             'block_number': transaction.block_number,
             'block_timestamp': transaction.block_timestamp,
             'fee': transaction.fee,
+            'status': transaction.status,
         }

cli/solanaetl/services/__init__.py

@@ -0,0 +1,16 @@
+# The MIT License (MIT)
+# Copyright (c) 2022 Gamejam.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+# and associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

cli/solanaetl/services/token_service.py

@@ -0,0 +1,80 @@
+# The MIT License (MIT)
+# Copyright (c) 2022 Gamejam.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+# and associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+from solanaetl.domain.token import Token
+
+
+class TokenService(object):
+    def __init__(self, web3, function_call_result_transformer=None):
+        self._web3 = web3
+        self._function_call_result_transformer = function_call_result_transformer
+
+    def get_token(self, token_address) -> Token:
+        # checksum_address = self._web3.toChecksumAddress(token_address)
+        # contract = self._web3.eth.contract(address=checksum_address, abi=ERC20_ABI)
+
+        # symbol = self._get_first_result(contract.functions.symbol(), contract.functions.SYMBOL())
+        # name = self._get_first_result(contract.functions.name(), contract.functions.NAME())
+        # decimals = self._get_first_result(contract.functions.decimals(), contract.functions.DECIMALS())
+        # total_supply = self._get_first_result(contract.functions.totalSupply())
+
+        # token = EthToken()
+        # token.address = token_address
+        # token.symbol = symbol
+        # token.name = name
+        # token.decimals = decimals
+        # token.total_supply = total_supply
+
+        return None
+
+    def _get_first_result(self, *funcs):
+        # for func in funcs:
+        #     result = self._call_contract_function(func)
+        #     if result is not None:
+        #         return result
+        # return None
+        pass
+
+    def _call_contract_function(self, func):
+        # BadFunctionCallOutput exception happens if the token doesn't implement a particular function
+        # or was self-destructed
+        # OverflowError exception happens if the return type of the function doesn't match the expected type
+        # result = call_contract_function(
+        #     func=func,
+        #     ignore_errors=(BadFunctionCallOutput, OverflowError, ValueError),
+        #     default_value=None)
+
+        # if self._function_call_result_transformer is not None:
+        #     return self._function_call_result_transformer(result)
+        # else:
+        #     return result
+        pass
+
+
+def call_contract_function(func, ignore_errors, default_value=None):
+    pass
+    # try:
+    #     result = func.call()
+    #     return result
+    # except Exception as ex:
+    #     if type(ex) in ignore_errors:
+    #         logger.exception('An exception occurred in function {} of contract {}. '.format(func.fn_name, func.address)
+    #                          + 'This exception can be safely ignored.')
+    #         return default_value
+    #     else:
+    #         raise ex