bump version, parallelization, hypertypes fixes (#119)

pypechain/main.py

@@ -57,8 +57,12 @@ def pypechain(
     # Parse the files and gather AbiInfos
     abi_infos: list[AbiInfo] = []
     for json_file in json_files_to_process:
-        infos = load_abi_infos_from_file(json_file)
-        abi_infos.extend(infos)
+        # Skip failing files
+        try:
+            infos = load_abi_infos_from_file(json_file)
+            abi_infos.extend(infos)
+        except Exception as e:  # pylint: disable=broad-except
+            print(f"Skipping {json_file} due to error {e}")
 
     # Create/clear the output directory
     setup_directory(output_dir)

pypechain/render/main.py

@@ -3,17 +3,23 @@
 from __future__ import annotations
 
 import os
+from functools import partial
+from multiprocessing import Pool
 from pathlib import Path
 
-from pypechain.render.contract import get_contract_infos, render_contract_file
+from pypechain.render.contract import ContractInfo, get_contract_infos, render_contract_file
 from pypechain.render.types import render_types_file
 from pypechain.utilities.abi import AbiInfo
 from pypechain.utilities.file import write_string_to_file
 from pypechain.utilities.format import format_file
 
 
 def render_files(
-    abi_infos: list[AbiInfo], output_dir: str, line_length: int = 120, apply_formatting: bool = True
+    abi_infos: list[AbiInfo],
+    output_dir: str,
+    line_length: int = 120,
+    apply_formatting: bool = True,
+    chunksize: int | None = None,
 ) -> list[str]:
     """Processes a single JSON file to generate class and types files.
 
@@ -28,6 +34,10 @@ def render_files(
         Black's line-length config option.
     apply_formatting : bool, optional
         If True, autoflake, isort and black will be applied to the file in that order, by default True.
+    chunksize: int, optional
+        If set, abis are processed in approximately groups of chunksize.
+        Use 1 for one process per ABI.
+        Defaults to the number of ABIs // 10.
 
     Returns
     -------
@@ -37,34 +47,68 @@ def render_files(
 
     contract_infos = get_contract_infos(abi_infos)
 
-    # This is what we return.
+    # this is what we are returning
     file_names: list[str] = []
 
-    # Now, for every [ContractName]
-    # generate a:
+    # for every [ContractName] generate a:
     #    1. [ContractName]Contract.py
     #    2. [ContractName]Types.py
     #  The Contract file defines the Contract class and functions.
     #  The Types file has the structs and events defined in the solidity contract.
-    for contract_info in contract_infos.values():
-        file_path = Path(output_dir)
-
-        rendered_contract_code = render_contract_file(contract_info)
-        if rendered_contract_code:
-            contract_file_name = contract_info.contract_name + "Contract.py"
-            contract_file_path = Path(os.path.join(file_path, contract_file_name))
-            write_string_to_file(contract_file_path, rendered_contract_code)
-            if apply_formatting is True:
-                format_file(contract_file_path, line_length)
-            file_names.append(f"{contract_info.contract_name}Contract")
-
-        rendered_types_code = render_types_file(contract_info)
-        if rendered_types_code:
-            types_file_name = contract_info.contract_name + "Types.py"
-            types_file_path = Path(os.path.join(file_path, types_file_name))
-            write_string_to_file(types_file_path, rendered_types_code)
-            if apply_formatting is True:
-                format_file(types_file_path, line_length)
-            file_names.append(f"{contract_info.contract_name}Types")
+
+    # make a parallel pool; defaults to the number of available CPUs
+    with Pool() as pool:
+        # chunksize should be increased with the number of iterables
+        chunksize = len(contract_infos) // 10
+        for file_name_sublist in pool.imap(
+            partial(get_file_names, output_dir=output_dir, line_length=line_length, apply_formatting=apply_formatting),
+            list(contract_infos.values()),
+            chunksize=chunksize,
+        ):
+            file_names.extend(file_name_sublist)
+
+    return file_names
+
+
+def get_file_names(contract_info: ContractInfo, output_dir: str, line_length: int, apply_formatting: bool) -> list[str]:
+    """Get the file name for a single ContractInfo object
+
+
+    Parameters
+    ----------
+    contract_info : ContractInfo
+        A ContractInfo object created from an ABI.
+    output_dir : str
+        The directory where files will be written to.
+    line_length : int, optional
+        Black's line-length config option.
+    apply_formatting : bool, optional
+        If True, autoflake, isort and black will be applied to the file in that order, by default True.
+
+    Returns
+    -------
+    list[str]
+        A list of filenames for the generated Contract and Types files.
+    """
+    file_names: list[str] = []
+    file_path = Path(output_dir)
+
+    rendered_contract_code = render_contract_file(contract_info)
+    if rendered_contract_code:
+        contract_file_name = contract_info.contract_name + "Contract.py"
+        contract_file_path = Path(os.path.join(file_path, contract_file_name))
+        write_string_to_file(contract_file_path, rendered_contract_code)
+        if apply_formatting is True:
+            format_file(contract_file_path, line_length)
+        file_names.append(f"{contract_info.contract_name}Contract")
+
+    rendered_types_code = render_types_file(contract_info)
+    if rendered_types_code:
+        types_file_name = contract_info.contract_name + "Types.py"
+        types_file_path = Path(os.path.join(file_path, types_file_name))
+        write_string_to_file(types_file_path, rendered_types_code)
+        if apply_formatting is True:
+            format_file(types_file_path, line_length)
+        file_names.append(f"{contract_info.contract_name}Types")
 
     return file_names

pypechain/utilities/abi.py

@@ -529,9 +529,7 @@ def get_struct_name(
     }
 
     We are assuming 'internalType's value to have the form:
-    'struct [ContractName].[StructName]'
-
-    If there is no ContractName, then this code will break on purpose.
+    'struct [ContractName].[StructName]' or 'struct [StructName]'
 
     Parameters
     ----------
@@ -544,7 +542,13 @@ def get_struct_name(
         The name of the item.
     """
     internal_type = cast(str, param_or_component.get("internalType", ""))
-    struct_name = internal_type.split(".")[1].rstrip("[]")
+    # grab subtype if there is one
+    if "." in internal_type:
+        internal_type = internal_type.split(".")[1]
+    # it is possible that the internal type has a "struct" label
+    # we want to strip that to only include the struct name itself
+    internal_type = internal_type.replace("struct ", "")
+    struct_name = internal_type.rstrip("[]")
     return capitalize_first_letter_only(struct_name)
 
 
@@ -566,8 +570,13 @@ def get_struct_type(
         The type of the item.
     """
     internal_type = cast(str, param_or_component.get("internalType", ""))
-    struct_full_name = internal_type.split(".")[1]
-    if struct_full_name[-2:] == "[]":  # ends with [] indicates an array
+    # grab subtype if there is one
+    if "." in internal_type:
+        internal_type = internal_type.split(".")[1]
+    # it is possible that the internal type has a "struct" label
+    # we want to strip that to only include the struct name itself
+    internal_type = internal_type.replace("struct ", "")
+    if internal_type[-2:] == "[]":  # ends with [] indicates an array
         return "list[" + get_struct_name(param_or_component) + "]"
     return get_struct_name(param_or_component)
 

pypechain/utilities/types.py

@@ -155,6 +155,14 @@ def solidity_to_python_type(solidity_type: str, custom_types: list[str] = []) ->
         return "str"
     if solidity_type == "string[]":
         return "list[str]"
+    if solidity_type.startswith("string") and solidity_type.endswith("]"):
+        # possible to have e.g. "string[2][]" as the type
+        list_depth = solidity_type.count("[")
+        out_type_string = ""
+        for _ in range(list_depth):
+            out_type_string += "list["
+        out_type_string += "str" + "]" * list_depth
+        return out_type_string
 
     # tuple
     if solidity_type == "tuple":

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 
 name = "pypechain"
-version = "0.0.32"
+version = "0.0.33"
 authors = [
     { name = "Matthew Brown", email = "[email protected]" },
     { name = "Dylan Paiton", email = "[email protected]" },
@@ -16,7 +16,7 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Operating System :: OS Independent",
 ]
-dependencies = ["autoflake", "black", "eth-account", "isort", "jinja2", "toml", "web3>=6.20.2"]
+dependencies = ["autoflake", "black", "eth-account", "isort", "jinja2", "toml", "web3>=6.20.2, <7.0.0"]
 
 [project.scripts]
 pypechain = "pypechain.main:main"