diff --git a/pypechain/main.py b/pypechain/main.py index 17a31333..96e472e2 100644 --- a/pypechain/main.py +++ b/pypechain/main.py @@ -57,8 +57,12 @@ def pypechain( # Parse the files and gather AbiInfos abi_infos: list[AbiInfo] = [] for json_file in json_files_to_process: - infos = load_abi_infos_from_file(json_file) - abi_infos.extend(infos) + # Skip failing files + try: + infos = load_abi_infos_from_file(json_file) + abi_infos.extend(infos) + except Exception as e: # pylint: disable=broad-except + print(f"Skipping {json_file} due to error {e}") # Create/clear the output directory setup_directory(output_dir) diff --git a/pypechain/render/main.py b/pypechain/render/main.py index 89f3b9f2..026239f2 100644 --- a/pypechain/render/main.py +++ b/pypechain/render/main.py @@ -3,9 +3,11 @@ from __future__ import annotations import os +from functools import partial +from multiprocessing import Pool from pathlib import Path -from pypechain.render.contract import get_contract_infos, render_contract_file +from pypechain.render.contract import ContractInfo, get_contract_infos, render_contract_file from pypechain.render.types import render_types_file from pypechain.utilities.abi import AbiInfo from pypechain.utilities.file import write_string_to_file @@ -13,7 +15,11 @@ def render_files( - abi_infos: list[AbiInfo], output_dir: str, line_length: int = 120, apply_formatting: bool = True + abi_infos: list[AbiInfo], + output_dir: str, + line_length: int = 120, + apply_formatting: bool = True, + chunksize: int | None = None, ) -> list[str]: """Processes a single JSON file to generate class and types files. @@ -28,6 +34,10 @@ def render_files( Black's line-length config option. apply_formatting : bool, optional If True, autoflake, isort and black will be applied to the file in that order, by default True. + chunksize: int, optional + If set, abis are processed in approximately groups of chunksize. + Use 1 for one process per ABI. + Defaults to the number of ABIs // 10. Returns ------- @@ -37,34 +47,68 @@ def render_files( contract_infos = get_contract_infos(abi_infos) - # This is what we return. + # this is what we are returning file_names: list[str] = [] - # Now, for every [ContractName] - # generate a: + # for every [ContractName] generate a: # 1. [ContractName]Contract.py # 2. [ContractName]Types.py # The Contract file defines the Contract class and functions. # The Types file has the structs and events defined in the solidity contract. - for contract_info in contract_infos.values(): - file_path = Path(output_dir) - - rendered_contract_code = render_contract_file(contract_info) - if rendered_contract_code: - contract_file_name = contract_info.contract_name + "Contract.py" - contract_file_path = Path(os.path.join(file_path, contract_file_name)) - write_string_to_file(contract_file_path, rendered_contract_code) - if apply_formatting is True: - format_file(contract_file_path, line_length) - file_names.append(f"{contract_info.contract_name}Contract") - - rendered_types_code = render_types_file(contract_info) - if rendered_types_code: - types_file_name = contract_info.contract_name + "Types.py" - types_file_path = Path(os.path.join(file_path, types_file_name)) - write_string_to_file(types_file_path, rendered_types_code) - if apply_formatting is True: - format_file(types_file_path, line_length) - file_names.append(f"{contract_info.contract_name}Types") + + # make a parallel pool; defaults to the number of available CPUs + with Pool() as pool: + # chunksize should be increased with the number of iterables + chunksize = len(contract_infos) // 10 + for file_name_sublist in pool.imap( + partial(get_file_names, output_dir=output_dir, line_length=line_length, apply_formatting=apply_formatting), + list(contract_infos.values()), + chunksize=chunksize, + ): + file_names.extend(file_name_sublist) + + return file_names + + +def get_file_names(contract_info: ContractInfo, output_dir: str, line_length: int, apply_formatting: bool) -> list[str]: + """Get the file name for a single ContractInfo object + + + Parameters + ---------- + contract_info : ContractInfo + A ContractInfo object created from an ABI. + output_dir : str + The directory where files will be written to. + line_length : int, optional + Black's line-length config option. + apply_formatting : bool, optional + If True, autoflake, isort and black will be applied to the file in that order, by default True. + + Returns + ------- + list[str] + A list of filenames for the generated Contract and Types files. + """ + file_names: list[str] = [] + file_path = Path(output_dir) + + rendered_contract_code = render_contract_file(contract_info) + if rendered_contract_code: + contract_file_name = contract_info.contract_name + "Contract.py" + contract_file_path = Path(os.path.join(file_path, contract_file_name)) + write_string_to_file(contract_file_path, rendered_contract_code) + if apply_formatting is True: + format_file(contract_file_path, line_length) + file_names.append(f"{contract_info.contract_name}Contract") + + rendered_types_code = render_types_file(contract_info) + if rendered_types_code: + types_file_name = contract_info.contract_name + "Types.py" + types_file_path = Path(os.path.join(file_path, types_file_name)) + write_string_to_file(types_file_path, rendered_types_code) + if apply_formatting is True: + format_file(types_file_path, line_length) + file_names.append(f"{contract_info.contract_name}Types") return file_names diff --git a/pypechain/utilities/abi.py b/pypechain/utilities/abi.py index 5a3dc373..42e8202d 100644 --- a/pypechain/utilities/abi.py +++ b/pypechain/utilities/abi.py @@ -529,9 +529,7 @@ def get_struct_name( } We are assuming 'internalType's value to have the form: - 'struct [ContractName].[StructName]' - - If there is no ContractName, then this code will break on purpose. + 'struct [ContractName].[StructName]' or 'struct [StructName]' Parameters ---------- @@ -544,7 +542,13 @@ def get_struct_name( The name of the item. """ internal_type = cast(str, param_or_component.get("internalType", "")) - struct_name = internal_type.split(".")[1].rstrip("[]") + # grab subtype if there is one + if "." in internal_type: + internal_type = internal_type.split(".")[1] + # it is possible that the internal type has a "struct" label + # we want to strip that to only include the struct name itself + internal_type = internal_type.replace("struct ", "") + struct_name = internal_type.rstrip("[]") return capitalize_first_letter_only(struct_name) @@ -566,8 +570,13 @@ def get_struct_type( The type of the item. """ internal_type = cast(str, param_or_component.get("internalType", "")) - struct_full_name = internal_type.split(".")[1] - if struct_full_name[-2:] == "[]": # ends with [] indicates an array + # grab subtype if there is one + if "." in internal_type: + internal_type = internal_type.split(".")[1] + # it is possible that the internal type has a "struct" label + # we want to strip that to only include the struct name itself + internal_type = internal_type.replace("struct ", "") + if internal_type[-2:] == "[]": # ends with [] indicates an array return "list[" + get_struct_name(param_or_component) + "]" return get_struct_name(param_or_component) diff --git a/pypechain/utilities/types.py b/pypechain/utilities/types.py index c3981a1d..238e1e78 100644 --- a/pypechain/utilities/types.py +++ b/pypechain/utilities/types.py @@ -155,6 +155,14 @@ def solidity_to_python_type(solidity_type: str, custom_types: list[str] = []) -> return "str" if solidity_type == "string[]": return "list[str]" + if solidity_type.startswith("string") and solidity_type.endswith("]"): + # possible to have e.g. "string[2][]" as the type + list_depth = solidity_type.count("[") + out_type_string = "" + for _ in range(list_depth): + out_type_string += "list[" + out_type_string += "str" + "]" * list_depth + return out_type_string # tuple if solidity_type == "tuple": diff --git a/pyproject.toml b/pyproject.toml index 1a58955b..e3778ea0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "pypechain" -version = "0.0.32" +version = "0.0.33" authors = [ { name = "Matthew Brown", email = "matt@delv.tech" }, { name = "Dylan Paiton", email = "dylan@delv.tech" }, @@ -16,7 +16,7 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ] -dependencies = ["autoflake", "black", "eth-account", "isort", "jinja2", "toml", "web3>=6.20.2"] +dependencies = ["autoflake", "black", "eth-account", "isort", "jinja2", "toml", "web3>=6.20.2, <7.0.0"] [project.scripts] pypechain = "pypechain.main:main"