From 69a192c52a89d737d0dbe0e322eaafb7bc77334a Mon Sep 17 00:00:00 2001 From: angie Date: Wed, 22 May 2024 17:59:28 -0400 Subject: [PATCH 1/9] Turn off symbol filtering on psp and ps2 --- src/splat/platforms/ps2.py | 3 +++ src/splat/platforms/psp.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/splat/platforms/ps2.py b/src/splat/platforms/ps2.py index fbc0ffe7..7bf7668f 100644 --- a/src/splat/platforms/ps2.py +++ b/src/splat/platforms/ps2.py @@ -9,3 +9,6 @@ def init(target_bytes: bytes): spimdisasm.common.GlobalConfig.ABI = spimdisasm.common.Abi.EABI64 spimdisasm.common.GlobalConfig.SYMBOL_ALIGNMENT_REQUIRES_ALIGNED_SECTION = True + + spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTER_LOW_ADDRESSES = False + spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTER_HIGH_ADDRESSES = False diff --git a/src/splat/platforms/psp.py b/src/splat/platforms/psp.py index 6f65e266..ad189288 100644 --- a/src/splat/platforms/psp.py +++ b/src/splat/platforms/psp.py @@ -1,5 +1,8 @@ +import spimdisasm import rabbitizer def init(target_bytes: bytes): rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = False + spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTER_LOW_ADDRESSES = False + spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTER_HIGH_ADDRESSES = False From 54a66b923a8da00134668f5573d8b3fe8b3dd7b3 Mon Sep 17 00:00:00 2001 From: angie Date: Thu, 23 May 2024 08:56:24 -0400 Subject: [PATCH 2/9] `global_vram_start` and `global_vram_end` --- src/splat/platforms/ps2.py | 3 --- src/splat/platforms/psp.py | 3 --- src/splat/util/options.py | 6 ++++++ src/splat/util/symbols.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/splat/platforms/ps2.py b/src/splat/platforms/ps2.py index 7bf7668f..fbc0ffe7 100644 --- a/src/splat/platforms/ps2.py +++ b/src/splat/platforms/ps2.py @@ -9,6 +9,3 @@ def init(target_bytes: bytes): spimdisasm.common.GlobalConfig.ABI = spimdisasm.common.Abi.EABI64 spimdisasm.common.GlobalConfig.SYMBOL_ALIGNMENT_REQUIRES_ALIGNED_SECTION = True - - spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTER_LOW_ADDRESSES = False - spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTER_HIGH_ADDRESSES = False diff --git a/src/splat/platforms/psp.py b/src/splat/platforms/psp.py index ad189288..6f65e266 100644 --- a/src/splat/platforms/psp.py +++ b/src/splat/platforms/psp.py @@ -1,8 +1,5 @@ -import spimdisasm import rabbitizer def init(target_bytes: bytes): rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = False - spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTER_LOW_ADDRESSES = False - spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTER_HIGH_ADDRESSES = False diff --git a/src/splat/util/options.py b/src/splat/util/options.py index efaa866a..1a5ddb39 100644 --- a/src/splat/util/options.py +++ b/src/splat/util/options.py @@ -216,6 +216,10 @@ class SplatOpts: detect_redundant_function_end: bool # Don't skip disassembling already matched functions and migrated sections disassemble_all: bool + # Allow specifying a global segment that may be larger than the automatically detected. + # Useful for projects where splat is used in multiple individual files, meaning the expected global segment may not be properly detected because each instance of splat can't see the info from other files. + global_vram_start: Optional[int] + global_vram_end: Optional[int] ################################################################################ # N64-specific options @@ -531,6 +535,8 @@ def parse_endianness() -> Literal["big", "little"]: disassemble_all=( disasm_all if disasm_all else p.parse_opt("disassemble_all", bool, False) ), + global_vram_start=p.parse_optional_opt("global_vram_start", int), + global_vram_end=p.parse_optional_opt("global_vram_end", int), ) p.check_no_unread_opts() return ret diff --git a/src/splat/util/symbols.py b/src/splat/util/symbols.py index baaf2f23..75c85dab 100644 --- a/src/splat/util/symbols.py +++ b/src/splat/util/symbols.py @@ -314,8 +314,8 @@ def initialize(all_segments: "List[Segment]"): def initialize_spim_context(all_segments: "List[Segment]") -> None: global_vrom_start = None global_vrom_end = None - global_vram_start = None - global_vram_end = None + global_vram_start = options.opts.global_vram_start + global_vram_end = options.opts.global_vram_end overlay_segments: Set[spimdisasm.common.SymbolsSegment] = set() spim_context.bannedSymbols |= ignored_addresses From ae8617c9044d63e30aaf2d697976297e69ef541f Mon Sep 17 00:00:00 2001 From: angie Date: Wed, 12 Jun 2024 11:09:52 -0400 Subject: [PATCH 3/9] version bump --- CHANGELOG.md | 6 ++++++ README.md | 2 +- pyproject.toml | 2 +- src/splat/__init__.py | 2 +- src/splat/util/options.py | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e43771f..23c22e4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # splat Release Notes +### 0.24.5 + +* New yaml option: `global_vram_start` and `global_vram_end`. + * Allow specifying that the global segment may be larger than what was automatically detected. + * Useful for projects where splat is used in multiple individual files, meaning the expected global segment may not be properly detected because each instance of splat can't see the info from other files (like in PSX and PSP projects). + ### 0.24.4 * New yaml option: `matchings_path` diff --git a/README.md b/README.md index 920f7a18..3208841a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin If you use a `requirements.txt` file in your repository, then you can add this library with the following line: ```txt -splat64[mips]>=0.24.4,<1.0.0 +splat64[mips]>=0.24.5,<1.0.0 ``` ### Optional dependencies diff --git a/pyproject.toml b/pyproject.toml index 6e3eb018..01b5615c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "splat64" # Should be synced with src/splat/__init__.py -version = "0.24.4" +version = "0.24.5" description = "A binary splitting tool to assist with decompilation and modding projects" readme = "README.md" license = {file = "LICENSE"} diff --git a/src/splat/__init__.py b/src/splat/__init__.py index 804df3d7..e7fd2137 100644 --- a/src/splat/__init__.py +++ b/src/splat/__init__.py @@ -3,7 +3,7 @@ __package_name__ = __name__ # Should be synced with pyproject.toml -__version__ = "0.24.4" +__version__ = "0.24.5" __author__ = "ethteck" from . import util as util diff --git a/src/splat/util/options.py b/src/splat/util/options.py index 142d6fa6..a59ecff5 100644 --- a/src/splat/util/options.py +++ b/src/splat/util/options.py @@ -220,7 +220,7 @@ class SplatOpts: detect_redundant_function_end: bool # Don't skip disassembling already matched functions and migrated sections disassemble_all: bool - # Allow specifying a global segment that may be larger than the automatically detected. + # Allow specifying that the global segment may be larger than what was automatically detected. # Useful for projects where splat is used in multiple individual files, meaning the expected global segment may not be properly detected because each instance of splat can't see the info from other files. global_vram_start: Optional[int] global_vram_end: Optional[int] From 5d874a5df5af9ce03732e51a84f01b898670cc7a Mon Sep 17 00:00:00 2001 From: angie Date: Wed, 12 Jun 2024 11:14:44 -0400 Subject: [PATCH 4/9] typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3208841a..b8585356 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ splat64[mips]>=0.24.5,<1.0.0 ### Optional dependencies -- `mips`: Required when using the N64, PSX, PS2 or PSp platforms. +- `mips`: Required when using the N64, PSX, PS2 or PSP platforms. - `dev`: Installs all the available dependencies groups and other packages for development. ### Gamecube / Wii From 3137cb5b3132e06c27a08cc8239b2049521ac441 Mon Sep 17 00:00:00 2001 From: angie Date: Wed, 12 Jun 2024 11:15:05 -0400 Subject: [PATCH 5/9] typo 2 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b8585356..af8220e1 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,4 @@ splat64[mips]>=0.24.5,<1.0.0 ### Gamecube / Wii -For Gamecube / Wii projects, see [decomp-toolkit](https://github.com/encounter/decomp-toolkit)! \ No newline at end of file +For Gamecube / Wii projects, see [decomp-toolkit](https://github.com/encounter/decomp-toolkit)! From a28f991d00d5dd3a1e6eff314be60e8d6586a3d7 Mon Sep 17 00:00:00 2001 From: angie Date: Thu, 13 Jun 2024 11:25:47 -0400 Subject: [PATCH 6/9] Pass symbols to spimdisasm that are not part of the current binary --- src/splat/util/symbols.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/splat/util/symbols.py b/src/splat/util/symbols.py index 0bcbc831..bb0b0434 100644 --- a/src/splat/util/symbols.py +++ b/src/splat/util/symbols.py @@ -418,6 +418,19 @@ def initialize_spim_context(all_segments: "List[Segment]") -> None: for sym in symbols_list: add_symbol_to_spim_segment(spim_context.globalSegment, sym) + if global_vram_start and global_vram_end: + # Pass global symbols to spimdisasm that are not part of any segment on the binary we are splitting (for psx and psp) + for sym in all_symbols: + if sym.segment is not None: + # We already handled this symbol somewhere else + continue + + if sym.vram_start < global_vram_start or sym.vram_end > global_vram_end: + # Not global + continue + + add_symbol_to_spim_segment(spim_context.globalSegment, sym) + def add_symbol_to_spim_segment( segment: spimdisasm.common.SymbolsSegment, sym: "Symbol" From 7ecee9b325da5a572f63f305cc648dffc88e2218 Mon Sep 17 00:00:00 2001 From: angie Date: Thu, 13 Jun 2024 11:29:58 -0400 Subject: [PATCH 7/9] a bit of docs --- docs/Configuration.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/Configuration.md b/docs/Configuration.md index 6111876b..7b4197a4 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -623,6 +623,11 @@ Tries to detect redundant and unreferenced functions ends and merge them togethe Don't skip disassembling already matched functions and migrated sections +### global_vram_start and global_vram_end + +Allow specifying that the global segment may be larger than what was automatically detected. + +Useful for projects where splat is used in multiple individual files, meaning the expected global segment may not be properly detected because each instance of splat can't see the info from other files, like in PSX and PSP projects. ## N64-specific options From d42635ac5e6978d2cf54625c50e27bfcf69032c2 Mon Sep 17 00:00:00 2001 From: angie Date: Thu, 13 Jun 2024 11:44:41 -0400 Subject: [PATCH 8/9] Do not ignore symbols with addends --- src/splat/segtypes/common/codesubsegment.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/splat/segtypes/common/codesubsegment.py b/src/splat/segtypes/common/codesubsegment.py index ec4de8d8..e903879b 100644 --- a/src/splat/segtypes/common/codesubsegment.py +++ b/src/splat/segtypes/common/codesubsegment.py @@ -144,9 +144,7 @@ def process_insns( if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset: sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset] - context_sym = self.spim_section.get_section().getSymbol( - sym_address, tryPlusOffset=False - ) + context_sym = self.spim_section.get_section().getSymbol(sym_address) if context_sym is not None: symbols.create_symbol_from_spim_symbol( self.get_most_parent(), context_sym From bb223f31c8a4c414fdf7bf55e51ca5ba539363ca Mon Sep 17 00:00:00 2001 From: angie Date: Thu, 4 Jul 2024 14:19:05 -0400 Subject: [PATCH 9/9] global memory range --- CHANGELOG.md | 2 +- docs/Configuration.md | 2 +- src/splat/util/options.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23c22e4e..fc7514d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### 0.24.5 * New yaml option: `global_vram_start` and `global_vram_end`. - * Allow specifying that the global segment may be larger than what was automatically detected. + * Allow specifying that the global memory range may be larger than what was automatically detected. * Useful for projects where splat is used in multiple individual files, meaning the expected global segment may not be properly detected because each instance of splat can't see the info from other files (like in PSX and PSP projects). ### 0.24.4 diff --git a/docs/Configuration.md b/docs/Configuration.md index 7b4197a4..afc679e4 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -625,7 +625,7 @@ Don't skip disassembling already matched functions and migrated sections ### global_vram_start and global_vram_end -Allow specifying that the global segment may be larger than what was automatically detected. +Allow specifying that the global memory range may be larger than what was automatically detected. Useful for projects where splat is used in multiple individual files, meaning the expected global segment may not be properly detected because each instance of splat can't see the info from other files, like in PSX and PSP projects. diff --git a/src/splat/util/options.py b/src/splat/util/options.py index a59ecff5..e610acba 100644 --- a/src/splat/util/options.py +++ b/src/splat/util/options.py @@ -220,7 +220,7 @@ class SplatOpts: detect_redundant_function_end: bool # Don't skip disassembling already matched functions and migrated sections disassemble_all: bool - # Allow specifying that the global segment may be larger than what was automatically detected. + # Allow specifying that the global memory range may be larger than what was automatically detected. # Useful for projects where splat is used in multiple individual files, meaning the expected global segment may not be properly detected because each instance of splat can't see the info from other files. global_vram_start: Optional[int] global_vram_end: Optional[int]