diff --git a/CHANGELOG.md b/CHANGELOG.md index 14f15a3c..3d7ddbdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # splat Release Notes +### 0.27.0 + +* BREAKING: Renamed `auto_all_sections` to `auto_link_sections` and documented its behavior. +* BREAKING: Removed redundant `N64Segment` stub class. Any references to this should be instead to the base `Segment` +* Promoted `linker_offset` segment type to common, so it's now usable by all platforms. +* Added documentation for the remaining undocumented segment types and did some general doc tidying. +* Splat will now error when the last segment is `pad`, as this will not work as expected. +* Attempting to retrieve the `subalign` property of a non-top-level segment will now return an error. + ### 0.26.2 * Fixed not being able to disable the `subalign` directive for a given segment. @@ -223,7 +232,7 @@ ```py from util import log, options - from segtypes.n64.segment import N64Segment + from segtypes.common.segment import Segment from segtypes.common.data import CommonSegData ``` @@ -235,7 +244,7 @@ ```py from splat.util import log, options - from splat.segtypes.n64.segment import N64Segment + from splat.segtypes.common.segment import Segment from splat.segtypes.common.data import CommonSegData ``` @@ -250,7 +259,7 @@ ```py from ...util import log, options - from .segment import N64Segment + from ..common.segment import Segment from ..common.data import CommonSegData ``` diff --git a/README.md b/README.md index c16a1c7d..46359f97 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin If you use a `requirements.txt` file in your repository, then you can add this library with the following line: ```txt -splat64[mips]>=0.26.2,<1.0.0 +splat64[mips]>=0.27.0,<1.0.0 ``` ### Optional dependencies diff --git a/docs/Configuration.md b/docs/Configuration.md index 3955e10e..2daf124a 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -308,7 +308,7 @@ extensions_path: path/to/extensions/folder ### lib_path -Determines the path to library files that are to be linked into the target binary +Determines the path to library files that are to be linked into the target binary when the [`lib`](https://github.com/ethteck/splat/wiki/Segments#lib) segment type is used. ### elf_section_list_path @@ -341,9 +341,12 @@ subalign: 4 `16` -### auto_all_sections +### auto_link_sections -TODO +A list of linker sections for which entries will be automatically added to the linker script. If a segment contains 10 "c" subsegments, one can rely on this feature to automatically create linker entries for these files in the specified sections. This feature reduces the need to manually add lines to your yaml which only would serve to add linker entries for common sections, such as .data, .rodata, and .bss. + +#### Default +`[".data", ".rodata", ".bss"]` ### ld_script_path diff --git a/docs/General-Workflow.md b/docs/General-Workflow.md index a459107f..7ff051de 100644 --- a/docs/General-Workflow.md +++ b/docs/General-Workflow.md @@ -245,7 +245,7 @@ to: **NOTE:** -If using `auto_all_section` and there are no other `data`/`.data`/`rodata`/`.rodata` in the subsegments in the code segment, the subsegments can also be changed to +If using `auto_link_sections` and data is fully migrated, the subsegments can also be changed to the following and splat will add the appropriate entries into the linker script. ```yaml - [0x42100, c, energy_orb_wave] diff --git a/docs/Quickstart.md b/docs/Quickstart.md index 6a833079..078d8bee 100644 --- a/docs/Quickstart.md +++ b/docs/Quickstart.md @@ -79,7 +79,7 @@ options: asm_data_macro: dlabel # section_order: [".text", ".data", ".rodata", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] + # auto_link_sections: [".data", ".rodata", ".bss"] symbol_addrs_path: - symbol_addrs.txt diff --git a/docs/Segments.md b/docs/Segments.md index e561f257..02427bc7 100644 --- a/docs/Segments.md +++ b/docs/Segments.md @@ -121,6 +121,10 @@ This is platform specific; parses the data and interprets as a header for e.g. N start: 0xABC ``` +## `cpp` + +The `cpp` segment behaves the same as the `c` segment but uses the .cpp file extension (for C++ source files). + ## `data` **Description:** @@ -161,6 +165,10 @@ Data located in the ROM that is linked from a C file. Use the `.data` segment to **NOTE:** `splat` will not generate any `.data.s` files for these `.` (dot) sections. +## `.sdata` + +The `.sdata` segment behaves the same as the `.data` segment but supports "small data" linker sections named `.sdata`. + ## `rodata` **Description:** @@ -199,6 +207,10 @@ Read-only data located in the ROM, linked to a C file. Use the `.rodata` segment start: 0xABC ``` +## `.rdata` + +The `.rdata` segment behaves the same as the `.rodata` segment but supports rodata linker sections that happened to be named `.rdata` rather than `.rodata`. + **NOTE:** `splat` will not generate any `.rodata.s` files for these `.` (dot) sections. ## `bss` @@ -227,39 +239,34 @@ Links the `.bss` section of the associated `c` file. - { start: 0x7D1AD0, type: .bss, name: filepath, vram: 0x803C0420 } ``` -## Images +## `.sbss` -**Description:** +The `.sbss` segment behaves the same as the `.bss` segment but supports "small bss" linker sections named `.sbss`. -**splat** supports most of the [N64 image formats](https://n64squid.com/homebrew/n64-sdk/textures/image-formats/): - -- `i`, i.e. `i4` and `i8` -- `ia`, i.e. `ia4`, `ia8`, and `ia16` -- `ci`, i.e. `ci4` and `ci8` -- `rgb`, i.e. `rgba32` and `rgba16` +## `lib` -These segments will parse the image data and dump out a `png` file. +The `lib` segment can be used to link to a section of an object in an existing library. It is purely used to configure the output linker script and does not do any extraction. -**Note:** Using the dictionary syntax allows for richer configuration. +It looks for libraries in the [`lib_path`](https://github.com/ethteck/splat/wiki/Configuration#lib_path) global option. **Example:** +(link to .text of b_obj in a_lib) ```yaml -# as list -- [0xABC, i4, filename, width, height] -# as a dictionary -- name: filename - type: i4 - start: 0xABC - width: 64 - height: 64 - flip_x: yes - flip_y: no +- [auto, lib, a_lib, b_obj] ``` -`ci` (paletted) segments have a `palettes: []` setting that represents the list of palettes that should be linked to the `ci`. For each linked palette, an image will be exported. The implicit value of `palettes` is a one-element list containing the name of the raster, which means palettes and rasters with the same name will automatically be linked. +(link to .data of b_obj in a_lib) +```yaml +- [auto, lib, a_lib, b_obj, .data] +``` + +(dict representation) +(link to .text of b_obj in a_lib) +```yaml +- { type: lib, name: a_lib, object: b_obj, section: .text } +``` -Palette segments can specify a `global_id`, which can be referred to from a `ci`'s `palettes` list. The `global_id` space is searched first, and this allows cross-segment links between palettes and rasters. ## `pad` @@ -275,6 +282,9 @@ While this kind of segment can be represented by other segment types ([`asm`](#a - [0x00B250, pad, nops_00B250] ``` +**Warning:** `pad` cannot be the last segment in your yaml, as the way it is implemented requires a linked object to follow it. +If the rom contains padding at the end, we recommend treating only the non-padded portion of the rom with splat and padding the rest during the build process. + ## incbins incbin segments correpond to a family of segments used for extracting binary blobs. @@ -312,7 +322,118 @@ Used by certain compilers (like GCC) to store the Exception Handler Frame, used This frame contains more metadata used by exceptions at runtime. -## PS2 exclusive segments +## `linker_offset` + +This segment adds a symbol into the linker script at its relative section position. + +A segment named "john" with type `linker_offset` will cause a generated symbol with the name `john_OFFSET` to be placed into the linker script. +This can be useful for naming and referencing certain address locations from source code. + +# Platform-specific segments + +## N64 + +### Images + +**Description:** + +**splat** supports most of the [N64 image formats](https://n64squid.com/homebrew/n64-sdk/textures/image-formats/): + +- `i`, i.e. `i4` and `i8` +- `ia`, i.e. `ia4`, `ia8`, and `ia16` +- `ci`, i.e. `ci4` and `ci8` +- `rgb`, i.e. `rgba32` and `rgba16` + +These segments will parse the image data and dump out a `png` file. + +**Note:** Using the dictionary syntax allows for richer configuration. + +**Example:** + +```yaml +# as list +- [0xABC, i4, filename, width, height] +# as a dictionary +- name: filename + type: i4 + start: 0xABC + width: 64 + height: 64 + flip_x: yes + flip_y: no +``` + +`ci` (paletted) segments have a `palettes: []` setting that represents the list of palettes that should be linked to the `ci`. For each linked palette, an image will be exported. The implicit value of `palettes` is a one-element list containing the name of the raster, which means palettes and rasters with the same name will automatically be linked. + +Palette segments can specify a `global_id`, which can be referred to from a `ci`'s `palettes` list. The `global_id` space is searched first, and this allows cross-segment links between palettes and rasters. + +We recommend using [pigment64](https://github.com/decompals/pigment64) to convert extracted images back into original formats. + +### `gfx` + +`gfx` can be used to extract static f3dex ["display lists"](https://hackmd.io/@Roman971/Hk01jRxRr#Static-Data) into a .gfx.inc.c file, which is meant to be `#include`d from a source c file. + +These segments support an optional `data_only` attribute, which is False by default. If enabled, the extracted file will contain only the data rather than the enclosing symbol definition. + +Example output with `data_only` off (default): + +``` +Gfx displayList[] = { + gsDPPipeSync(), + gsDPSetPrimColor(0, 0, 0x80, 0x80, 0x80, 0x80), + gsDPSetEnvColor(0x80, 0x80, 0x80, 0x80), + gsSPEndDisplayList(), +}; +``` + +to be used in a source c file like +``` +#include "example.gfx.inc.c" +``` + +Example output with `data_only` on: +``` +gsDPPipeSync(), +gsDPSetPrimColor(0, 0, 0x80, 0x80, 0x80, 0x80), +gsDPSetEnvColor(0x80, 0x80, 0x80, 0x80), +gsSPEndDisplayList(), +``` + +to be used in a source c file like +``` +Gfx displayList[] = { #include "example.gfx.inc.c" }; +``` + +Some may prefer to define symbol names in source c files, rather than having splat be responsible for naming these symbols, which is why this option is provided. + +[Example usage](https://github.com/pmret/papermario/blob/c43d15e/ver/us/splat.yaml#L1707) + +### `vtx` + +`vtx` can be used to extract arrays of Vtx struct data, into a .vtx.inc.c file, which is meant to be `#include`d from a source c file. + +This option also supports the `data_only` attribute. See the section on the `gfx` segment for more details. + +[Example usage](https://github.com/pmret/papermario/blob/c43d15e/ver/us/splat.yaml#L1706) + +### `rsp` + +The `rsp` segment is used for disassembling RSP microcode. It is an extension of the `hasm` segment type and enables special instruction handling in the disassembler. + +### `ipl3` + +The `ipl3` segment is used for disassembling ipl3 code. It is an extension of the `hasm` segment type and opts out of standard symbol-tracking behavior, since it lives in an unconventional memory space. + +### Compressed segment types + +splat supports the compression types MIO0 and Yay0 with segment type names `mio0` and `yay0`, respectively. Both of these output a .bin file, which is expected to be re-compressed as part of the project's build system. +The generated linker script then will expect a .`type`.o file to exist. + +For example, for a `yay0` segment named "john", splat will create a decompressed john.bin file. The build system should then compress this file into `john.Yay0.bin` and then turn that into an object named `john.Yay0.o`, which will be linked into the output rom. + +We recommend using [crunch64](https://github.com/decompals/crunch64) to re-compress MIO0 and Yay0 assets that are extracted with splat. + +## PS2 ### `lit4` @@ -338,7 +459,7 @@ The disassembly of this section is tweaked to avoid confusing its data with othe The disassembly of this section is tweaked to avoid confusing its data with other types of data, this is because the disassembler can sometimes get confused and disassemble a pointer as a float, string, etc. -## General segment options +# General segment options All splat's segments can be passed extra options for finer configuration. Note that those extra options require to rewrite the entry using the dictionary yaml notation instead of the list one. diff --git a/pyproject.toml b/pyproject.toml index 7d2c2a8e..e31aa9c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "splat64" # Should be synced with src/splat/__init__.py -version = "0.26.2" +version = "0.27.0" description = "A binary splitting tool to assist with decompilation and modding projects" readme = "README.md" license = {file = "LICENSE"} diff --git a/src/splat/__init__.py b/src/splat/__init__.py index 6390e4ca..fce1fb00 100644 --- a/src/splat/__init__.py +++ b/src/splat/__init__.py @@ -1,7 +1,7 @@ __package_name__ = __name__ # Should be synced with pyproject.toml -__version__ = "0.26.2" +__version__ = "0.27.0" __author__ = "ethteck" from . import util as util diff --git a/src/splat/scripts/create_config.py b/src/splat/scripts/create_config.py index 4d504d5b..e63ef74f 100644 --- a/src/splat/scripts/create_config.py +++ b/src/splat/scripts/create_config.py @@ -67,7 +67,7 @@ def create_n64_config(rom_path: Path): asm_data_macro: dlabel # section_order: [".text", ".data", ".rodata", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] + # auto_link_sections: [".data", ".rodata", ".bss"] symbol_addrs_path: - symbol_addrs.txt @@ -184,7 +184,7 @@ def create_psx_config(exe_path: Path, exe_bytes: bytes): asm_data_macro: dlabel section_order: [".rodata", ".text", ".data", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] + # auto_link_sections: [".data", ".rodata", ".bss"] symbol_addrs_path: - symbol_addrs.txt diff --git a/src/splat/scripts/split.py b/src/splat/scripts/split.py index f5181922..8f5436f7 100644 --- a/src/splat/scripts/split.py +++ b/src/splat/scripts/split.py @@ -3,7 +3,6 @@ import argparse import hashlib import importlib -import pickle from typing import Any, Dict, List, Optional, Set, Tuple, Union from pathlib import Path @@ -41,7 +40,7 @@ def initialize_segments(config_segments: Union[dict, list]) -> List[Segment]: segment_rams = IntervalTree() segments_by_name: Dict[str, Segment] = {} - ret = [] + ret: List[Segment] = [] last_rom_end = 0 @@ -110,6 +109,9 @@ def initialize_segments(config_segments: Union[dict, list]) -> List[Segment]: segments_by_name[segment.given_follows_vram] ) + if ret[-1].type == "pad": + log.error("Last segment in config cannot be a pad segment; see https://github.com/ethteck/splat/wiki/Segments#pad") + return ret diff --git a/src/splat/segtypes/common/__init__.py b/src/splat/segtypes/common/__init__.py index fe020ee9..bf1ad25c 100644 --- a/src/splat/segtypes/common/__init__.py +++ b/src/splat/segtypes/common/__init__.py @@ -11,6 +11,7 @@ from . import hasm as hasm from . import header as header from . import lib as lib +from . import linker_offset as linker_offset from . import rdata as rdata from . import rodatabin as rodatabin from . import rodata as rodata diff --git a/src/splat/segtypes/common/c.py b/src/splat/segtypes/common/c.py index 3b3f069d..d7a214ee 100644 --- a/src/splat/segtypes/common/c.py +++ b/src/splat/segtypes/common/c.py @@ -171,7 +171,7 @@ def split(self, rom_bytes: bytes): if rodata_sibling is None: continue - if rodata_sibling.is_auto_all: + if rodata_sibling.is_generated: continue assert isinstance( diff --git a/src/splat/segtypes/common/code.py b/src/splat/segtypes/common/code.py index 0d96b68a..da896994 100644 --- a/src/splat/segtypes/common/code.py +++ b/src/splat/segtypes/common/code.py @@ -52,7 +52,7 @@ def vram_end(self) -> Optional[int]: else: return None - # Generates a placeholder segment for the auto_all_sections option + # Generates a placeholder segment for the auto_link_sections option def _generate_segment_from_all( self, rep_type: str, @@ -80,7 +80,7 @@ def _generate_segment_from_all( rep.given_symbol_name_format_no_rom = self.symbol_name_format_no_rom rep.sibling = base_seg rep.parent = self - rep.is_auto_all = True + rep.is_generated = True if rep.special_vram_segment: self.special_vram_segment = True rep.bss_contains_common = self.bss_contains_common @@ -92,7 +92,7 @@ def _insert_all_auto_sections( base_segments: OrderedDict[str, Segment], readonly_before: bool, ) -> List[Segment]: - if len(options.opts.auto_all_sections) == 0: + if len(options.opts.auto_link_sections) == 0: return ret base_segments_list: List[Tuple[str, Segment]] = list(base_segments.items()) @@ -107,7 +107,7 @@ def _insert_all_auto_sections( if seg.is_rodata(): last_inserted_index = i - for i, sect in enumerate(options.opts.auto_all_sections): + for i, sect in enumerate(options.opts.auto_link_sections): for name, seg in base_segments_list: link_section = seg.get_linker_section_order() if link_section == sect or link_section == "": @@ -133,7 +133,7 @@ def _insert_all_auto_sections( link_section = ret[last_inserted_index].get_linker_section_order() if ( link_section != "" - and link_section not in options.opts.auto_all_sections[: i + 1] + and link_section not in options.opts.auto_link_sections[: i + 1] ): last_inserted_index -= 1 if last_inserted_index < 0: diff --git a/src/splat/segtypes/n64/linker_offset.py b/src/splat/segtypes/common/linker_offset.py similarity index 91% rename from src/splat/segtypes/n64/linker_offset.py rename to src/splat/segtypes/common/linker_offset.py index 8af2e68e..e941715a 100644 --- a/src/splat/segtypes/n64/linker_offset.py +++ b/src/splat/segtypes/common/linker_offset.py @@ -1,7 +1,6 @@ from pathlib import Path from typing import List -from .segment import N64Segment from ..linker_entry import LinkerEntry, LinkerWriter from ..segment import Segment @@ -18,7 +17,7 @@ def emit_entry(self, linker_writer: LinkerWriter): linker_writer._write_symbol(f"{self.segment.get_cname()}_OFFSET", ".") -class N64SegLinker_offset(N64Segment): +class CommonSegLinker_offset(Segment): def get_linker_section_order(self) -> str: return "" diff --git a/src/splat/segtypes/n64/__init__.py b/src/splat/segtypes/n64/__init__.py index 0fca9c7f..90feef14 100644 --- a/src/splat/segtypes/n64/__init__.py +++ b/src/splat/segtypes/n64/__init__.py @@ -14,7 +14,6 @@ from . import ia8 as ia8 from . import img as img from . import ipl3 as ipl3 -from . import linker_offset as linker_offset from . import mio0 as mio0 from . import palette as palette from . import rgba16 as rgba16 diff --git a/src/splat/segtypes/n64/decompressor.py b/src/splat/segtypes/n64/decompressor.py index ec9b8645..1d992212 100644 --- a/src/splat/segtypes/n64/decompressor.py +++ b/src/splat/segtypes/n64/decompressor.py @@ -1,9 +1,9 @@ from ...util import log, options -from .segment import N64Segment +from ..segment import Segment -class CommonSegDecompressor(N64Segment): +class CommonSegDecompressor(Segment): def split(self, rom_bytes): out_dir = options.opts.asset_path / self.dir out_dir.mkdir(parents=True, exist_ok=True) @@ -33,7 +33,7 @@ def get_linker_entries(self): [options.opts.asset_path / self.dir / f"{self.name}.bin"], options.opts.asset_path / self.dir - / f"{self.name}.{self.compression_type}", # "Mio0" -> filename.Mio0.o + / f"{self.name}.{self.compression_type}", # "MIO0" -> filename.MIO0.o self.get_linker_section_order(), self.get_linker_section_linksection(), self.is_noload(), diff --git a/src/splat/segtypes/n64/img.py b/src/splat/segtypes/n64/img.py index 2ae054ab..edfb35aa 100644 --- a/src/splat/segtypes/n64/img.py +++ b/src/splat/segtypes/n64/img.py @@ -4,10 +4,10 @@ from n64img.image import Image from ...util import log, options -from .segment import N64Segment +from ..segment import Segment -class N64SegImg(N64Segment): +class N64SegImg(Segment): @staticmethod def parse_dimensions(yaml: Union[Dict, List]) -> Tuple[int, int]: if isinstance(yaml, dict): @@ -88,7 +88,7 @@ def split(self, rom_bytes): @staticmethod def estimate_size(yaml: Union[Dict, List]) -> int: width, height = N64SegImg.parse_dimensions(yaml) - typ = N64Segment.parse_segment_type(yaml) + typ = Segment.parse_segment_type(yaml) if typ == "ci4" or typ == "i4" or typ == "ia4": return width * height // 2 diff --git a/src/splat/segtypes/n64/ipl3.py b/src/splat/segtypes/n64/ipl3.py index 5bc8761f..810cd93a 100644 --- a/src/splat/segtypes/n64/ipl3.py +++ b/src/splat/segtypes/n64/ipl3.py @@ -1,4 +1,3 @@ -from ..common.code import CommonSegCode from ..common.hasm import CommonSegHasm diff --git a/src/splat/segtypes/n64/palette.py b/src/splat/segtypes/n64/palette.py index 740b1da4..046a7cd6 100644 --- a/src/splat/segtypes/n64/palette.py +++ b/src/splat/segtypes/n64/palette.py @@ -5,13 +5,13 @@ from ...util import log, options from ...util.color import unpack_color -from .segment import N64Segment +from ..segment import Segment VALID_SIZES = [0x20, 0x40, 0x80, 0x100, 0x200] -class N64SegPalette(N64Segment): +class N64SegPalette(Segment): require_unique_name = False def __init__(self, *args, **kwargs): diff --git a/src/splat/segtypes/n64/segment.py b/src/splat/segtypes/n64/segment.py deleted file mode 100644 index f71d89bb..00000000 --- a/src/splat/segtypes/n64/segment.py +++ /dev/null @@ -1,5 +0,0 @@ -from ..segment import Segment - - -class N64Segment(Segment): - pass diff --git a/src/splat/segtypes/segment.py b/src/splat/segtypes/segment.py index 4dd44a9c..f2e5067b 100644 --- a/src/splat/segtypes/segment.py +++ b/src/splat/segtypes/segment.py @@ -298,8 +298,8 @@ def __init__( yaml ) - # True if this segment was generated based on auto_all_sections - self.is_auto_all: bool = False + # True if this segment was generated based on auto_link_sections + self.is_generated: bool = False if self.rom_start is not None and self.rom_end is not None: if self.rom_start > self.rom_end: @@ -447,9 +447,8 @@ def symbol_name_format_no_rom(self) -> str: @property def subalign(self) -> Optional[int]: if self.parent: - return self.parent.subalign - else: - return self.given_subalign + log.error("subalign is not valid for non-top-level segments") + return self.given_subalign @property def vram_symbol(self) -> Optional[str]: diff --git a/src/splat/util/options.py b/src/splat/util/options.py index c331db92..76f3a59b 100644 --- a/src/splat/util/options.py +++ b/src/splat/util/options.py @@ -92,10 +92,8 @@ class SplatOpts: # Linker script # Determines the default subalign value to be specified in the generated linker script subalign: Optional[int] - # The following option determines whether to automatically configure the linker script to link against - # specified sections for all "base" (asm/c) files when the yaml doesn't have manual configurations - # for these sections. - auto_all_sections: List[str] + # The following option determines a list of sections for which automatic linker script entries should be added + auto_link_sections: List[str] # Determines the desired path to the linker script that splat will generate ld_script_path: Path # Determines the desired path to the linker symbol header, @@ -435,8 +433,8 @@ def parse_endianness() -> Literal["big", "little"]: lib_path=p.parse_path(base_path, "lib_path", "lib"), elf_section_list_path=p.parse_optional_path(base_path, "elf_section_list_path"), subalign=p.parse_optional_opt_with_default("subalign", int, 16), - auto_all_sections=p.parse_opt( - "auto_all_sections", list, [".data", ".rodata", ".bss"] + auto_link_sections=p.parse_opt( + "auto_link_sections", list, [".data", ".rodata", ".bss"] ), ld_script_path=p.parse_path(base_path, "ld_script_path", f"{basename}.ld"), ld_symbol_header_path=p.parse_optional_path(base_path, "ld_symbol_header_path"),