CI Revamp (#338)

* WIP

* bump spimdisasm ver

* Update test

* some suggestions + global id rename

* halfway

* docs & cleanup

* Clarified loss of

* whoopz

* Fix error

* feedback

* suggz

CHANGELOG.md

@@ -1,5 +1,14 @@
 # splat Release Notes
 
+### 0.22.0: Palette Revamp
+
+* The N64 ci/palette system has been rewritten to be more versatile and support a larger variety of configurations.
+  * ci segments now have a "palettes:" argument, which can be a list of palettes or a single palette to be linked to the ci for extraction. The implicit value of `palettes:` is a one-element list containing the name of the ci, meaning palettes whose names match a ci will automatically be linked to the ci. Each palette linked to a ci will result in a separate png.
+  * the `raster_name` field on palettes and the `palette` field on rasters no longer exist. Instead, rasters point to palettes via the `palettes:` property of the ci segment (or the final argument after width and height, if using list format).
+  * palette segments can provide a `global_id` field, which serves as a globally searchable palette id. This can be used for cross-segment ci/palette linking.
+  * added option `image_type_in_extension`, which puts the type of an image in the file extension. For example, with the setting enabled, an image named `texture` would export with filename `texture.ci4.png`. 
+* `spimdisasm` 1.21.0 or above is now required.
+
 ### 0.21.12
 
 * Fixed issue that prevented symbols from being added to undefined_funcs_auto

README.md

@@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.21.12,<1.0.0
+splat64[mips]>=0.22.0,<1.0.0
 ```
 
 ### Optional dependencies

docs/Configuration.md

@@ -648,6 +648,10 @@ Use named libultra symbols by default. Those will need to be added to a linker s
 
 Use named hardware register symbols by default. Those will need to be added to a linker script manually by the user
 
+### image_type_in_extension
+
+Append the type of an image to its file extension. For example, when enabled, a ci4 named `texture` would export with filename `texture.ci4.png`.
+
 ## Compiler-specific options
 
 ### use_legacy_include_asm

docs/Segments.md

@@ -257,6 +257,10 @@ These segments will parse the image data and dump out a `png` file.
   flip_y: no
 ```
 
+`ci` (paletted) segments have a `palettes: []` setting that represents the list of palettes that should be linked to the `ci`. For each linked palette, an image will be exported. The implicit value of `palettes` is a one-element list containing the name of the raster, which means palettes and rasters with the same name will automatically be linked.
+
+Palette segments can specify a `global_id`, which can be referred to from a `ci`'s `palettes` list. The `global_id` space is searched first, and this allows cross-segment links between palettes and rasters.
+
 ## `pad`
 
 `pad` is a segment that represents a rom region that's filled with zeroes and decomping it doesn't have much value.

mypy.ini

@@ -1,4 +1,3 @@
 [mypy]
 ignore_missing_imports = True
 check_untyped_defs = True
-mypy_path = stubs

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "splat64"
 # Should be synced with src/splat/__init__.py
-version = "0.21.12"
+version = "0.22.0"
 description = "A binary splitting tool to assist with decompilation and modding projects"
 readme = "README.md"
 license = {file = "LICENSE"}
@@ -20,7 +20,7 @@ dependencies = [
 
 [project.optional-dependencies]
 mips = [
-    "spimdisasm>=1.18.0,<2.0.0", # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
+    "spimdisasm>=1.21.0,<2.0.0", # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
     "rabbitizer>=1.8.0,<2.0.0",
     "pygfxd",
     "n64img>=0.1.4",
@@ -31,6 +31,7 @@ dev = [
     "mypy",
     "black",
     "types-PyYAML",
+    "types-colorama",
 ]
 
 [project.urls]

requirements.txt

@@ -4,7 +4,7 @@ tqdm
 intervaltree
 colorama
 # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
-spimdisasm>=1.18.0
+spimdisasm>=1.21.0
 rabbitizer>=1.8.0
 pygfxd
 n64img>=0.1.4

src/splat/__init__.py

@@ -3,8 +3,7 @@
 __package_name__ = __name__
 
 # Should be synced with pyproject.toml
-__version_info__: Tuple[int, int, int] = (0, 21, 12)
-__version__ = ".".join(map(str, __version_info__))
+__version__ = "0.22.0"
 __author__ = "ethteck"
 
 from . import util as util

src/splat/disassembler/spimdisasm_disassembler.py

@@ -7,7 +7,7 @@
 
 class SpimdisasmDisassembler(disassembler.Disassembler):
     # This value should be kept in sync with the version listed on requirements.txt
-    SPIMDISASM_MIN = (1, 18, 0)
+    SPIMDISASM_MIN = (1, 21, 0)
 
     def configure(self):
         # Configure spimdisasm

src/splat/segtypes/n64/ci.py

@@ -1,6 +1,7 @@
-from typing import Optional, TYPE_CHECKING
+from pathlib import Path
+from typing import List, TYPE_CHECKING
 
-from ...util import log
+from ...util import log, options
 
 from .img import N64SegImg
 
@@ -10,33 +11,59 @@
 
 # Base class for CI4/CI8
 class N64SegCi(N64SegImg):
-    def parse_palette_name(self, yaml, args) -> str:
-        ret = self.name
+    def parse_palette_names(self, yaml, args) -> List[str]:
+        ret = [self.name]
         if isinstance(yaml, dict):
-            if "palette" in yaml:
-                ret = yaml["palette"]
+            if "palettes" in yaml:
+                ret = yaml["palettes"]
         elif len(args) > 2:
             ret = args[2]
 
+        if isinstance(ret, str):
+            ret = [ret]
         return ret
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.palette: "Optional[N64SegPalette]" = None
-        self.palette_name = self.parse_palette_name(self.yaml, self.args)
+        self.palettes: "List[N64SegPalette]" = []
+        self.palette_names = self.parse_palette_names(self.yaml, self.args)
 
     def scan(self, rom_bytes: bytes) -> None:
         self.n64img.data = rom_bytes[self.rom_start : self.rom_end]
 
+    def out_path_pal(self, pal_name) -> Path:
+        type_extension = f".{self.type}" if options.opts.image_type_in_extension else ""
+
+        if len(self.palettes) == 1:
+            # If there's only one palette, use the ci name
+            out_name = self.name
+        elif pal_name.startswith(self.name):
+            # Otherwise, if the palette name starts with / equals the ci name, use that
+            out_name = pal_name
+        else:
+            # Otherwise, just append the palette name to the ci name
+            out_name = f"{self.name}_{pal_name}"
+
+        return options.opts.asset_path / self.dir / f"{out_name}{type_extension}.png"
+
     def split(self, rom_bytes):
-        if self.palette is None:
+        assert self.palettes is not None
+        if len(self.palettes) == 0:
             # TODO: output with blank palette
             log.error(
-                f"no palette sibling segment exists\n(hint: add a segment with type 'palette' and name '{self.name}')"
+                f"no palettes have been mapped to ci segment `{self.name}`\n(hint: add a palette segment with the same name or use the `palettes:` field of this segment to specify palettes by name')"
             )
-        assert self.palette is not None
-        self.palette.extract = False
-        self.n64img.palette = self.palette.parse_palette(rom_bytes)
 
-        super().split(rom_bytes)
+        assert isinstance(self.rom_start, int)
+        assert isinstance(self.rom_end, int)
+        self.n64img.data = rom_bytes[self.rom_start : self.rom_end]
+
+        for palette in self.palettes:
+            path = self.out_path_pal(palette.name)
+            path.parent.mkdir(parents=True, exist_ok=True)
+
+            self.n64img.palette = palette.parse_palette(rom_bytes)
+            self.n64img.write(path)
+
+            self.log(f"Wrote {path.name} to {path}")

src/splat/segtypes/n64/img.py

@@ -66,7 +66,9 @@ def check_len(self) -> None:
             )
 
     def out_path(self) -> Path:
-        return options.opts.asset_path / self.dir / f"{self.name}.png"
+        type_extension = f".{self.type}" if options.opts.image_type_in_extension else ""
+
+        return options.opts.asset_path / self.dir / f"{self.name}{type_extension}.png"
 
     def should_split(self) -> bool:
         return options.opts.is_mode_active("img")

src/splat/segtypes/n64/palette.py

@@ -1,20 +1,11 @@
 from itertools import zip_longest
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, TYPE_CHECKING, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from ...util import log, options
 from ...util.color import unpack_color
 
 from .segment import N64Segment
-from ...util.symbols import to_cname
-
-if TYPE_CHECKING:
-    from .ci import N64SegCi as Raster
-
-
-def iter_in_groups(iterable, n, fillvalue=None):
-    args = [iter(iterable)] * n
-    return zip_longest(*args, fillvalue=fillvalue)
 
 
 VALID_SIZES = [0x20, 0x40, 0x80, 0x100, 0x200]
@@ -26,18 +17,6 @@ class N64SegPalette(N64Segment):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.raster: "Optional[Raster]" = None
-
-        # palette segments must be named as one of the following:
-        #  1) same as the relevant raster segment name (max. 1 palette)
-        #  2) relevant raster segment name + "." + unique palette name
-        #  3) unique, referencing the relevant raster segment using `raster_name`
-        self.raster_name = (
-            self.yaml.get("raster_name", self.name.split(".")[0])
-            if isinstance(self.yaml, dict)
-            else self.name.split(".")[0]
-        )
-
         if self.extract:
             if self.rom_end is None:
                 log.error(
@@ -61,38 +40,38 @@ def __init__(self, *args, **kwargs):
                         f"Error: {self.name} (0x{actual_len:X} bytes) is not a valid palette size ({', '.join(hex(s) for s in VALID_SIZES)})\n{hint_msg}"
                     )
 
+        self.global_id: Optional[str] = (
+            self.yaml.get("global_id") if isinstance(self.yaml, dict) else None
+        )
+
     def get_cname(self) -> str:
         return super().get_cname() + "_pal"
 
-    def split(self, rom_bytes):
-        path = self.out_path()
-        path.parent.mkdir(parents=True, exist_ok=True)
-
-        if self.raster is None:
-            # TODO: output with no raster
-            log.error(f"orphaned palette segment: {self.name} lacks ci4/ci8 sibling")
-
-        assert self.raster is not None
-        self.raster.n64img.palette = self.parse_palette(rom_bytes)  # type: ignore
-
-        self.raster.n64img.write(path)
-        self.raster.extract = False
+    @staticmethod
+    def parse_palette_bytes(data) -> List[Tuple[int, int, int, int]]:
+        def iter_in_groups(iterable, n, fillvalue=None):
+            args = [iter(iterable)] * n
+            return zip_longest(*args, fillvalue=fillvalue)
 
-    def parse_palette(self, rom_bytes) -> List[Tuple[int, int, int, int]]:
-        data = rom_bytes[self.rom_start : self.rom_end]
         palette = []
 
         for a, b in iter_in_groups(data, 2):
             palette.append(unpack_color([a, b]))
 
         return palette
 
-    def out_path(self) -> Path:
-        return options.opts.asset_path / self.dir / f"{self.name}.png"
+    def parse_palette(self, rom_bytes) -> List[Tuple[int, int, int, int]]:
+        data = rom_bytes[self.rom_start : self.rom_end]
+
+        return N64SegPalette.parse_palette_bytes(data)
 
     def should_split(self) -> bool:
         return self.extract and options.opts.is_mode_active("img")
 
+    def out_path(self) -> Path:
+        return options.opts.asset_path / self.dir / f"{self.name}.png"
+
+    # TODO NEED NAMES...
     def get_linker_entries(self):
         from ..linker_entry import LinkerEntry
 

src/splat/util/__init__.py

@@ -7,7 +7,6 @@
 from . import palettes as palettes
 from . import progress_bar as progress_bar
 from . import psx as psx
-from . import range as range
 from . import relocs as relocs
 from . import statistics as statistics
 from . import symbols as symbols

src/splat/util/options.py

@@ -233,6 +233,8 @@ class SplatOpts:
     ique_symbols: bool
     # Use named hardware register symbols by default. Those will need to be added to a linker script manually by the user
     hardware_regs: bool
+    # Append the image type to the output file extension
+    image_type_in_extension: bool
 
     ################################################################################
     # Compiler-specific options
@@ -520,6 +522,7 @@ def parse_endianness() -> Literal["big", "little"]:
         libultra_symbols=p.parse_opt("libultra_symbols", bool, False),
         ique_symbols=p.parse_opt("ique_symbols", bool, False),
         hardware_regs=p.parse_opt("hardware_regs", bool, False),
+        image_type_in_extension=p.parse_opt("image_type_in_extension", bool, False),
         use_legacy_include_asm=p.parse_opt("use_legacy_include_asm", bool, True),
         disasm_unknown=p.parse_opt("disasm_unknown", bool, False),
         detect_redundant_function_end=p.parse_opt(