From 24f50c29fc2d1b463519cdfd7ce6052a653718c7 Mon Sep 17 00:00:00 2001 From: gbMichelle Date: Sat, 7 Mar 2020 21:06:34 +0100 Subject: [PATCH 1/6] Calculate crc of a tag on serialization. --- reclaimer/hek/defs/objs/tag.py | 49 +++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/reclaimer/hek/defs/objs/tag.py b/reclaimer/hek/defs/objs/tag.py index 919e3809..83be1429 100644 --- a/reclaimer/hek/defs/objs/tag.py +++ b/reclaimer/hek/defs/objs/tag.py @@ -6,21 +6,68 @@ # Reclaimer is free software under the GNU General Public License v3.0. # See LICENSE for more information. # +import zlib + +from pathlib import Path from supyr_struct.defs.constants import DEFAULT from supyr_struct.tag import Tag +def calc_halo_crc32(buffer, offset=None, size=None, crc=0xFFffFFff): + if offset is not None: + buffer.seek(offset) + + return zlib.crc32(buffer.read(size), crc ^ 0xFFffFFff) ^ 0xFFffFFff + class HekTag(Tag): def __init__(self, **kwargs): self.calc_pointers = False Tag.__init__(self, **kwargs) + def serialize(self, **kwargs): + ''' + Overload of the supyr serialization function that retroactively adds + a CRC to the tag. + ''' + filepath = kwargs.get('filepath', self.filepath) + buffer = kwargs.get('buffer', None) + + # Run the normal serialization. + result = Tag.serialize(self, **kwargs) + + # If there is neither a buffer or filepath just return the result. + if (buffer is None) and (not filepath): + return result + + # Prefer to use the buffer as that is how Tag.serialize does it. + f = buffer + if buffer is None: + f = Path(filepath).open('rb+') + + # Calculate the crc from the offset 64 to the end. 64 is the size of + # the header which is skipped in this calculation. + crc = calc_halo_crc32(f, 64) + # Write the crc to offset 40 in the buffer. + # Matches up with self.data.blam_header.checksum. + f.seek(40) + f.write(crc.to_bytes(4, byteorder='big', signed=False)) + # Flush the stream. + f.flush() + # Only close if it is a file. Because the only case where we own + # this buffer is if there was no buffer kwarg. + if not buffer: + f.close() + + # Update the tag object so it won't have to be deserialized again. + self.data.blam_header.checksum = crc + return result + def calc_internal_data(self): # recalculate the header data head = self.data.blam_header head.tag_class.data = head.tag_class.get_desc(DEFAULT) - head.checksum = head.get_desc(DEFAULT, 'checksum') + head.flags.edited_with_mozz = True head.header_size = head.get_desc(DEFAULT, 'header_size') head.version = head.get_desc(DEFAULT, 'version') head.integrity0 = head.get_desc(DEFAULT, 'integrity0') From a6a97b4c324a88ba45721d4d6e59f8345ff9a857 Mon Sep 17 00:00:00 2001 From: gbMichelle Date: Sat, 7 Mar 2020 21:09:55 +0100 Subject: [PATCH 2/6] Bump version --- CHANGELOG.MD | 4 ++++ reclaimer/__init__.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 1b120975..1bb45985 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.8.0] +### Added + - CRCs of tags are now properly calculated on save. + ## [2.7.3] ### Changed - (Linux) Fix wrong path separators used in C extensions. diff --git a/reclaimer/__init__.py b/reclaimer/__init__.py index 4b6e222e..2dc76f8b 100644 --- a/reclaimer/__init__.py +++ b/reclaimer/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Devin Bobadilla, Michelle van der Graaf" # YYYY.MM.DD -__date__ = "2020.02.18" -__version__ = (2, 7, 3) +__date__ = "2020.03.07" +__version__ = (2, 8, 0) __website__ = "https://github.com/Sigmmma/reclaimer" __all__ = ( "animation", "bitmaps", "h2", "h3", "halo_script", "hek", "meta", "misc", From a7cb2f8c667a7cc93adc579a4e141dad9d285f5e Mon Sep 17 00:00:00 2001 From: gbMichelle Date: Sat, 7 Mar 2020 21:28:16 +0100 Subject: [PATCH 3/6] Remove magic numbers --- reclaimer/hek/defs/objs/tag.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/reclaimer/hek/defs/objs/tag.py b/reclaimer/hek/defs/objs/tag.py index 83be1429..cc312532 100644 --- a/reclaimer/hek/defs/objs/tag.py +++ b/reclaimer/hek/defs/objs/tag.py @@ -29,6 +29,7 @@ def serialize(self, **kwargs): Overload of the supyr serialization function that retroactively adds a CRC to the tag. ''' + head = self.data.blam_header filepath = kwargs.get('filepath', self.filepath) buffer = kwargs.get('buffer', None) @@ -44,12 +45,12 @@ def serialize(self, **kwargs): if buffer is None: f = Path(filepath).open('rb+') - # Calculate the crc from the offset 64 to the end. 64 is the size of - # the header which is skipped in this calculation. - crc = calc_halo_crc32(f, 64) - # Write the crc to offset 40 in the buffer. - # Matches up with self.data.blam_header.checksum. - f.seek(40) + # Calculate the crc from after the header to the end. + crc = calc_halo_crc32(f, offset=head.get_desc('SIZE')) + # Write the crc to the offset of the checksum value in the header. + # The way we retrieve this offset from supyr is insane. + attr_index = head.get_desc('NAME_MAP')['checksum'] + f.seek(head.get_desc('ATTR_OFFS')[attr_index]) f.write(crc.to_bytes(4, byteorder='big', signed=False)) # Flush the stream. f.flush() @@ -59,7 +60,7 @@ def serialize(self, **kwargs): f.close() # Update the tag object so it won't have to be deserialized again. - self.data.blam_header.checksum = crc + head.checksum = crc return result def calc_internal_data(self): From 785b03d8050d49ce4ff46b200416b44a6dcd2b4c Mon Sep 17 00:00:00 2001 From: gbMichelle Date: Sat, 7 Mar 2020 21:37:23 +0100 Subject: [PATCH 4/6] Update changelog --- CHANGELOG.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 1bb45985..fc19b854 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [2.8.0] ### Added - - CRCs of tags are now properly calculated on save. + - CRCs of tags are now properly calculated on save. Use the first bit on offset 48 to determine if a tag is edited by reclaimer. (If not using this library.) ## [2.7.3] ### Changed From 1f94627da5125cce792f2e28c24dbf202cc7ea4d Mon Sep 17 00:00:00 2001 From: gbMichelle Date: Sat, 7 Mar 2020 21:43:16 +0100 Subject: [PATCH 5/6] Have default CRC be 0 so it is easier to spot one that isn't calculated --- reclaimer/common_descs.py | 4 ++-- reclaimer/h2/common_descs.py | 2 +- reclaimer/h3/common_descs.py | 2 +- reclaimer/shadowrun_prototype/common_descs.py | 2 +- reclaimer/stubbs/common_descs.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/reclaimer/common_descs.py b/reclaimer/common_descs.py index d0d33368..f0782c9e 100644 --- a/reclaimer/common_descs.py +++ b/reclaimer/common_descs.py @@ -565,7 +565,7 @@ def anim_src_func_per_pha_sca_rot_macro(name, **desc): UEnum32("tag_class", GUI_NAME="tag_class", INCLUDE=valid_tags, EDITABLE=False ), - UInt32("checksum", DEFAULT=0x4D6F7A7A, EDITABLE=False), + UInt32("checksum", DEFAULT=0, EDITABLE=False), UInt32("header_size", DEFAULT=64, EDITABLE=False), BBool64("flags", "edited_with_mozz", @@ -813,7 +813,7 @@ def blam_header_os(tagid, version=1): UEnum32("tag_class", GUI_NAME="tag_class", INCLUDE=valid_tags_os, EDITABLE=False ), - UInt32("checksum", DEFAULT=0x4D6F7A7A, EDITABLE=False), + UInt32("checksum", DEFAULT=0, EDITABLE=False), UInt32("header_size", DEFAULT=64, EDITABLE=False), Bool64("flags", "edited_with_mozz", diff --git a/reclaimer/h2/common_descs.py b/reclaimer/h2/common_descs.py index 7f8f663a..cb8225d2 100644 --- a/reclaimer/h2/common_descs.py +++ b/reclaimer/h2/common_descs.py @@ -244,7 +244,7 @@ def h2_blam_header(tagid, version=1): UEnum32("tag class", GUI_NAME="tag class", INCLUDE=valid_h2_tags, EDITABLE=False ), - UInt32("checksum", DEFAULT=0x4D6F7A7A, EDITABLE=False), + UInt32("checksum", DEFAULT=0, EDITABLE=False), UInt32("header size", DEFAULT=64, EDITABLE=False), Bool64("flags", "edited with mozz", diff --git a/reclaimer/h3/common_descs.py b/reclaimer/h3/common_descs.py index 9cd92d2c..8864ba77 100644 --- a/reclaimer/h3/common_descs.py +++ b/reclaimer/h3/common_descs.py @@ -132,7 +132,7 @@ def h3_blam_header(tagid, version=1): UEnum32("tag class", GUI_NAME="tag class", INCLUDE=valid_h3_tags, EDITABLE=False ), - UInt32("checksum", DEFAULT=0x4D6F7A7A, EDITABLE=False), + UInt32("checksum", DEFAULT=0, EDITABLE=False), UInt32("header size", DEFAULT=64, EDITABLE=False), Bool64("flags", "edited with mozz", diff --git a/reclaimer/shadowrun_prototype/common_descs.py b/reclaimer/shadowrun_prototype/common_descs.py index cb87f4d2..c7b359b5 100644 --- a/reclaimer/shadowrun_prototype/common_descs.py +++ b/reclaimer/shadowrun_prototype/common_descs.py @@ -57,7 +57,7 @@ def blam_header(tagid, version=1): UEnum32("tag_class", GUI_NAME="tag_class", INCLUDE=sr_valid_tags, EDITABLE=False ), - UInt32("checksum", DEFAULT=0x4D6F7A7A, EDITABLE=False), + UInt32("checksum", DEFAULT=0, EDITABLE=False), UInt32("header_size", DEFAULT=64, EDITABLE=False), BBool64("flags", "edited_with_mozz", diff --git a/reclaimer/stubbs/common_descs.py b/reclaimer/stubbs/common_descs.py index aec8252f..958dd631 100644 --- a/reclaimer/stubbs/common_descs.py +++ b/reclaimer/stubbs/common_descs.py @@ -172,7 +172,7 @@ def blam_header_stubbs(tagid, version=1): UEnum32("tag_class", GUI_NAME="tag_class", INCLUDE=stubbs_valid_tags, EDITABLE=False ), - UInt32("checksum", DEFAULT=0x4D6F7A7A, EDITABLE=False), + UInt32("checksum", DEFAULT=0, EDITABLE=False), UInt32("header_size", DEFAULT=64, EDITABLE=False), BBool64("flags", "edited_with_mozz", From 16700976d6def0be132ee035d81db329e1cf53f9 Mon Sep 17 00:00:00 2001 From: gbMichelle Date: Sun, 8 Mar 2020 00:51:10 +0100 Subject: [PATCH 6/6] Move calc_halo_crc to util --- reclaimer/hek/defs/objs/tag.py | 8 +------- reclaimer/util/__init__.py | 7 +++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/reclaimer/hek/defs/objs/tag.py b/reclaimer/hek/defs/objs/tag.py index cc312532..889037c6 100644 --- a/reclaimer/hek/defs/objs/tag.py +++ b/reclaimer/hek/defs/objs/tag.py @@ -6,18 +6,12 @@ # Reclaimer is free software under the GNU General Public License v3.0. # See LICENSE for more information. # -import zlib - from pathlib import Path from supyr_struct.defs.constants import DEFAULT from supyr_struct.tag import Tag -def calc_halo_crc32(buffer, offset=None, size=None, crc=0xFFffFFff): - if offset is not None: - buffer.seek(offset) - - return zlib.crc32(buffer.read(size), crc ^ 0xFFffFFff) ^ 0xFFffFFff +from reclaimer.util import calc_halo_crc32 class HekTag(Tag): def __init__(self, **kwargs): diff --git a/reclaimer/util/__init__.py b/reclaimer/util/__init__.py index c02072db..323a0ea9 100644 --- a/reclaimer/util/__init__.py +++ b/reclaimer/util/__init__.py @@ -1,4 +1,5 @@ import re +import zlib from math import log @@ -124,4 +125,10 @@ def is_valid_ascii_name_str(string): return False return True +def calc_halo_crc32(buffer, offset=None, size=None, crc=0xFFffFFff): + if offset is not None: + buffer.seek(offset) + + return zlib.crc32(buffer.read(size), crc ^ 0xFFffFFff) ^ 0xFFffFFff + del re