diff --git a/AUTHORS.rst b/AUTHORS.rst index 101f631..25a301b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -24,6 +24,7 @@ Contributors: * Stefan Schmitt * Theo Sbrissa * Zachary Clifford +* Piotr Korowacki * "durexyl" @ GitHub * "erki1993" @ GitHub * "mentaal" @ GitHub diff --git a/NEWS.rst b/NEWS.rst index b174c70..12cbba5 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -2,6 +2,13 @@ IntelHex releases ***************** +2.X (2020-XX-XX) +---------------- +* API changes: ``IntelHex.write_hex_file`` method: add support for new + parameter: ``ext_addr_mode = linear | segment | none | auto``. + Option dicedes how Extended Address records are resolved. + Default value is ``linear`` for backward comaptibility. (Piotr Korowacki) + 2.3.0 (2020-10-20) ------------------ * Add ``IntelHex.find()`` method to find a given byte pattern. (Scott Armitage) diff --git a/docs/manual/part2-5.txt b/docs/manual/part2-5.txt index 6b5dd7d..e052fea 100644 --- a/docs/manual/part2-5.txt +++ b/docs/manual/part2-5.txt @@ -6,7 +6,7 @@ including HEX, bin, or python dictionaries. You can write out HEX data contained in object by method ``.write_hex_file(f)``. Parameter ``f`` should be filename or file-like object. Note that this can include builtins like sys.stdout. -Also you can use the universal tofile. +Also you can use the universal ``tofile``. To convert data of IntelHex object to HEX8 file format without actually saving it to disk you can use the builtin StringIO file-like object, e.g.:: @@ -25,10 +25,37 @@ Variable ``hexstr`` will contain a string with the content of a HEX8 file. You can customize hex file output with following optional arguments to ``write_hex_file`` call: - * ``write_start_addr`` - you can disable start address record in new hex file; + * ``write_start_addr`` - you can disable start address record in new hex file. * ``eolstyle`` - you can force ``CRLF`` line endings in new hex file. * ``byte_count`` - you can control how many bytes should be written to each data record. + * ``ext_addr_mode`` - you can decide how extended address records should be + resolved and written in new hex file (explained below). + +Extended Address records are describing an address in binary space from which +a block of data is being written. Normally without those record we could write +only 64KB of binary data into one single ``.hex`` file. This is because in +one signle data record there are only 2 bytes for address. To have possibility +to write more, Extended Records are needed to increase address resolution. +Currently there are two types of records defined in IntelHex format: + + * ``Extended Segment Address`` [02] - you can write up to 1MB of binary data + * ``Extended Linear Address`` [04] - you can write up to 4GB of binary data + +There are 4 modes given by ``ext_addr_mode`` parameter to support every type +of memory address resolution: + + * ``linear`` - forces to use Extended Linear Address records, + in most of cases this mode is commonly desired (default). + * ``segment`` - forces to use Extended Segment Address records. + * ``none`` - forces to not use any Extended Address records (legacy option). + * ``auto`` - automatically decides which mode to use using last known adress + where data need to be written. + +Whenever data overflow for different adres resoution is detected adequat +exception will thrown. Mixing of ``linear`` and ``segment`` records isn't +allowed. There won't be any Extened Address records will written in +any mode if data to write need to be placed in address under 64KB. Data converters diff --git a/intelhex/__init__.py b/intelhex/__init__.py index c6423a6..df2cef4 100644 --- a/intelhex/__init__.py +++ b/intelhex/__init__.py @@ -544,7 +544,7 @@ def _get_eol_textfile(eolstyle, platform): raise ValueError("wrong eolstyle %s" % repr(eolstyle)) _get_eol_textfile = staticmethod(_get_eol_textfile) - def write_hex_file(self, f, write_start_addr=True, eolstyle='native', byte_count=16): + def write_hex_file(self, f, write_start_addr=True, eolstyle='native', byte_count=16, ext_addr_mode='linear'): """Write data to file f in HEX format. @param f filename or file-like object for writing @@ -556,20 +556,55 @@ def write_hex_file(self, f, write_start_addr=True, eolstyle='native', byte_count for output file on different platforms. Supported eol styles: 'native', 'CRLF'. @param byte_count number of bytes in the data field + @param ext_addr_mode used this to decide which record type to + to write for Extended Address records: + Linear (32-bit addressing) or Segment (20-bit addressing) + or without Extended Address records (16-bit addressing) + Supported modes: 'linear', 'segment', 'none', 'auto'. """ if byte_count > 255 or byte_count < 1: raise ValueError("wrong byte_count value: %s" % byte_count) - fwrite = getattr(f, "write", None) - if fwrite: - fobj = f - fclose = None - else: - fobj = open(f, 'w') - fwrite = fobj.write - fclose = fobj.close eol = IntelHex._get_eol_textfile(eolstyle, sys.platform) + addresses = dict_keys(self._buf) + addresses.sort() + addr_len = len(addresses) + minaddr = addresses[0] if addr_len else 0 + maxaddr = addresses[-1] if addr_len else 0 + + # make parameter case-insensitive + ext_addr_mode = ext_addr_mode.lower() + # resolve extended address type + if ext_addr_mode == 'linear': + # enforces Extended Linear Address record type (default) + extaddr_rectyp = 4 + elif ext_addr_mode == 'segment': + # enforces Extended Segment Address record type + extaddr_rectyp = 2 + elif ext_addr_mode == 'none': + # enforces no Extended Address records format + extaddr_rectyp = 0 + elif ext_addr_mode == 'auto': + # Extended Address record type is resolved by Max Address + if maxaddr > 0x0FFFFF: + extaddr_rectyp = 4 + else: # 1MB sapcing is max for Segement + extaddr_rectyp = 2 + else: + raise ValueError('ext_addr_mode should be one of:' + ' "linear", "segment", "none", "auto";' + ' got %r instead' % ext_addr_mode) + + # check max address with resolved format + if addr_len: + if extaddr_rectyp == 4 and maxaddr > 0xFFFFFFFF: + raise LinearSpacingOverflowError(overflwd_len=(maxaddr-0xFFFFFFFF)) + elif extaddr_rectyp == 2 and maxaddr > 0x0FFFFF: + raise SegmentSpacingOverflowError(overflwd_len=(maxaddr-0x0FFFFF)) + elif extaddr_rectyp == 0 and maxaddr > 0xFFFF: + raise BasicSpacingOverflowError(overflwd_len=(maxaddr-0xFFFF)) + # Translation table for uppercasing hex ascii string. # timeit shows that using hexstr.translate(table) # is faster than hexstr.upper(): @@ -581,6 +616,15 @@ def write_hex_file(self, f, write_start_addr=True, eolstyle='native', byte_count # Python 2 table = ''.join(chr(i).upper() for i in range_g(256)) + fwrite = getattr(f, "write", None) + if fwrite: + fobj = f + fclose = None + else: + fobj = open(f, 'w') + fwrite = fobj.write + fclose = fobj.close + # start address record if any if self.start_addr and write_start_addr: keys = dict_keys(self.start_addr) @@ -623,19 +667,13 @@ def write_hex_file(self, f, write_start_addr=True, eolstyle='native', byte_count raise InvalidStartAddressValueError(start_addr=self.start_addr) # data - addresses = dict_keys(self._buf) - addresses.sort() - addr_len = len(addresses) if addr_len: - minaddr = addresses[0] - maxaddr = addresses[-1] - if maxaddr > 65535: need_offset_record = True else: need_offset_record = False - high_ofs = 0 + high_ofs = 0 cur_addr = minaddr cur_ix = 0 @@ -645,9 +683,15 @@ def write_hex_file(self, f, write_start_addr=True, eolstyle='native', byte_count bin[0] = 2 # reclen bin[1] = 0 # offset msb bin[2] = 0 # offset lsb - bin[3] = 4 # rectyp - high_ofs = int(cur_addr>>16) + bin[3] = extaddr_rectyp # rectyp + + if extaddr_rectyp == 4: + high_ofs = int(cur_addr>>16) + else: # extaddr_rectyp == 2: + # 0x00X0000 => 0xX000 + high_ofs = int((cur_addr & 0x00F0000) >> 4) b = divmod(high_ofs, 256) + bin[4] = b[0] # msb of high_ofs bin[5] = b[1] # lsb of high_ofs bin[6] = (-sum(bin)) & 0x0FF # chksum @@ -1096,7 +1140,7 @@ def bin2hex(fin, fout, offset=0): return 1 try: - h.tofile(fout, format='hex') + h.write_hex_file(fout) except IOError: e = sys.exc_info()[1] # current exception txt = "ERROR: Could not write to file: %s: %s" % (fout, str(e)) @@ -1281,6 +1325,10 @@ def ascii_hex_to_int(ascii): # BadAccess16bit - not enough data to read 16 bit value (deprecated, see NotEnoughDataError) # NotEnoughDataError - not enough data to read N contiguous bytes # EmptyIntelHexError - requested operation cannot be performed with empty object +# OverflowError - data overflow general error +# LinearSpacingOverflowError - 32-bit address spacing data overflow +# SegmentSpacingOverflowError - 20-bit address spacing data overflow +# BasicSpacingOverflowError - 16-bit address spacing data overflow class IntelHexError(Exception): '''Base Exception class for IntelHex module''' @@ -1370,3 +1418,16 @@ class BadAccess16bit(NotEnoughDataError): class EmptyIntelHexError(IntelHexError): _fmt = "Requested operation cannot be executed with empty object" + + +class OverflowError(IntelHexError): + _fmt = 'Base class for overflowed data' + +class LinearSpacingOverflowError(OverflowError): + _fmt = 'Data overflow Linear (32-bit) address spacing. Overflowed data: %(overflwd_len)d bytes' + +class SegmentSpacingOverflowError(OverflowError): + _fmt = 'Data overflow Segment (20-bit) address spacing. Overflowed data: %(overflwd_len)d bytes' + +class BasicSpacingOverflowError(OverflowError): + _fmt = 'Data overflow 16-bit address spacing. Overflowed data: %(overflwd_len)d bytes' diff --git a/intelhex/test.py b/intelhex/test.py index 50276b0..3e53572 100755 --- a/intelhex/test.py +++ b/intelhex/test.py @@ -62,6 +62,9 @@ BadAccess16bit, hex2bin, Record, + LinearSpacingOverflowError, + SegmentSpacingOverflowError, + BasicSpacingOverflowError, ) from intelhex import compat from intelhex.compat import ( @@ -333,6 +336,14 @@ """ data64k = {0: 1, 0x10000: 2} +hex16M = """:020000040000FA +:0100000001FE +:020000040100F9 +:0100000002FD +:00000001FF +""" +data_hex16M = {0: 1, 0x100000: 2} + hex_rectype3 = """:0400000312345678E5 :0100000001FE @@ -365,7 +376,6 @@ :00000001FF """ - ## # Test cases @@ -1793,6 +1803,264 @@ def test_write_hex_file_byte_count_255(self): self.assertEqual(ih.tobinstr(), ih2.tobinstr(), "Written hex file does not equal with original") +class TestWriteHexFileExtAddrMode(unittest.TestCase): + + def test_write_hex_file_ext_addr_mode_param(self): + ih = intelhex.IntelHex() + sio = StringIO() + + # invalid param + self.assertRaises(ValueError, ih.write_hex_file, sio, ext_addr_mode='bad') + + # insesitive casing check + isValueError = False + try: + ih.write_hex_file(sio, ext_addr_mode='AUTO') + except ValueError: + isValueError = True + self.assertFalse(isValueError) + + isValueError = False + try: + ih.write_hex_file(sio, ext_addr_mode='Linear') + except ValueError: + isValueError = True + self.assertFalse(isValueError) + + isValueError = False + try: + ih.write_hex_file(sio, ext_addr_mode='segmenT') + except ValueError: + isValueError = True + self.assertFalse(isValueError) + + isValueError = False + try: + ih.write_hex_file(sio, ext_addr_mode='NoNe') + except ValueError: + isValueError = True + self.assertFalse(isValueError) + + sio.close() + + def test_write_hex_file_no_starting_address_simple(self): + #prepare + ih = intelhex.IntelHex(StringIO(hex_simple)) + + #check 'auto' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='auto') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, hex_simple, + "Written hex file does not equal with original") + sio.close() + + #check 'segment' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='segment') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, hex_simple, + "Written hex file does not equal with original") + sio.close() + + #check 'linear' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='linear') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, hex_simple, + "Written hex file does not equal with original") + sio.close() + + #check 'none' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='none') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, hex_simple, + "Written hex file does not equal with original") + sio.close() + + def test_write_hex_file_segment_address_type(self): + #prepare + ih_records = [Record.start_segment_address(0x1234, 0x5678), + Record.extended_segment_address(0x3000), + Record.data(0x0, [0xAB, 0xCD]), + Record.eof()] + ih_records_str = '\n'.join(ih_records)+'\n' + ih = intelhex.IntelHex(StringIO(ih_records_str)) + + ih_records_segment_expected_str = ih_records_str + #ih_records won't be needed anymore, so it can be modified + ih_records[1] = Record.extended_linear_address(0x3) + ih_records_linear_expected_str = '\n'.join(ih_records)+'\n' + + #check 'auto' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='auto') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, ih_records_segment_expected_str, + "Written hex file does not equal with expected") + sio.close() + + #check 'segment' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='segment') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, ih_records_segment_expected_str, + "Written hex file does not equal with expected") + sio.close() + + #check 'linear' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='linear') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, ih_records_linear_expected_str, + "Written hex file does not equal with expected") + sio.close() + + # 'none' mode is causing exceptions + # this mode will be checked in different test suite + + def test_write_hex_file_linear_address_type(self): + #prepare + ih_records = [Record.start_linear_address(0x12345678), + Record.extended_linear_address(0x3000), + Record.data(0x0, [0xAB, 0xCD]), + Record.eof()] + ih_records_str = '\n'.join(ih_records)+'\n' + ih = intelhex.IntelHex(StringIO(ih_records_str)) + + ih_records_linear_expected_str = ih_records_str + + #check 'auto' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='auto') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, ih_records_linear_expected_str, + "Written hex file does not equal with expected") + sio.close() + + #check 'linear' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='linear') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, ih_records_linear_expected_str, + "Written hex file does not equal with expected") + sio.close() + + # 'none' and 'segment' mode are causing exceptions + # these modes will be checked in different test suite + + def test_write_hex_file_no_starting_address_big1M(self): + #prepare + ih = intelhex.IntelHex(StringIO(hex16M)) + + #check 'auto' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='auto') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, hex16M, + "Written hex file does not equal with expected") + sio.close() + + #check 'linear' mode + sio = StringIO() + ih.write_hex_file(sio, ext_addr_mode='linear') + sio.seek(0) + sioLines = sio.readlines() + s = ''.join(sioLines) + self.assertEqual(s, hex16M, + "Written hex file does not equal with expected") + sio.close() + + # 'none' and 'segment' mode are causing exceptions + # these modes will be checked in different test suite + +# here only negative tests - positive are checked earlier +class TestWriteHexFileAddrSpaceInvalidaiton(unittest.TestCase): + + def setUp(self): + self.sio = StringIO() + + def tearDown(self): + self.sio.close() + + def test_write_hex_file_big32Mplus(self): + #prepare + ih = intelhex.IntelHex() + addr32Mplus = 0x100000000 + linearBytesOverflowed = addr32Mplus - 0xFFFFFFFF # = 0x1 + segmentBytesOverflowed = addr32Mplus - 0x0FFFFF # = 0xFFF00001, 4293918721 (DEC) + basicBytesOverflowed = addr32Mplus - 0xFFFF # = 0xFFFF0001, 4294901761 (DEC) + ih[addr32Mplus] = 0x01 + + with self.assertRaises(LinearSpacingOverflowError) as ctx: + ih.write_hex_file(self.sio, ext_addr_mode='auto') + # check also overflowed bytes length + self.assertEquals(ctx.exception.overflwd_len, linearBytesOverflowed) + + with self.assertRaises(LinearSpacingOverflowError) as ctx: + ih.write_hex_file(self.sio, ext_addr_mode='linear') + # check also overflowed bytes length + self.assertEquals(ctx.exception.overflwd_len, linearBytesOverflowed) + + with self.assertRaises(SegmentSpacingOverflowError) as ctx: + ih.write_hex_file(self.sio, ext_addr_mode='segment') + # check also overflowed bytes length + self.assertEquals(ctx.exception.overflwd_len, segmentBytesOverflowed) + + with self.assertRaises(BasicSpacingOverflowError) as ctx: + ih.write_hex_file(self.sio, ext_addr_mode='none') + # check also overflowed bytes length + self.assertEquals(ctx.exception.overflwd_len, basicBytesOverflowed) + + def test_write_hex_file_big1Mplus(self): + ih = intelhex.IntelHex() + addr1Mplus = 0x100000 + segmentBytesOverflowed = addr1Mplus - 0x0FFFFF # = 0x1 + basicBytesOverflowed = addr1Mplus - 0xFFFF # = 0xF0001, 983041 (DEC) + ih[addr1Mplus] = 0x01 + + with self.assertRaises(SegmentSpacingOverflowError) as ctx: + ih.write_hex_file(self.sio, ext_addr_mode='segment') + # check also overflowed bytes length + self.assertEquals(ctx.exception.overflwd_len, segmentBytesOverflowed) + + with self.assertRaises(BasicSpacingOverflowError) as ctx: + ih.write_hex_file(self.sio, ext_addr_mode='none') + # check also overflowed bytes length + self.assertEquals(ctx.exception.overflwd_len, basicBytesOverflowed) + + def test_write_hex_file_big64kplus(self): + ih = intelhex.IntelHex() + addr64kplus = 0x10000 + basicBytesOverflowed = addr64kplus - 0xFFFF # = 0x1 + ih[addr64kplus] = 0x01 + + with self.assertRaises(BasicSpacingOverflowError) as ctx: + ih.write_hex_file(self.sio, ext_addr_mode='none') + # check also overflowed bytes length + self.assertEquals(ctx.exception.overflwd_len, basicBytesOverflowed) + ## # MAIN if __name__ == '__main__':