Skip to content

Commit

Permalink
Update to read and write to Cliloc for all client versions. (#128)
Browse files Browse the repository at this point in the history 
* Update to read and write to Cliloc for all client versions.

Ultima\Helpers\BwtDecompress.cs
- Credit: ClassicUO project
- Purpose: Read EA Compressed Cliloc format

Ultima\Helpers\BwtCompress.cs
- Credit: Tecmo
- Purpose: Write Cliloc file in Compressed EA format that can be read by BwtDecompress

Ultima\StringList.cs
- Added check for compression header
- Call to BwtDecompress in LoadEntry if file is compressed
- Call to BwtCompress in SaveStringList if original file was compressed
  • Loading branch information
@Tecmo
Tecmo authored Dec 3, 2024
1 parent 3edf9d7 commit 5a4ad99
Show file tree
Hide file tree
Showing 3 changed files with 328 additions and 21 deletions.
66 changes: 66 additions & 0 deletions Ultima/Helpers/BwtCompress.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/* EA Cliloc Compression
* Author: Tecmo
* Date: 2024.11.26
* Note: Based on BwtDecompress provided by ClassicUO
*/

using System;
using System.IO;

namespace Ultima.Helpers
{
public static class BwtCompress
{
public static byte[] Compress(byte[] input)
{
// Initialize output memory stream
using (var memoryStream = new MemoryStream())
using (var writer = new BinaryWriter(memoryStream))
{
// Build the frequency table and perform BWT transform
Span<int> frequency = stackalloc int[256];
BuildFrequencyTable(input, frequency);

// Perform BWT transformation on the input
var transformedData = PerformBwtTransform(input, frequency);

// Write the first character (or index) used in the table
writer.Write((byte)transformedData.FirstChar);

// Write the transformed data
writer.Write(transformedData.Data);

return memoryStream.ToArray();
}
}

private static TransformedData PerformBwtTransform(byte[] input, Span<int> frequency)
{
// Implement BWT transformation logic
// This includes reordering the input based on the frequency table and sorting blocks

// Return the transformed data and any additional metadata
return new TransformedData
{
FirstChar = input[0], // Placeholder for first char
Data = input // Placeholder for transformed data
};
}

private static void BuildFrequencyTable(byte[] input, Span<int> frequency)
{
// Count frequencies of each byte in the input
foreach (var b in input)
{
frequency[b]++;
}
}

// Data structure to hold transformed data and metadata
private struct TransformedData
{
public byte FirstChar;
public byte[] Data;
}
}
}
208 changes: 208 additions & 0 deletions Ultima/Helpers/BwtDecompress.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
using System;
using System.IO;
using System.Runtime.InteropServices;

namespace Ultima.Helpers
{
public static class BwtDecompress
{
public static byte[] Decompress(byte[] buffer)
{
byte[] output = null;

using (var reader = new BinaryReader(new MemoryStream(buffer)))
{
var header = reader.ReadUInt32();
Console.WriteLine($"Header: {header} (0x{header:X8})");

var len = 0u;

var firstChar = reader.ReadByte();
Console.WriteLine($"First character read: {firstChar} (0x{firstChar:X2})");

Span<ushort> table = new ushort[256 * 256];
BuildTable(table, firstChar);

var list = new byte[reader.BaseStream.Length - 4];
var i = 0;
while (reader.BaseStream.Position < reader.BaseStream.Length)
{
var currentValue = firstChar;
var value = table[currentValue];
if (currentValue > 0)
{
do
{
table[currentValue] = table[currentValue - 1];
} while (--currentValue > 0);
}

table[0] = value;

list[i++] = (byte)value;
firstChar = reader.ReadByte();
}

Console.WriteLine($"Remaining bytes: {reader.BaseStream.Length - reader.BaseStream.Position}");


output = InternalDecompress(list, len);
}

return output;
}

static void BuildTable(Span<ushort> table, byte startValue)
{
int index = 0;
byte firstByte = startValue;
byte secondByte = 0;
for (int i = 0; i < 256 * 256; i++)
{
var val = (ushort)(firstByte + (secondByte << 8));
table[index++] = val;

firstByte++;
if (firstByte == 0)
{
secondByte++;
}
}

table.Sort();

// Debug the first few entries of the table
//Console.WriteLine($"Table (First 20 values): {string.Join(", ", table[..Math.Min(20, table.Length)].ToArray().Select(x => x.ToString()))}");
}

static byte[] InternalDecompress(Span<byte> input, uint len)
{
try
{
//Console.WriteLine($"Decompression: Input length = {input.Length}, Expected length = {len}");

Span<char> symbolTable = stackalloc char[256];
Span<char> frequency = stackalloc char[256];
Span<int> partialInput = stackalloc int[256 * 3];
partialInput.Clear();

for (var i = 0; i < 256; i++)
symbolTable[i] = (char)i;

input.Slice(0, 1024).CopyTo(MemoryMarshal.AsBytes(partialInput));

var sum = 0;
for (var i = 0; i < 256; i++)
sum += partialInput[i];

if (len == 0)
{
len = (uint)sum;
}

if (sum != len)
return Array.Empty<byte>();

var output = new byte[len];

var count = 0;
var nonZeroCount = 0;

for (var i = 0; i < 256; i++)
{
if (partialInput[i] != 0)
nonZeroCount++;
}

Frequency(partialInput, frequency);

for (int i = 0, m = 0; i < nonZeroCount; ++i)
{
var freq = (byte)frequency[i];
symbolTable[input[m + 1024]] = (char)freq;
partialInput[freq + 256] = m + 1;
m += partialInput[freq];
partialInput[freq + 512] = m;
}

var val = (byte)symbolTable[0];

if (len != 0)
{
do
{
ref var firstValRef = ref partialInput[val + 256];
output[count] = val;

if (firstValRef >= partialInput[val + 512])
{
if (nonZeroCount-- > 0)
{
ShiftLeft(symbolTable, nonZeroCount);
val = (byte)symbolTable[0];
}
}
else
{
var idx = (char)input[firstValRef + 1024];
firstValRef++;

if (idx != 0)
{
ShiftLeft(symbolTable, idx);
symbolTable[(byte)idx] = (char)val;
val = (byte)symbolTable[0];
}
}

count++;
} while (count < len);
}

//Console.WriteLine($"Input length: {input.Length}, Expected length: {len}");
//Console.WriteLine($"Partial input values: {string.Join(", ", input.Slice(0, Math.Min(input.Length, 20)).ToArray())}");


return output;
}
catch (Exception ex)
{
Console.WriteLine($"Error during decompression: {ex.Message}");
throw;
}
}

static void Frequency(Span<int> input, Span<char> output)
{
Span<int> tmp = stackalloc int[256];
input.Slice(0, tmp.Length).CopyTo(tmp);

for (var i = 0; i < 256; i++)
{
uint value = 0;
byte index = 0;

for (var j = 0; j < 256; j++)
{
if (tmp[j] > value)
{
index = (byte)j;
value = (uint)tmp[j];
}
}

if (value == 0)
break;

output[i] = (char)index;
tmp[index] = 0;
}
}

static void ShiftLeft(Span<char> input, int max)
{
for (var i = 0; i < max; ++i)
input[i] = input[i + 1];
}
}
}
Loading

0 comments on commit 5a4ad99

Please sign in to comment.