Skip to content

Commit

Permalink
Adds support for reading binary normalized half-precision floats; add…
Browse files Browse the repository at this point in the history 
…s more tests for numeric types.
  • Loading branch information
@tgregg
tgregg committed Nov 20, 2023
1 parent 7c870c6 commit 7b9305e
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 8 deletions.
10 changes: 4 additions & 6 deletions src/com/amazon/ion/impl/IonCursorBinary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -948,7 +948,10 @@ private long uncheckedReadFlexUInt_1_1() {
*/
private long slowReadFlexUInt_1_1() {
// TODO perf: try 1-byte special case checks. Least-significant bits of 1 indicate 1-byte
int currentByte = slowPeekByte();
int currentByte = slowReadByte();
if (currentByte < 0) {
return -1;
}
if (currentByte == 0) {
throw new IonException("Found a VarUInt that was too large to fit in a `long`");
}
Expand Down Expand Up @@ -1345,11 +1348,6 @@ private boolean slowReadValueHeader(IonTypeID valueTid, boolean isAnnotated, Mar
if (valueTid.isDelimited) {
endIndex = DELIMITED_MARKER;
} else if (valueTid.variableLength) {
// At this point the value must be at least 2 more bytes: 1 for the smallest-possible value length
// and 1 for the smallest-possible value representation.
if (!fillAt(peekIndex, 2)) {
return true;
}
valueLength = minorVersion == 0 ? slowReadVarUInt_1_0() : slowReadFlexUInt_1_1();
if (valueLength < 0) {
return true;
Expand Down
54 changes: 52 additions & 2 deletions src/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ class IonReaderContinuableCoreBinary extends IonCursorBinary implements IonReade
private static final int LOWER_SEVEN_BITS_BITMASK = 0x7F;

private static final int SINGLE_BYTE_MASK = 0xFF;
private static final int TWO_BYTE_MASK = 0xFFFF;

// Isolates the lowest six bits in a byte.
private static final int LOWER_SIX_BITS_BITMASK = 0x3F;
Expand Down Expand Up @@ -64,7 +65,8 @@ class IonReaderContinuableCoreBinary extends IonCursorBinary implements IonReade
// The second-most significant bit in the most significant byte of a VarInt is the sign.
private static final int VAR_INT_SIGN_BITMASK = 0x40;

// 32-bit floats must declare length 4.
private static final int FLOAT_16_BYTE_LENGTH = 2;

private static final int FLOAT_32_BYTE_LENGTH = 4;

// Initial capacity of the ArrayList used to hold the symbol IDs of the annotations on the current value.
Expand Down Expand Up @@ -993,6 +995,49 @@ public int intValue() {
return (int) longValue();
}

// IEEE-754 half-precision (s=sign, e=exponent, f=fraction): seee_eeff_ffff_ffff
private static final int FLOAT_16_SIGN_MASK = 0b1000_0000_0000_0000;
private static final int FLOAT_16_EXPONENT_MASK = 0b0111_1100_0000_0000;
private static final int FLOAT_16_FRACTION_MASK = 0b0000_0011_1111_1111;

// float64 bias: 1023; float16 bias: 15. Shift left to align with the masked exponent bits.
private static final int FLOAT_16_TO_64_EXPONENT_BIAS_CONVERSION = (1023 - 15) << Integer.numberOfTrailingZeros(FLOAT_16_EXPONENT_MASK);
// The float16 sign bit has bit index 15; the float64 sign bit has bit index 63.
private static final int FLOAT_16_TO_64_SIGN_SHIFT = 63 - 15;
// The 5 float16 exponent bits start at index 10; the 11 float64 exponent bits start at index 52.
private static final int FLOAT_16_TO_64_EXPONENT_SHIFT = 52 - 10;
// The most significant float16 fraction bit is at index 9; the most significant float64 fraction bit is at index 51.
private static final int FLOAT_16_TO_64_FRACTION_SHIFT = 51 - 9;

/**
* Reads the next two bytes from the given ByteBuffer as a 16-bit float, returning the value as a Java double.
* @param byteBuffer a buffer positioned at the first byte of the 16-bit float.
* @return the value.
*/
private static double readFloat16(ByteBuffer byteBuffer) {
int bits = byteBuffer.getShort() & TWO_BYTE_MASK;
int sign = bits & FLOAT_16_SIGN_MASK;
int exponent = bits & FLOAT_16_EXPONENT_MASK;
int fraction = bits & FLOAT_16_FRACTION_MASK;
if (exponent == 0) {
if (fraction == 0) {
return sign == 0 ? -0e0 : 0e0;
}
// Denormalized
throw new UnsupportedOperationException("Support for denormalized half-precision floats not yet added.");
} else if ((exponent ^ FLOAT_16_EXPONENT_MASK) == 0) {
if (fraction == 0) {
return sign == 0 ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
}
return Double.NaN;
}
return Double.longBitsToDouble(
((long) sign << FLOAT_16_TO_64_SIGN_SHIFT)
| ((long) (exponent + FLOAT_16_TO_64_EXPONENT_BIAS_CONVERSION) << FLOAT_16_TO_64_EXPONENT_SHIFT)
| ((long) fraction << FLOAT_16_TO_64_FRACTION_SHIFT)
);
}

@Override
public double doubleValue() {
double value;
Expand All @@ -1003,7 +1048,12 @@ public double doubleValue() {
return 0.0d;
}
ByteBuffer bytes = prepareByteBuffer(valueMarker.startIndex, valueMarker.endIndex);
if (length == FLOAT_32_BYTE_LENGTH) {
if (length == FLOAT_16_BYTE_LENGTH) {
if (minorVersion == 0) {
throw new IonException("Ion 1.0 floats may may only have length 0, 4, or 8.");
}
value = readFloat16(bytes);
} else if (length == FLOAT_32_BYTE_LENGTH) {
value = bytes.getFloat();
} else {
// Note: there is no need to check for other lengths here; the type ID byte is validated during next().
Expand Down
49 changes: 49 additions & 0 deletions test/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3690,6 +3690,9 @@ private void assertLongCorrectlyParsed(boolean constructFromBytes, long expected
" 1, 51 01",
" 17, 51 11",
" 127, 51 7F",
" 127, 52 7F 00",
" 127, 54 7F 00 00 00",
" 127, 58 7F 00 00 00 00 00 00 00",
" 128, 52 80 00",
" 5555, 52 B3 15",
" 32767, 52 FF 7F",
Expand Down Expand Up @@ -3721,6 +3724,30 @@ public void readLongValue(long expectedValue, String inputBytes) throws Exceptio
assertLongCorrectlyParsed(false, expectedValue, inputBytes);
}

@ParameterizedTest
@CsvSource({
" 0, F5 01",
" 1, F5 03 01",
" 17, F5 03 11",
" 127, F5 03 7F",
" 128, F5 05 80 00",
" 2147483647, F5 09 FF FF FF 7F", // Integer.MAX_VALUE
" 72624976668147840, F5 11 80 40 20 10 08 04 02 01",
" 9223372036854775807, F5 11 FF FF FF FF FF FF FF 7F", // Long.MAX_VALUE
" -1, F5 03 FF",
" -2, F5 03 FE",
" -14, F5 03 F2",
" -128, F5 03 80",
" -129, F5 05 7F FF",
" -2147483648, F5 09 00 00 00 80", // Integer.MIN_VALUE
" -72624976668147841, F5 11 7F BF DF EF F7 FB FD FE",
" -9223372036854775808, F5 11 00 00 00 00 00 00 00 80", // Long.MIN_VALUE
})
public void readLongValueFromVariableLengthEncoding(long expectedValue, String inputBytes) throws Exception {
assertLongCorrectlyParsed(true, expectedValue, inputBytes);
assertLongCorrectlyParsed(false, expectedValue, inputBytes);
}

/**
* Checks that the reader reads the expected BigInteger from the given input bits.
*/
Expand All @@ -3739,6 +3766,9 @@ private void assertBigIntegerCorrectlyParsed(boolean constructFromBytes, BigInte
" 1, 51 01",
" 17, 51 11",
" 127, 51 7F",
" 127, 52 7F 00",
" 127, 54 7F 00 00 00",
" 127, 58 7F 00 00 00 00 00 00 00",
" 128, 52 80 00",
" 5555, 52 B3 15",
" 32767, 52 FF 7F",
Expand Down Expand Up @@ -3789,25 +3819,44 @@ private void assertDoubleCorrectlyParsed(boolean constructFromBytes, double expe
@ParameterizedTest
@CsvSource({
" 0.0, 5A",
" 0.0, 5B 00 00",
" 0.0, 5C 00 00 00 00",
" 0.0, 5D 00 00 00 00 00 00 00 00",
" -0.0, 5B 80 00",
" -0.0, 5C 80 00 00 00",
" -0.0, 5D 80 00 00 00 00 00 00 00",
" 1.0, 5B 3C 00",
" 1.0, 5C 3F 80 00 00",
" 1.0, 5D 3F F0 00 00 00 00 00 00",
" 1.5, 5C 3F C0 00 00",
" 0.00006103515625, 5B 04 00", // Smallest positive normal half-precision float
" 0.333251953125, 5B 35 55", // Nearest half-precision representation of one third
" 3.141592653589793, 5D 40 09 21 FB 54 44 2D 18",
" 4.00537109375, 5C 40 80 2C 00",
" 4.11111111111, 5D 40 10 71 C7 1C 71 C2 39",
" 65504, 5B 7B FF", // Largest normal half-precision float
" 423542.09375, 5C 48 CE CE C3",
" 8236423542.09375, 5D 41 FE AE DD 97 61 80 00",
" 1.79769313486231570e+308, 5D 7F EF FF FF FF FF FF FF", // Double.MAX_VALUE
" -1.0, 5C BF 80 00 00",
" -1.5, 5C BF C0 00 00",
" -2, 5B C0 00",
" -3.141592653589793, 5D C0 09 21 FB 54 44 2D 18",
" -4.00537109375, 5C C0 80 2C 00",
" -4.11111111111, 5D C0 10 71 C7 1C 71 C2 39",
" -65504, 5B FB FF", // Smallest normal half-precision float
" -423542.09375, 5C C8 CE CE C3",
" -8236423542.09375, 5D C1 FE AE DD 97 61 80 00",
"-1.79769313486231570e+308, 5D FF EF FF FF FF FF FF FF", // Double.MIN_VALUE
" NaN, 5B 7C 01",
" Infinity, 5B 7C 00",
" -Infinity, 5B FC 00",
" NaN, 5C 7F C0 00 00",
" Infinity, 5C 7F 80 00 00",
" -Infinity, 5C FF 80 00 00",
" NaN, 5D 7F F0 00 00 00 00 00 01",
" Infinity, 5D 7F F0 00 00 00 00 00 00",
" -Infinity, 5D FF F0 00 00 00 00 00 00",
})
public void readDoubleValue(double expectedValue, String inputBytes) throws Exception {
assertDoubleCorrectlyParsed(true, expectedValue, inputBytes);
Expand Down

0 comments on commit 7b9305e

Please sign in to comment.