diff --git a/src/com/amazon/ion/impl/IonCursorBinary.java b/src/com/amazon/ion/impl/IonCursorBinary.java index 2224d2bc6f..0948114cbb 100644 --- a/src/com/amazon/ion/impl/IonCursorBinary.java +++ b/src/com/amazon/ion/impl/IonCursorBinary.java @@ -948,7 +948,10 @@ private long uncheckedReadFlexUInt_1_1() { */ private long slowReadFlexUInt_1_1() { // TODO perf: try 1-byte special case checks. Least-significant bits of 1 indicate 1-byte - int currentByte = slowPeekByte(); + int currentByte = slowReadByte(); + if (currentByte < 0) { + return -1; + } if (currentByte == 0) { throw new IonException("Found a VarUInt that was too large to fit in a `long`"); } @@ -1345,11 +1348,6 @@ private boolean slowReadValueHeader(IonTypeID valueTid, boolean isAnnotated, Mar if (valueTid.isDelimited) { endIndex = DELIMITED_MARKER; } else if (valueTid.variableLength) { - // At this point the value must be at least 2 more bytes: 1 for the smallest-possible value length - // and 1 for the smallest-possible value representation. - if (!fillAt(peekIndex, 2)) { - return true; - } valueLength = minorVersion == 0 ? slowReadVarUInt_1_0() : slowReadFlexUInt_1_1(); if (valueLength < 0) { return true; diff --git a/src/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java b/src/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java index 471ead7e13..1db18d1e4d 100644 --- a/src/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java +++ b/src/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java @@ -33,6 +33,7 @@ class IonReaderContinuableCoreBinary extends IonCursorBinary implements IonReade private static final int LOWER_SEVEN_BITS_BITMASK = 0x7F; private static final int SINGLE_BYTE_MASK = 0xFF; + private static final int TWO_BYTE_MASK = 0xFFFF; // Isolates the lowest six bits in a byte. private static final int LOWER_SIX_BITS_BITMASK = 0x3F; @@ -64,7 +65,8 @@ class IonReaderContinuableCoreBinary extends IonCursorBinary implements IonReade // The second-most significant bit in the most significant byte of a VarInt is the sign. private static final int VAR_INT_SIGN_BITMASK = 0x40; - // 32-bit floats must declare length 4. + private static final int FLOAT_16_BYTE_LENGTH = 2; + private static final int FLOAT_32_BYTE_LENGTH = 4; // Initial capacity of the ArrayList used to hold the symbol IDs of the annotations on the current value. @@ -993,6 +995,49 @@ public int intValue() { return (int) longValue(); } + // IEEE-754 half-precision (s=sign, e=exponent, f=fraction): seee_eeff_ffff_ffff + private static final int FLOAT_16_SIGN_MASK = 0b1000_0000_0000_0000; + private static final int FLOAT_16_EXPONENT_MASK = 0b0111_1100_0000_0000; + private static final int FLOAT_16_FRACTION_MASK = 0b0000_0011_1111_1111; + + // float64 bias: 1023; float16 bias: 15. Shift left to align with the masked exponent bits. + private static final int FLOAT_16_TO_64_EXPONENT_BIAS_CONVERSION = (1023 - 15) << Integer.numberOfTrailingZeros(FLOAT_16_EXPONENT_MASK); + // The float16 sign bit has bit index 15; the float64 sign bit has bit index 63. + private static final int FLOAT_16_TO_64_SIGN_SHIFT = 63 - 15; + // The 5 float16 exponent bits start at index 10; the 11 float64 exponent bits start at index 52. + private static final int FLOAT_16_TO_64_EXPONENT_SHIFT = 52 - 10; + // The most significant float16 fraction bit is at index 9; the most significant float64 fraction bit is at index 51. + private static final int FLOAT_16_TO_64_FRACTION_SHIFT = 51 - 9; + + /** + * Reads the next two bytes from the given ByteBuffer as a 16-bit float, returning the value as a Java double. + * @param byteBuffer a buffer positioned at the first byte of the 16-bit float. + * @return the value. + */ + private static double readFloat16(ByteBuffer byteBuffer) { + int bits = byteBuffer.getShort() & TWO_BYTE_MASK; + int sign = bits & FLOAT_16_SIGN_MASK; + int exponent = bits & FLOAT_16_EXPONENT_MASK; + int fraction = bits & FLOAT_16_FRACTION_MASK; + if (exponent == 0) { + if (fraction == 0) { + return sign == 0 ? -0e0 : 0e0; + } + // Denormalized + throw new UnsupportedOperationException("Support for denormalized half-precision floats not yet added."); + } else if ((exponent ^ FLOAT_16_EXPONENT_MASK) == 0) { + if (fraction == 0) { + return sign == 0 ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + } + return Double.NaN; + } + return Double.longBitsToDouble( + ((long) sign << FLOAT_16_TO_64_SIGN_SHIFT) + | ((long) (exponent + FLOAT_16_TO_64_EXPONENT_BIAS_CONVERSION) << FLOAT_16_TO_64_EXPONENT_SHIFT) + | ((long) fraction << FLOAT_16_TO_64_FRACTION_SHIFT) + ); + } + @Override public double doubleValue() { double value; @@ -1003,7 +1048,12 @@ public double doubleValue() { return 0.0d; } ByteBuffer bytes = prepareByteBuffer(valueMarker.startIndex, valueMarker.endIndex); - if (length == FLOAT_32_BYTE_LENGTH) { + if (length == FLOAT_16_BYTE_LENGTH) { + if (minorVersion == 0) { + throw new IonException("Ion 1.0 floats may may only have length 0, 4, or 8."); + } + value = readFloat16(bytes); + } else if (length == FLOAT_32_BYTE_LENGTH) { value = bytes.getFloat(); } else { // Note: there is no need to check for other lengths here; the type ID byte is validated during next(). diff --git a/test/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java b/test/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java index de9c2f6c40..40d1fe8461 100644 --- a/test/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java +++ b/test/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java @@ -3690,6 +3690,9 @@ private void assertLongCorrectlyParsed(boolean constructFromBytes, long expected " 1, 51 01", " 17, 51 11", " 127, 51 7F", + " 127, 52 7F 00", + " 127, 54 7F 00 00 00", + " 127, 58 7F 00 00 00 00 00 00 00", " 128, 52 80 00", " 5555, 52 B3 15", " 32767, 52 FF 7F", @@ -3721,6 +3724,30 @@ public void readLongValue(long expectedValue, String inputBytes) throws Exceptio assertLongCorrectlyParsed(false, expectedValue, inputBytes); } + @ParameterizedTest + @CsvSource({ + " 0, F5 01", + " 1, F5 03 01", + " 17, F5 03 11", + " 127, F5 03 7F", + " 128, F5 05 80 00", + " 2147483647, F5 09 FF FF FF 7F", // Integer.MAX_VALUE + " 72624976668147840, F5 11 80 40 20 10 08 04 02 01", + " 9223372036854775807, F5 11 FF FF FF FF FF FF FF 7F", // Long.MAX_VALUE + " -1, F5 03 FF", + " -2, F5 03 FE", + " -14, F5 03 F2", + " -128, F5 03 80", + " -129, F5 05 7F FF", + " -2147483648, F5 09 00 00 00 80", // Integer.MIN_VALUE + " -72624976668147841, F5 11 7F BF DF EF F7 FB FD FE", + " -9223372036854775808, F5 11 00 00 00 00 00 00 00 80", // Long.MIN_VALUE + }) + public void readLongValueFromVariableLengthEncoding(long expectedValue, String inputBytes) throws Exception { + assertLongCorrectlyParsed(true, expectedValue, inputBytes); + assertLongCorrectlyParsed(false, expectedValue, inputBytes); + } + /** * Checks that the reader reads the expected BigInteger from the given input bits. */ @@ -3739,6 +3766,9 @@ private void assertBigIntegerCorrectlyParsed(boolean constructFromBytes, BigInte " 1, 51 01", " 17, 51 11", " 127, 51 7F", + " 127, 52 7F 00", + " 127, 54 7F 00 00 00", + " 127, 58 7F 00 00 00 00 00 00 00", " 128, 52 80 00", " 5555, 52 B3 15", " 32767, 52 FF 7F", @@ -3789,25 +3819,44 @@ private void assertDoubleCorrectlyParsed(boolean constructFromBytes, double expe @ParameterizedTest @CsvSource({ " 0.0, 5A", + " 0.0, 5B 00 00", + " 0.0, 5C 00 00 00 00", + " 0.0, 5D 00 00 00 00 00 00 00 00", + " -0.0, 5B 80 00", + " -0.0, 5C 80 00 00 00", + " -0.0, 5D 80 00 00 00 00 00 00 00", + " 1.0, 5B 3C 00", " 1.0, 5C 3F 80 00 00", + " 1.0, 5D 3F F0 00 00 00 00 00 00", " 1.5, 5C 3F C0 00 00", + " 0.00006103515625, 5B 04 00", // Smallest positive normal half-precision float + " 0.333251953125, 5B 35 55", // Nearest half-precision representation of one third " 3.141592653589793, 5D 40 09 21 FB 54 44 2D 18", " 4.00537109375, 5C 40 80 2C 00", " 4.11111111111, 5D 40 10 71 C7 1C 71 C2 39", + " 65504, 5B 7B FF", // Largest normal half-precision float " 423542.09375, 5C 48 CE CE C3", " 8236423542.09375, 5D 41 FE AE DD 97 61 80 00", " 1.79769313486231570e+308, 5D 7F EF FF FF FF FF FF FF", // Double.MAX_VALUE " -1.0, 5C BF 80 00 00", " -1.5, 5C BF C0 00 00", + " -2, 5B C0 00", " -3.141592653589793, 5D C0 09 21 FB 54 44 2D 18", " -4.00537109375, 5C C0 80 2C 00", " -4.11111111111, 5D C0 10 71 C7 1C 71 C2 39", + " -65504, 5B FB FF", // Smallest normal half-precision float " -423542.09375, 5C C8 CE CE C3", " -8236423542.09375, 5D C1 FE AE DD 97 61 80 00", "-1.79769313486231570e+308, 5D FF EF FF FF FF FF FF FF", // Double.MIN_VALUE + " NaN, 5B 7C 01", + " Infinity, 5B 7C 00", + " -Infinity, 5B FC 00", " NaN, 5C 7F C0 00 00", " Infinity, 5C 7F 80 00 00", " -Infinity, 5C FF 80 00 00", + " NaN, 5D 7F F0 00 00 00 00 00 01", + " Infinity, 5D 7F F0 00 00 00 00 00 00", + " -Infinity, 5D FF F0 00 00 00 00 00 00", }) public void readDoubleValue(double expectedValue, String inputBytes) throws Exception { assertDoubleCorrectlyParsed(true, expectedValue, inputBytes);