diff --git a/src/com/amazon/ion/impl/IonCursorBinary.java b/src/com/amazon/ion/impl/IonCursorBinary.java index fe27513c5a..0948114cbb 100644 --- a/src/com/amazon/ion/impl/IonCursorBinary.java +++ b/src/com/amazon/ion/impl/IonCursorBinary.java @@ -948,7 +948,10 @@ private long uncheckedReadFlexUInt_1_1() { */ private long slowReadFlexUInt_1_1() { // TODO perf: try 1-byte special case checks. Least-significant bits of 1 indicate 1-byte - int currentByte = slowPeekByte(); + int currentByte = slowReadByte(); + if (currentByte < 0) { + return -1; + } if (currentByte == 0) { throw new IonException("Found a VarUInt that was too large to fit in a `long`"); } @@ -1262,6 +1265,9 @@ private boolean uncheckedReadHeader(final int typeIdByte, final boolean isAnnota if (endIndex > limit) { isValueIncomplete = true; } + if (minorVersion == 1 && valueTid.isNull && valueTid.length > 0) { + valueTid = IonTypeID.NULL_TYPE_IDS_1_1[buffer[(int)(peekIndex++) & SINGLE_BYTE_MASK]]; + } } markerToSet.typeId = valueTid; if (event == Event.START_CONTAINER) { @@ -1306,6 +1312,13 @@ private boolean slowReadHeader(final int typeIdByte, final boolean isAnnotated, } return true; } + if (minorVersion == 1 && valueTid.isNull && valueTid.length > 0) { + int nullTypeIndex = slowReadByte(); + if (nullTypeIndex < 0) { + return true; + } + valueTid = IonTypeID.NULL_TYPE_IDS_1_1[nullTypeIndex]; + } markerToSet.typeId = valueTid; if (checkpointLocation == CheckpointLocation.AFTER_SCALAR_HEADER) { return true; @@ -1335,11 +1348,6 @@ private boolean slowReadValueHeader(IonTypeID valueTid, boolean isAnnotated, Mar if (valueTid.isDelimited) { endIndex = DELIMITED_MARKER; } else if (valueTid.variableLength) { - // At this point the value must be at least 2 more bytes: 1 for the smallest-possible value length - // and 1 for the smallest-possible value representation. - if (!fillAt(peekIndex, 2)) { - return true; - } valueLength = minorVersion == 0 ? slowReadVarUInt_1_0() : slowReadFlexUInt_1_1(); if (valueLength < 0) { return true; diff --git a/src/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java b/src/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java index 1f06c46c76..1db18d1e4d 100644 --- a/src/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java +++ b/src/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java @@ -33,6 +33,7 @@ class IonReaderContinuableCoreBinary extends IonCursorBinary implements IonReade private static final int LOWER_SEVEN_BITS_BITMASK = 0x7F; private static final int SINGLE_BYTE_MASK = 0xFF; + private static final int TWO_BYTE_MASK = 0xFFFF; // Isolates the lowest six bits in a byte. private static final int LOWER_SIX_BITS_BITMASK = 0x3F; @@ -64,7 +65,8 @@ class IonReaderContinuableCoreBinary extends IonCursorBinary implements IonReade // The second-most significant bit in the most significant byte of a VarInt is the sign. private static final int VAR_INT_SIGN_BITMASK = 0x40; - // 32-bit floats must declare length 4. + private static final int FLOAT_16_BYTE_LENGTH = 2; + private static final int FLOAT_32_BYTE_LENGTH = 4; // Initial capacity of the ArrayList used to hold the symbol IDs of the annotations on the current value. @@ -376,8 +378,8 @@ private Timestamp readTimestamp_1_0() { offset = readVarInt_1_0(firstByte); } int year = readVarUInt_1_0(); - int month = 0; - int day = 0; + int month = 1; + int day = 1; int hour = 0; int minute = 0; int second = 0; @@ -475,77 +477,89 @@ private int readVarSym_1_1(Marker marker) { throw new UnsupportedOperationException(); } - private BigDecimal readBigDecimal_1_1() { - throw new UnsupportedOperationException(); + /** + * Reads a FixedInt into a long. After this method returns, `peekIndex` points to the first byte after the end of + * the FixedInt. + * @return the value. + */ + private long readFixedInt_1_1() { + if (peekIndex >= valueMarker.endIndex) { + return 0; + } + long startIndex = peekIndex; + peekIndex = valueMarker.endIndex; + // Note: the following line performs sign extension via the cast to long without masking with 0xFF. + long value = buffer[(int) --peekIndex]; + while (peekIndex > startIndex) { + value = (value << 8) | (buffer[(int) --peekIndex] & SINGLE_BYTE_MASK); + } + peekIndex = valueMarker.endIndex; + return value; } - private Decimal readDecimal_1_1() { - throw new UnsupportedOperationException(); + /** + * Copies a FixedInt or FixedUInt into scratch space, converting it to its equivalent big-endian two's complement + * representation. If the provided length is longer than the actual length of the value, the most significant + * byte in the two's complement representation will be zero. + * @param startIndex the index of the second byte in the FixedInt or FixedUInt representation. + * @param length the number of bytes remaining in the FixedInt or FixedUInt representation. + * @return a byte[] (either new or reused) containing the big-endian two's complement representation of the value. + */ + private byte[] copyFixedIntOrFixedUIntAsTwosComplementBytes(long startIndex, int length) { + // FixedInt is a little-endian two's complement representation. Simply reverse the bytes. + byte[] bytes = getScratchForSize(length); + // Clear the most significant byte in case the scratch space is padded to accommodate an unsigned value with + // its highest bit set. + bytes[0] = 0; + int copyIndex = bytes.length; + for (long i = startIndex; i < valueMarker.endIndex; i++) { + bytes[--copyIndex] = buffer[(int) i]; + } + peekIndex = valueMarker.endIndex; + return bytes; } - private long readLong_1_1() { + /** + * Reads a FixedInt or FixedUInt value into a BigInteger. + * @param length the length of the two's complement representation of the value. For FixedInts, this is always + * equal to the length of the value; for FixedUInts, this is one byte larger than the length of the + * value if the highest bit in the unsigned representation is set. + * @return the value. + */ + private BigInteger readFixedIntOrFixedUIntAsBigInteger_1_1(int length) { + BigInteger value; + if (length > 0) { + value = new BigInteger(copyFixedIntOrFixedUIntAsTwosComplementBytes(peekIndex, length)); + } else { + value = BigInteger.ZERO; + } + return value; + } + + private BigDecimal readBigDecimal_1_1() { throw new UnsupportedOperationException(); } - private BigInteger readBigInteger_1_1() { + private Decimal readDecimal_1_1() { throw new UnsupportedOperationException(); } /** - * Copies a FlexUInt into scratch space, converting it to its equivalent big-endian two's complement representation. - * @param firstByte the first (least-significant) byte in the FlexUInt representation. - * @param bitsToShiftRight the number of continuation bits that must be shifted out of every byte. - * @param startIndex the index of the second byte in the FlexUInt representation. - * @param length the number of bytes remaining in the FlexUInt representation. - * @return a byte[] (either new or reused) containing the big-endian two's complement representation of the value. + * Reads the FixedInt bounded by `valueMarker` into a `long`. + * @return the value. */ - private byte[] copyFlexUIntAsTwosComplementBytes(int firstByte, int bitsToShiftRight, long startIndex, int length) { - // If the most significant bit is set, the value would be interpreted as a negative two's complement integer. To - // avoid that, make sure the most significant byte in the copy is 0 by over-allocating the destination buffer. - // Additionally, one more byte than 'length' is always required because 'length' does not include the first - // byte. - byte[] bytes = getScratchForSize(length + 1 + ((buffer[(int) startIndex + length - 1] < 0) ? 1 : 0)); - bytes[0] = 0; - int copyIndex = bytes.length; - bytes[--copyIndex] = (byte) (firstByte >>> bitsToShiftRight); - int lowerBitsMask = ~(-1 << bitsToShiftRight); - for (int i = 0; i < length; i++) { - byte b = buffer[(int) startIndex + i]; - // The following implements a byte-by-byte bit shift. The lower bits in each byte are or'd with the higher - // bits from the previous byte. - bytes[copyIndex] |= (byte) ((b & lowerBitsMask) << (8 - bitsToShiftRight)); - bytes[--copyIndex] = (byte) ((b & SINGLE_BYTE_MASK) >>> bitsToShiftRight); - } - peekIndex = startIndex + length; - return bytes; + private long readLong_1_1() { + peekIndex = valueMarker.startIndex; + return readFixedInt_1_1(); } /** - * Reads an FlexUInt value into a BigInteger. - * @return the value as a BigInteger. + * Reads the FixedInt bounded by `valueMarker` into a BigInteger. + * @return the value. */ - private BigInteger readFlexUIntAsBigInteger_1_1() { - BigInteger value; - int currentByte = buffer[(int) peekIndex++] & SINGLE_BYTE_MASK; - int length = 0; - while (currentByte == 0) { - // Each byte of continuation bits without a set bit adds 8 to the length of the FlexUInt, but since the - // length includes the continuation byte(s), each empty byte adds a net 7 to the total length. - length += 7; - currentByte = buffer[(int) peekIndex++] & SINGLE_BYTE_MASK; - } - int numberOfLengthBits = Integer.numberOfTrailingZeros(currentByte); - length += numberOfLengthBits; - if (length > 0) { - // NOTE: copying into scratch space is required because the encoded bytes, which are unsigned little-endian, - // need to be translated into two's complement big-endian bytes as required by this BigInteger constructor. - // This is expensive, but is cheaper than using arithmetic operations on a BigInteger directly, as this - // would require a new BigInteger to be allocated for each intermediate step. - value = new BigInteger(copyFlexUIntAsTwosComplementBytes(currentByte, numberOfLengthBits + 1, peekIndex, length)); - } else { - value = BigInteger.ZERO; - } - return value; + private BigInteger readBigInteger_1_1() { + peekIndex = valueMarker.startIndex; + return readFixedIntOrFixedUIntAsBigInteger_1_1((int) (valueMarker.endIndex - peekIndex)); } /** @@ -553,20 +567,30 @@ private BigInteger readFlexUIntAsBigInteger_1_1() { * @return the value as a BigDecimal. */ private BigDecimal readTimestampFraction_1_1() { - // The fractional seconds are encoded as a (coefficient, scale) pair, + // The fractional seconds are encoded as a (scale, coefficient) pair, // which is similar to a decimal. The primary difference is that the scale represents a negative // exponent because it is illegal for the fractional seconds value to be greater than or equal to 1.0 - // or less than 0.0. The coefficient is encoded as a FlexUInt (instead of FlexInt) to prevent the - // encoding of fractional seconds less than 0.0. The scale is encoded as a FixedUInt (instead of FixedInt) + // or less than 0.0. The coefficient is encoded as a FixedUInt (instead of FixedInt) to prevent the + // encoding of fractional seconds less than 0.0. The scale is encoded as a FlexUInt (instead of FlexInt) // to discourage the encoding of decimal numbers greater than 1.0. BigDecimal value; peekIndex = valueMarker.startIndex + L_TIMESTAMP_SECOND_BYTE_LENGTH; - if (buffer[(int) peekIndex] != 0) { + int scale = (int) readFlexUInt_1_1(); + if (peekIndex >= valueMarker.endIndex) { + return BigDecimal.valueOf(0, scale); + } + int length = (int) (valueMarker.endIndex - peekIndex); + // Since the coefficient is stored in a FixedUInt, some 8-byte values cannot fit in a signed 8-byte long. + // Take the quick path for values up to 7 bytes rather than performing additional checks. This should cover + // almost all real-world timestamp fractions. + if (length <= 7) { // No need to allocate a BigInteger to hold the coefficient. - value = BigDecimal.valueOf(readFlexUInt_1_1(), (int) readFixedUInt_1_1(peekIndex, valueMarker.endIndex)); + value = BigDecimal.valueOf(readFixedUInt_1_1(peekIndex, valueMarker.endIndex), scale); } else { // The coefficient may overflow a long, so a BigInteger is required. - value = new BigDecimal(readFlexUIntAsBigInteger_1_1(), (int) readFixedUInt_1_1(peekIndex, valueMarker.endIndex)); + // If the most-significant bit is set, pad the length by one byte so that the value remains unsigned. + length += (buffer[(int) valueMarker.endIndex - 1] < 0) ? 1 : 0; + value = new BigDecimal(readFixedIntOrFixedUIntAsBigInteger_1_1(length), scale); } if (BigDecimal.ONE.compareTo(value) < 1) { throw new IllegalArgumentException(String.format("Fractional seconds %s must be greater than or equal to 0 and less than 1", value)); @@ -580,8 +604,8 @@ private BigDecimal readTimestampFraction_1_1() { */ private Timestamp readTimestampLongForm_1_1() { int year; - int month = 0; - int day = 0; + int month = 1; + int day = 1; int hour = 0; int minute = 0; int second = 0; @@ -651,8 +675,8 @@ private Timestamp readTimestamp_1_1() { } Timestamp.Precision precision = S_TIMESTAMP_PRECISION_FOR_TYPE_ID_OFFSET[valueTid.lowerNibble]; int year = 0; - int month = 0; - int day = 0; + int month = 1; + int day = 1; int hour = 0; int minute = 0; int second = 0; @@ -750,8 +774,13 @@ private Timestamp readTimestamp_1_1() { } } + /** + * Reads the boolean value using the type ID of the current value. + * @return the value. + */ private boolean readBoolean_1_1() { - throw new UnsupportedOperationException(); + // Boolean 'true' is 0x5E; 'false' is 0x5F. + return valueTid.lowerNibble == 0xE; } @Override @@ -823,15 +852,16 @@ public IntegerSize getIntegerSize() { return null; } prepareScalar(); - if (valueTid.length < 0) { + int length = valueTid.variableLength ? ((int) (valueMarker.endIndex - valueMarker.startIndex)) : valueTid.length; + if (length < 0) { return IntegerSize.BIG_INTEGER; - } else if (valueTid.length < INT_SIZE_IN_BYTES) { + } else if (length < INT_SIZE_IN_BYTES) { return IntegerSize.INT; - } else if (valueTid.length == INT_SIZE_IN_BYTES) { + } else if (length == INT_SIZE_IN_BYTES) { return (minorVersion != 0 || classifyInteger_1_0()) ? IntegerSize.INT : IntegerSize.LONG; - } else if (valueTid.length < LONG_SIZE_IN_BYTES) { + } else if (length < LONG_SIZE_IN_BYTES) { return IntegerSize.LONG; - } else if (valueTid.length == LONG_SIZE_IN_BYTES) { + } else if (length == LONG_SIZE_IN_BYTES) { return (minorVersion != 0 || classifyInteger_1_0()) ? IntegerSize.LONG : IntegerSize.BIG_INTEGER; } return IntegerSize.BIG_INTEGER; @@ -965,6 +995,49 @@ public int intValue() { return (int) longValue(); } + // IEEE-754 half-precision (s=sign, e=exponent, f=fraction): seee_eeff_ffff_ffff + private static final int FLOAT_16_SIGN_MASK = 0b1000_0000_0000_0000; + private static final int FLOAT_16_EXPONENT_MASK = 0b0111_1100_0000_0000; + private static final int FLOAT_16_FRACTION_MASK = 0b0000_0011_1111_1111; + + // float64 bias: 1023; float16 bias: 15. Shift left to align with the masked exponent bits. + private static final int FLOAT_16_TO_64_EXPONENT_BIAS_CONVERSION = (1023 - 15) << Integer.numberOfTrailingZeros(FLOAT_16_EXPONENT_MASK); + // The float16 sign bit has bit index 15; the float64 sign bit has bit index 63. + private static final int FLOAT_16_TO_64_SIGN_SHIFT = 63 - 15; + // The 5 float16 exponent bits start at index 10; the 11 float64 exponent bits start at index 52. + private static final int FLOAT_16_TO_64_EXPONENT_SHIFT = 52 - 10; + // The most significant float16 fraction bit is at index 9; the most significant float64 fraction bit is at index 51. + private static final int FLOAT_16_TO_64_FRACTION_SHIFT = 51 - 9; + + /** + * Reads the next two bytes from the given ByteBuffer as a 16-bit float, returning the value as a Java double. + * @param byteBuffer a buffer positioned at the first byte of the 16-bit float. + * @return the value. + */ + private static double readFloat16(ByteBuffer byteBuffer) { + int bits = byteBuffer.getShort() & TWO_BYTE_MASK; + int sign = bits & FLOAT_16_SIGN_MASK; + int exponent = bits & FLOAT_16_EXPONENT_MASK; + int fraction = bits & FLOAT_16_FRACTION_MASK; + if (exponent == 0) { + if (fraction == 0) { + return sign == 0 ? -0e0 : 0e0; + } + // Denormalized + throw new UnsupportedOperationException("Support for denormalized half-precision floats not yet added."); + } else if ((exponent ^ FLOAT_16_EXPONENT_MASK) == 0) { + if (fraction == 0) { + return sign == 0 ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + } + return Double.NaN; + } + return Double.longBitsToDouble( + ((long) sign << FLOAT_16_TO_64_SIGN_SHIFT) + | ((long) (exponent + FLOAT_16_TO_64_EXPONENT_BIAS_CONVERSION) << FLOAT_16_TO_64_EXPONENT_SHIFT) + | ((long) fraction << FLOAT_16_TO_64_FRACTION_SHIFT) + ); + } + @Override public double doubleValue() { double value; @@ -975,7 +1048,12 @@ public double doubleValue() { return 0.0d; } ByteBuffer bytes = prepareByteBuffer(valueMarker.startIndex, valueMarker.endIndex); - if (length == FLOAT_32_BYTE_LENGTH) { + if (length == FLOAT_16_BYTE_LENGTH) { + if (minorVersion == 0) { + throw new IonException("Ion 1.0 floats may may only have length 0, 4, or 8."); + } + value = readFloat16(bytes); + } else if (length == FLOAT_32_BYTE_LENGTH) { value = bytes.getFloat(); } else { // Note: there is no need to check for other lengths here; the type ID byte is validated during next(). diff --git a/src/com/amazon/ion/impl/IonTypeID.java b/src/com/amazon/ion/impl/IonTypeID.java index 35332041e7..05f6ca9b83 100644 --- a/src/com/amazon/ion/impl/IonTypeID.java +++ b/src/com/amazon/ion/impl/IonTypeID.java @@ -5,6 +5,8 @@ import com.amazon.ion.IonType; +import static com.amazon.ion.impl.bin.OpCodes.*; + /** * Holds pre-computed information about a binary Ion type ID byte. */ @@ -75,6 +77,7 @@ final class IonTypeID { static final IonTypeID[] TYPE_IDS_NO_IVM; static final IonTypeID[] TYPE_IDS_1_0; static final IonTypeID[] TYPE_IDS_1_1; + static final IonTypeID[] NULL_TYPE_IDS_1_1; static { TYPE_IDS_NO_IVM = new IonTypeID[NUMBER_OF_BYTES]; TYPE_IDS_1_0 = new IonTypeID[NUMBER_OF_BYTES]; @@ -84,6 +87,24 @@ final class IonTypeID { TYPE_IDS_1_0[b] = new IonTypeID((byte) b, 0); TYPE_IDS_1_1[b] = new IonTypeID((byte) b, 1); } + // In Ion 1.1, typed nulls are represented by the type ID 0xEB followed by a 1-byte UInt indicating the type. + // Therefore, the type of the typed null cannot be precomputed in Ion 1.1. In order to avoid adding more hot + // path branching to the reader, we create IonTypeIDs that mimic precomputed typed nulls in Ion 1.1 by reusing + // the typed null type IDs from Ion 1.0. When the type of the typed null is determined, the reader's current + // IonTypeID will be replaced with one of these. The index is the one-byte value that follows 0xEB. + NULL_TYPE_IDS_1_1 = new IonTypeID[12]; + NULL_TYPE_IDS_1_1[0x0] = TYPE_IDS_1_0[0x1F]; // null.bool + NULL_TYPE_IDS_1_1[0x1] = TYPE_IDS_1_0[0x2F]; // null.int + NULL_TYPE_IDS_1_1[0x2] = TYPE_IDS_1_0[0x4F]; // null.float + NULL_TYPE_IDS_1_1[0x3] = TYPE_IDS_1_0[0x5F]; // null.decimal + NULL_TYPE_IDS_1_1[0x4] = TYPE_IDS_1_0[0x6F]; // null.timestamp + NULL_TYPE_IDS_1_1[0x5] = TYPE_IDS_1_0[0x8F]; // null.string + NULL_TYPE_IDS_1_1[0x6] = TYPE_IDS_1_0[0x7F]; // null.symbol + NULL_TYPE_IDS_1_1[0x7] = TYPE_IDS_1_0[0xAF]; // null.blob + NULL_TYPE_IDS_1_1[0x8] = TYPE_IDS_1_0[0x9F]; // null.clob + NULL_TYPE_IDS_1_1[0x9] = TYPE_IDS_1_0[0xBF]; // null.list + NULL_TYPE_IDS_1_1[0xA] = TYPE_IDS_1_0[0xCF]; // null.sexp + NULL_TYPE_IDS_1_1[0xB] = TYPE_IDS_1_0[0xDF]; // null.struct } final IonType type; @@ -181,10 +202,9 @@ private IonTypeID(byte id, int minorVersion) { byte upperNibble = (byte) ((id >> BITS_PER_NIBBLE) & LOW_NIBBLE_BITMASK); // For 0xF0 (delimited end byte) the entire byte is included. This avoids having to create a separate field // just to identify this byte. - lowerNibble = (id == (byte) 0xF0) ? (byte) 0xF0 : (byte) (id & LOW_NIBBLE_BITMASK); + lowerNibble = (id == DELIMITED_END_MARKER) ? DELIMITED_END_MARKER : (byte) (id & LOW_NIBBLE_BITMASK); isNegativeInt = false; // Not applicable for Ion 1.1; sign is conveyed by the representation. - // 0xF4 is a length-prefixed macro invocation; 0xEF is a system macro invocation. - isMacroInvocation = upperNibble <= 0x4 || id == (byte) 0xF4 || id == (byte) 0xEF; + isMacroInvocation = upperNibble <= 0x4 || id == LENGTH_PREFIXED_MACRO_INVOCATION || id == SYSTEM_MACRO_INVOCATION; boolean isNopPad = false; boolean isNull = false; int length = -1; @@ -208,17 +228,19 @@ private IonTypeID(byte id, int minorVersion) { macroId = -1; variableLength = (upperNibble == 0xF && lowerNibble >= 0x4) // Variable length, all types. - || (upperNibble == 0x6 && lowerNibble == 0xF) // Decimal with negative-zero coefficient. - || (upperNibble == 0xE && lowerNibble == 0x6) // Variable length annotation SIDs. - || (upperNibble == 0xE && lowerNibble == 0x9) // Variable length annotation FlexSyms. - || (upperNibble == 0xE && lowerNibble == 0xD); // Variable length NOP. + || id == POSITIVE_ZERO_DECIMAL + || id == ANNOTATIONS_MANY_SYMBOL_ADDRESS + || id == ANNOTATIONS_MANY_FLEX_SYM + || id == VARIABLE_LENGTH_NOP; isInlineable = // struct with VarSym field names. (upperNibble == 0xD && lowerNibble >= 0x2) - // Delimited struct, variable-length symbol, variable-length struct with FlexSym field names. - || (upperNibble == 0xF && (lowerNibble == 0x3 || lowerNibble == 0x9 || lowerNibble == 0xD)) - // Annotation wrappers with VarSyms. - || (upperNibble == 0xE && lowerNibble >= 0x7 && lowerNibble <= 9) + || id == DELIMITED_STRUCT + || id == VARIABLE_LENGTH_INLINE_SYMBOL + || id == VARIABLE_LENGTH_STRUCT_WITH_FLEXSYMS + || id == ANNOTATIONS_1_FLEX_SYM + || id == ANNOTATIONS_2_FLEX_SYM + || id == ANNOTATIONS_MANY_FLEX_SYM // Symbol values with inline text. || upperNibble == 0x9; IonType typeFromUpperNibble = BINARY_TOKEN_TYPES_1_1[upperNibble]; @@ -229,37 +251,37 @@ private IonTypeID(byte id, int minorVersion) { if (lowerNibble <= 0x8) { type = IonType.INT; length = lowerNibble; - } else if (lowerNibble >= 0xE) { + } else if (id == BOOLEAN_TRUE || id == BOOLEAN_FALSE) { type = IonType.BOOL; length = 0; } else { type = IonType.FLOAT; - if (lowerNibble == 0xA) { + if (id == FLOAT_ZERO_LENGTH) { length = 0; // 0e0 - } else if (lowerNibble == 0xB) { + } else if (id == FLOAT_16) { length = 2; - } else if (lowerNibble == 0xC) { + } else if (id == FLOAT_32) { length = 4; - } else if (lowerNibble == 0xD) { + } else if (id == FLOAT_64) { length = 8; } } } else if (upperNibble == 0xE) { - if (lowerNibble <= 0x3) { + if (id == SYMBOL_ADDRESS_1_BYTE || id == SYMBOL_ADDRESS_2_BYTES || id == SYMBOL_ADDRESS_MANY_BYTES) { type = IonType.SYMBOL; - length = lowerNibble == 0x3 ? -1 : lowerNibble; + length = id == SYMBOL_ADDRESS_MANY_BYTES ? -1 : lowerNibble; } else if (lowerNibble <= 0x9) { type = ION_TYPE_ANNOTATION_WRAPPER; - } else if (lowerNibble == 0xA) { - // null.null + } else if (id == NULL_UNTYPED) { type = IonType.NULL; isNull = true; length = 0; - } else if (lowerNibble == 0xB) { + } else if (id == NULL_TYPED) { // Typed null. Type byte follows. type = null; isNull = true; - } else if (lowerNibble <= 0xD) { + length = 1; + } else if (id == ONE_BYTE_NOP || id == VARIABLE_LENGTH_NOP) { isNopPad = true; type = null; length = variableLength ? -1 : 0; @@ -268,29 +290,28 @@ private IonTypeID(byte id, int minorVersion) { type = null; } } else { // 0xF - if (lowerNibble == 0) { - // Delimited end + if (id == DELIMITED_END_MARKER) { type = null; length = 0; - } else if (lowerNibble == 0x3 || lowerNibble == 0xC || lowerNibble == 0xD) { + } else if (id == DELIMITED_STRUCT || id == VARIABLE_LENGTH_STRUCT_WITH_SIDS || id == VARIABLE_LENGTH_STRUCT_WITH_FLEXSYMS) { type = IonType.STRUCT; - } else if (lowerNibble == 0x5) { + } else if (id == VARIABLE_LENGTH_INTEGER) { type = IonType.INT; - } else if (lowerNibble == 0x6) { + } else if (id == VARIABLE_LENGTH_DECIMAL) { type = IonType.DECIMAL; - } else if (lowerNibble == 0x7) { + } else if (id == VARIABLE_LENGTH_TIMESTAMP) { type = IonType.TIMESTAMP; - } else if (lowerNibble == 0x9) { + } else if (id == VARIABLE_LENGTH_INLINE_SYMBOL) { type = IonType.SYMBOL; - } else if (lowerNibble == 0x8) { + } else if (id == VARIABLE_LENGTH_STRING) { type = IonType.STRING; - } else if (lowerNibble == 0xE) { + } else if (id == VARIABLE_LENGTH_BLOB) { type = IonType.BLOB; - } else if (lowerNibble == 0xF) { + } else if (id == VARIABLE_LENGTH_CLOB) { type = IonType.CLOB; - } else if (lowerNibble == 0x1 || lowerNibble == 0xA) { + } else if (id == DELIMITED_LIST || id == VARIABLE_LENGTH_LIST) { type = IonType.LIST; - } else if (lowerNibble == 0x2 || lowerNibble == 0xB) { + } else if (id == DELIMITED_SEXP || id == VARIABLE_LENGTH_SEXP) { type = IonType.SEXP; } else { // 0x4 // Variable length macro invocation @@ -302,34 +323,34 @@ private IonTypeID(byte id, int minorVersion) { if (type == IonType.TIMESTAMP) { // Short-form timestamps. Long-form timestamps use the upper nibble 0xF, forcing them to take // the previous branch. - switch (lowerNibble) { - case 0x0: + switch (id) { + case TIMESTAMP_YEAR_PRECISION: length = 1; break; - case 0x1: - case 0x2: + case TIMESTAMP_MONTH_PRECISION: + case TIMESTAMP_DAY_PRECISION: length = 2; break; - case 0x3: + case TIMESTAMP_MINUTE_PRECISION: length = 4; break; - case 0x4: - case 0x8: - case 0x9: + case TIMESTAMP_SECOND_PRECISION: + case TIMESTAMP_MINUTE_PRECISION_WITH_OFFSET: + case TIMESTAMP_SECOND_PRECISION_WITH_OFFSET: length = 5; break; - case 0x5: + case TIMESTAMP_MILLIS_PRECISION: length = 6; break; - case 0x6: - case 0xA: + case TIMESTAMP_MICROS_PRECISION: + case TIMESTAMP_MILLIS_PRECISION_WITH_OFFSET: length = 7; break; - case 0x7: - case 0xB: + case TIMESTAMP_NANOS_PRECISION: + case TIMESTAMP_MICROS_PRECISION_WITH_OFFSET: length = 8; break; - case 0xC: + case TIMESTAMP_NANOS_PRECISION_WITH_OFFSET: length = 9; break; } @@ -339,7 +360,7 @@ private IonTypeID(byte id, int minorVersion) { } } } - isDelimited = upperNibble == 0xF && lowerNibble <= 0x3; + isDelimited = id == DELIMITED_LIST || id == DELIMITED_SEXP || id == DELIMITED_STRUCT; this.isNopPad = isNopPad; this.isNull = isNull; this.length = length; diff --git a/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java b/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java index bbac4ef5ed..21dce227ad 100644 --- a/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java +++ b/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java @@ -14,7 +14,7 @@ private Ion_1_1_Constants() {} public static final int NANOSECOND_SCALE = 9; public static final int MAX_MICROSECONDS = 999999; public static final int MICROSECOND_SCALE = 6; - public static final short MAX_MILLISECONDS = 999; + public static final int MAX_MILLISECONDS = 999; public static final int MILLISECOND_SCALE = 3; //////// Timestamp Field Constants //////// diff --git a/src/com/amazon/ion/impl/bin/OpCodes.java b/src/com/amazon/ion/impl/bin/OpCodes.java index a84c4bc853..1bcda996ef 100644 --- a/src/com/amazon/ion/impl/bin/OpCodes.java +++ b/src/com/amazon/ion/impl/bin/OpCodes.java @@ -50,15 +50,25 @@ private OpCodes() {} public static final byte ANNOTATIONS_MANY_FLEX_SYM = (byte) 0xE9; public static final byte NULL_UNTYPED = (byte) 0xEA; public static final byte NULL_TYPED = (byte) 0xEB; - // 0xEC, 0xED NOP + public static final byte ONE_BYTE_NOP = (byte) 0xEC; + public static final byte VARIABLE_LENGTH_NOP = (byte) 0xED; // 0xEE Reserved - // 0xEF System Macro Invocation + public static final byte SYSTEM_MACRO_INVOCATION = (byte) 0xEF; + public static final byte DELIMITED_END_MARKER = (byte) 0xF0; + public static final byte DELIMITED_LIST = (byte) 0xF1; + public static final byte DELIMITED_SEXP = (byte) 0xF2; + public static final byte DELIMITED_STRUCT = (byte) 0xF3; + public static final byte LENGTH_PREFIXED_MACRO_INVOCATION = (byte) 0xF4; public static final byte VARIABLE_LENGTH_INTEGER = (byte) 0xF5; public static final byte VARIABLE_LENGTH_DECIMAL = (byte) 0xF6; public static final byte VARIABLE_LENGTH_TIMESTAMP = (byte) 0xF7; public static final byte VARIABLE_LENGTH_STRING = (byte) 0xF8; public static final byte VARIABLE_LENGTH_INLINE_SYMBOL = (byte) 0xF9; + public static final byte VARIABLE_LENGTH_LIST = (byte) 0xFA; + public static final byte VARIABLE_LENGTH_SEXP = (byte) 0xFB; + public static final byte VARIABLE_LENGTH_STRUCT_WITH_SIDS = (byte) 0xFC; + public static final byte VARIABLE_LENGTH_STRUCT_WITH_FLEXSYMS = (byte) 0xFD; public static final byte VARIABLE_LENGTH_BLOB = (byte) 0xFE; public static final byte VARIABLE_LENGTH_CLOB = (byte) 0xFF; } diff --git a/test/com/amazon/ion/TestUtils.java b/test/com/amazon/ion/TestUtils.java index 80292a6b5d..100eac99fa 100644 --- a/test/com/amazon/ion/TestUtils.java +++ b/test/com/amazon/ion/TestUtils.java @@ -631,6 +631,28 @@ public static int byteLengthFromHexString(String hexString) { return (hexString.replaceAll("[^\\dA-F]", "").length()) / 2; } + /** + * Converts a string of octets in the given radix to a byte array. Octets must be separated by a space. + * @param octetString the string of space-separated octets. + * @param radix the radix of the octets in the string. + * @return a new byte array. + */ + private static byte[] octetStringToByteArray(String octetString, int radix) { + String[] bytesAsStrings = octetString.split(" "); + byte[] bytesAsBytes = new byte[bytesAsStrings.length]; + for (int i = 0; i < bytesAsBytes.length; i++) { + bytesAsBytes[i] = (byte) (Integer.parseInt(bytesAsStrings[i], radix) & 0xFF); + } + return bytesAsBytes; + } + + /** + * Converts a string of hex octets, such as "BE EF", to a byte array. + */ + public static byte[] hexStringToByteArray(String hexString) { + return octetStringToByteArray(hexString, 16); + } + /** * Converts a byte array to a string of bits, such as "00110110 10001001". * The purpose of this method is to make it easier to read and write test assertions. @@ -661,12 +683,7 @@ public static int byteLengthFromBitString(String bitString) { * Converts a string of bits, such as "00110110 10001001", to a byte array. */ public static byte[] bitStringToByteArray(String bitString) { - String[] bytesAsBits = bitString.split(" "); - byte[] bytesAsBytes = new byte[bytesAsBits.length]; - for (int i = 0; i < bytesAsBytes.length; i++) { - bytesAsBytes[i] = (byte) (Integer.parseInt(bytesAsBits[i], 2) & 0xFF); - } - return bytesAsBytes; + return octetStringToByteArray(bitString, 2); } /** diff --git a/test/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java b/test/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java index 86fb93f608..40d1fe8461 100644 --- a/test/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java +++ b/test/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java @@ -4,6 +4,7 @@ package com.amazon.ion.impl; import com.amazon.ion.BufferConfiguration; +import com.amazon.ion.IntegerSize; import com.amazon.ion.IonBufferConfiguration; import com.amazon.ion.IonDatagram; import com.amazon.ion.IonException; @@ -42,6 +43,7 @@ import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; @@ -56,6 +58,7 @@ import static com.amazon.ion.TestUtils.StringToTimestamp; import static com.amazon.ion.TestUtils.bitStringToByteArray; import static com.amazon.ion.TestUtils.gzippedBytes; +import static com.amazon.ion.TestUtils.hexStringToByteArray; import static com.amazon.ion.impl.IonCursorTestUtilities.Expectation; import static com.amazon.ion.impl.IonCursorTestUtilities.ExpectationProvider; import static com.amazon.ion.impl.IonCursorTestUtilities.type; @@ -344,10 +347,47 @@ static ExpectationProvider container(IonType }; } + static ExpectationProvider nullValue(IonType expectedType) { + return consumer -> consumer.accept(new Expectation<>( + String.format("null(%s)", expectedType), + reader -> { + assertTrue(reader.isNullValue()); + assertEquals(expectedType, reader.getType()); + } + )); + } + + static ExpectationProvider booleanValue(boolean expectedValue) { + return consumer -> consumer.accept(new Expectation<>( + String.format("boolean(%s)", expectedValue), + reader -> assertEquals(expectedValue, reader.booleanValue()) + )); + } + static ExpectationProvider intValue(int expectedValue) { return consumer -> consumer.accept(new Expectation<>( String.format("int(%d)", expectedValue), - reader -> assertEquals(expectedValue, reader.intValue()) + reader -> { + assertEquals(IntegerSize.INT, reader.getIntegerSize()); + assertEquals(expectedValue, reader.intValue()); + } + )); + } + + static ExpectationProvider longValue(long expectedValue) { + return consumer -> consumer.accept(new Expectation<>( + String.format("long(%d)", expectedValue), + reader -> { + assertTrue(reader.getIntegerSize().ordinal() <= IntegerSize.LONG.ordinal()); + assertEquals(expectedValue, reader.longValue()); + } + )); + } + + static ExpectationProvider bigIntegerValue(BigInteger expectedValue) { + return consumer -> consumer.accept(new Expectation<>( + String.format("bigInteger(%s)", expectedValue), + reader -> assertEquals(expectedValue, reader.bigIntegerValue()) )); } @@ -3523,12 +3563,311 @@ public void earlyStepOutNonIncremental(boolean constructFromBytes) throws Except } /** - * Checks that the reader reads the expected timestamp value from the given input bits. + * Creates an IonReader over the given data, which will be prepended with a binary Ion 1.1 IVM. + * @param data the data to read. + * @param constructFromBytes whether to construct the reader from bytes or an InputStream. + * @return a new reader. */ - private void assertIonTimestampCorrectlyParsed(boolean constructFromBytes, Timestamp expected, String inputBits) throws Exception { - byte[] inputBytes = new TestUtils.BinaryIonAppender(1).append(bitStringToByteArray(inputBits)).toByteArray(); + private IonReader readerForIon11(byte[] data, boolean constructFromBytes) throws Exception { + byte[] inputBytes = new TestUtils.BinaryIonAppender(1).append(data).toByteArray(); reader = readerFor(readerBuilder, constructFromBytes, inputBytes); byteCounter.set(0); + return reader; + } + + /** + * Checks that the reader reads a null value of the expected type from the given input bytes. + */ + private void assertNullCorrectlyParsed(boolean constructFromBytes, IonType expectedType, String inputBytes) throws Exception { + reader = readerForIon11(hexStringToByteArray(inputBytes), constructFromBytes); + assertSequence( + next(expectedType), nullValue(expectedType), + next(null) + ); + closeAndCount(); + } + + @ParameterizedTest + @CsvSource({ + " NULL, EA", + " BOOL, EB 00", + " INT, EB 01", + " FLOAT, EB 02", + " DECIMAL, EB 03", + "TIMESTAMP, EB 04", + " STRING, EB 05", + " SYMBOL, EB 06", + " BLOB, EB 07", + " CLOB, EB 08", + " LIST, EB 09", + " SEXP, EB 0A", + " STRUCT, EB 0B", + }) + public void readNullValue(IonType expectedType, String inputBytes) throws Exception { + assertNullCorrectlyParsed(true, expectedType, inputBytes); + assertNullCorrectlyParsed(false, expectedType, inputBytes); + } + + /** + * Checks that the reader reads the expected boolean from the given input bits. + */ + private void assertBooleanCorrectlyParsed(boolean constructFromBytes, boolean expectedValue, String inputBytes) throws Exception { + reader = readerForIon11(hexStringToByteArray(inputBytes), constructFromBytes); + assertSequence( + next(IonType.BOOL), booleanValue(expectedValue), + next(null) + ); + closeAndCount(); + } + + @ParameterizedTest + @CsvSource({ + "true, 5E", + "false, 5F", + }) + public void readBooleanValue(Boolean expectedValue, String inputBytes) throws Exception { + assertBooleanCorrectlyParsed(true, expectedValue, inputBytes); + assertBooleanCorrectlyParsed(false, expectedValue, inputBytes); + } + + /** + * Checks that the reader reads the expected int from the given input bits. + */ + private void assertIntCorrectlyParsed(boolean constructFromBytes, int expectedValue, String inputBytes) throws Exception { + reader = readerForIon11(hexStringToByteArray(inputBytes), constructFromBytes); + assertSequence( + next(IonType.INT), intValue(expectedValue), + next(null) + ); + closeAndCount(); + } + + @ParameterizedTest + @CsvSource({ + " 0, 50", + " 1, 51 01", + " 17, 51 11", + " 127, 51 7F", + " 128, 52 80 00", + " 5555, 52 B3 15", + " 32767, 52 FF 7F", + " 32768, 53 00 80 00", + " 292037, 53 C5 74 04", + " 321672342, 54 96 54 2C 13", + " 2147483647, 54 FF FF FF 7F", // Integer.MAX_VALUE + " -1, 51 FF", + " -2, 51 FE", + " -14, 51 F2", + " -128, 51 80", + " -129, 52 7F FF", + " -944, 52 50 FC", + " -32768, 52 00 80", + " -32769, 53 FF 7F FF", + " -8388608, 53 00 00 80", + " -8388609, 54 FF FF 7F FF", + " -2147483648, 54 00 00 00 80", // Integer.MIN_VALUE + }) + public void readIntValue(int expectedValue, String inputBytes) throws Exception { + assertIntCorrectlyParsed(true, expectedValue, inputBytes); + assertIntCorrectlyParsed(false, expectedValue, inputBytes); + } + + /** + * Checks that the reader reads the expected long from the given input bits. + */ + private void assertLongCorrectlyParsed(boolean constructFromBytes, long expectedValue, String inputBytes) throws Exception { + reader = readerForIon11(hexStringToByteArray(inputBytes), constructFromBytes); + assertSequence( + next(IonType.INT), longValue(expectedValue), + next(null) + ); + closeAndCount(); + } + + @ParameterizedTest + @CsvSource({ + " 0, 50", + " 1, 51 01", + " 17, 51 11", + " 127, 51 7F", + " 127, 52 7F 00", + " 127, 54 7F 00 00 00", + " 127, 58 7F 00 00 00 00 00 00 00", + " 128, 52 80 00", + " 5555, 52 B3 15", + " 32767, 52 FF 7F", + " 32768, 53 00 80 00", + " 292037, 53 C5 74 04", + " 321672342, 54 96 54 2C 13", + " 2147483647, 54 FF FF FF 7F", // Integer.MAX_VALUE + " 64121672342, 55 96 12 F3 ED 0E", + " 1274120283167, 56 1F A4 7C A7 28 01", + " 851274120283167, 57 1F C4 8B B3 3A 06 03", + " 72624976668147840, 58 80 40 20 10 08 04 02 01", + " 9223372036854775807, 58 FF FF FF FF FF FF FF 7F", // Long.MAX_VALUE + " -1, 51 FF", + " -2, 51 FE", + " -14, 51 F2", + " -128, 51 80", + " -129, 52 7F FF", + " -944, 52 50 FC", + " -32768, 52 00 80", + " -32769, 53 FF 7F FF", + " -8388608, 53 00 00 80", + " -8388609, 54 FF FF 7F FF", + " -2147483648, 54 00 00 00 80", // Integer.MIN_VALUE + " -72624976668147841, 58 7F BF DF EF F7 FB FD FE", + " -9223372036854775808, 58 00 00 00 00 00 00 00 80", // Long.MIN_VALUE + }) + public void readLongValue(long expectedValue, String inputBytes) throws Exception { + assertLongCorrectlyParsed(true, expectedValue, inputBytes); + assertLongCorrectlyParsed(false, expectedValue, inputBytes); + } + + @ParameterizedTest + @CsvSource({ + " 0, F5 01", + " 1, F5 03 01", + " 17, F5 03 11", + " 127, F5 03 7F", + " 128, F5 05 80 00", + " 2147483647, F5 09 FF FF FF 7F", // Integer.MAX_VALUE + " 72624976668147840, F5 11 80 40 20 10 08 04 02 01", + " 9223372036854775807, F5 11 FF FF FF FF FF FF FF 7F", // Long.MAX_VALUE + " -1, F5 03 FF", + " -2, F5 03 FE", + " -14, F5 03 F2", + " -128, F5 03 80", + " -129, F5 05 7F FF", + " -2147483648, F5 09 00 00 00 80", // Integer.MIN_VALUE + " -72624976668147841, F5 11 7F BF DF EF F7 FB FD FE", + " -9223372036854775808, F5 11 00 00 00 00 00 00 00 80", // Long.MIN_VALUE + }) + public void readLongValueFromVariableLengthEncoding(long expectedValue, String inputBytes) throws Exception { + assertLongCorrectlyParsed(true, expectedValue, inputBytes); + assertLongCorrectlyParsed(false, expectedValue, inputBytes); + } + + /** + * Checks that the reader reads the expected BigInteger from the given input bits. + */ + private void assertBigIntegerCorrectlyParsed(boolean constructFromBytes, BigInteger expectedValue, String inputBytes) throws Exception { + reader = readerForIon11(hexStringToByteArray(inputBytes), constructFromBytes); + assertSequence( + next(IonType.INT), bigIntegerValue(expectedValue), + next(null) + ); + closeAndCount(); + } + + @ParameterizedTest + @CsvSource({ + " 0, 50", + " 1, 51 01", + " 17, 51 11", + " 127, 51 7F", + " 127, 52 7F 00", + " 127, 54 7F 00 00 00", + " 127, 58 7F 00 00 00 00 00 00 00", + " 128, 52 80 00", + " 5555, 52 B3 15", + " 32767, 52 FF 7F", + " 32768, 53 00 80 00", + " 292037, 53 C5 74 04", + " 321672342, 54 96 54 2C 13", + " 2147483647, 54 FF FF FF 7F", // Integer.MAX_VALUE + " 64121672342, 55 96 12 F3 ED 0E", + " 1274120283167, 56 1F A4 7C A7 28 01", + " 851274120283167, 57 1F C4 8B B3 3A 06 03", + " 72624976668147840, 58 80 40 20 10 08 04 02 01", + " 9223372036854775807, 58 FF FF FF FF FF FF FF 7F", // Long.MAX_VALUE + " 9223372036854775808, F5 13 00 00 00 00 00 00 00 80 00", + "999999999999999999999999999999, F5 1B FF FF FF 3F EA ED 74 46 D0 9C 2C 9F 0C", + " -1, 51 FF", + " -2, 51 FE", + " -14, 51 F2", + " -128, 51 80", + " -129, 52 7F FF", + " -944, 52 50 FC", + " -32768, 52 00 80", + " -32769, 53 FF 7F FF", + " -8388608, 53 00 00 80", + " -8388609, 54 FF FF 7F FF", + " -2147483648, 54 00 00 00 80", // Integer.MIN_VALUE + " -72624976668147841, 58 7F BF DF EF F7 FB FD FE", + " -9223372036854775808, 58 00 00 00 00 00 00 00 80", // Long.MIN_VALUE + " -9223372036854775809, F5 13 FF FF FF FF FF FF FF 7F FF", + "-99999999999999999999999999999, F5 1B 01 00 00 60 35 E8 8D 92 51 F0 E1 BC FE", + }) + public void readBigIntegerValue(BigInteger expectedValue, String inputBytes) throws Exception { + assertBigIntegerCorrectlyParsed(true, expectedValue, inputBytes); + assertBigIntegerCorrectlyParsed(false, expectedValue, inputBytes); + } + + /** + * Checks that the reader reads the expected double from the given input bits. + */ + private void assertDoubleCorrectlyParsed(boolean constructFromBytes, double expectedValue, String inputBytes) throws Exception { + reader = readerForIon11(hexStringToByteArray(inputBytes), constructFromBytes); + assertSequence( + next(IonType.FLOAT), doubleValue(expectedValue), + next(null) + ); + closeAndCount(); + } + + @ParameterizedTest + @CsvSource({ + " 0.0, 5A", + " 0.0, 5B 00 00", + " 0.0, 5C 00 00 00 00", + " 0.0, 5D 00 00 00 00 00 00 00 00", + " -0.0, 5B 80 00", + " -0.0, 5C 80 00 00 00", + " -0.0, 5D 80 00 00 00 00 00 00 00", + " 1.0, 5B 3C 00", + " 1.0, 5C 3F 80 00 00", + " 1.0, 5D 3F F0 00 00 00 00 00 00", + " 1.5, 5C 3F C0 00 00", + " 0.00006103515625, 5B 04 00", // Smallest positive normal half-precision float + " 0.333251953125, 5B 35 55", // Nearest half-precision representation of one third + " 3.141592653589793, 5D 40 09 21 FB 54 44 2D 18", + " 4.00537109375, 5C 40 80 2C 00", + " 4.11111111111, 5D 40 10 71 C7 1C 71 C2 39", + " 65504, 5B 7B FF", // Largest normal half-precision float + " 423542.09375, 5C 48 CE CE C3", + " 8236423542.09375, 5D 41 FE AE DD 97 61 80 00", + " 1.79769313486231570e+308, 5D 7F EF FF FF FF FF FF FF", // Double.MAX_VALUE + " -1.0, 5C BF 80 00 00", + " -1.5, 5C BF C0 00 00", + " -2, 5B C0 00", + " -3.141592653589793, 5D C0 09 21 FB 54 44 2D 18", + " -4.00537109375, 5C C0 80 2C 00", + " -4.11111111111, 5D C0 10 71 C7 1C 71 C2 39", + " -65504, 5B FB FF", // Smallest normal half-precision float + " -423542.09375, 5C C8 CE CE C3", + " -8236423542.09375, 5D C1 FE AE DD 97 61 80 00", + "-1.79769313486231570e+308, 5D FF EF FF FF FF FF FF FF", // Double.MIN_VALUE + " NaN, 5B 7C 01", + " Infinity, 5B 7C 00", + " -Infinity, 5B FC 00", + " NaN, 5C 7F C0 00 00", + " Infinity, 5C 7F 80 00 00", + " -Infinity, 5C FF 80 00 00", + " NaN, 5D 7F F0 00 00 00 00 00 01", + " Infinity, 5D 7F F0 00 00 00 00 00 00", + " -Infinity, 5D FF F0 00 00 00 00 00 00", + }) + public void readDoubleValue(double expectedValue, String inputBytes) throws Exception { + assertDoubleCorrectlyParsed(true, expectedValue, inputBytes); + assertDoubleCorrectlyParsed(false, expectedValue, inputBytes); + } + + /** + * Checks that the reader reads the expected timestamp value from the given input bits. + */ + private void assertIonTimestampCorrectlyParsed(boolean constructFromBytes, Timestamp expected, String inputBits) throws Exception { + reader = readerForIon11(bitStringToByteArray(inputBits), constructFromBytes); assertSequence( next(IonType.TIMESTAMP), timestampValue(expected), next(null) @@ -3629,7 +3968,7 @@ public void readTimestampValueWithKnownOffsetShortForm(@ConvertWith(StringToTime @ParameterizedTest @CsvSource({ - // OpCode Length YYYYYYYY MMYYYYYY HDDDDDMM mmmmHHHH oooooomm ssoooooo ....ssss Coefficient+ Scale + // OpCode Length YYYYYYYY MMYYYYYY HDDDDDMM mmmmHHHH oooooomm ssoooooo ....ssss Scale+ Coefficient "0001T, 11110111 00000101 00000001 00000000", "1947T, 11110111 00000101 10011011 00000111", "9999T, 11110111 00000101 00001111 00100111", @@ -3642,37 +3981,43 @@ public void readTimestampValueWithKnownOffsetShortForm(@ConvertWith(StringToTime "1947-12-23T23:59Z, 11110111 00001101 10011011 00000111 11011111 10111011 10000011 00010110", "1947-12-23T23:59:00Z, 11110111 00001111 10011011 00000111 11011111 10111011 10000011 00010110 00000000", "1947-12-23T23:59:59Z, 11110111 00001111 10011011 00000111 11011111 10111011 10000011 11010110 00001110", - "1947-12-23T23:59:00.0Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000001", - "1947-12-23T23:59:00.00Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000010", - "1947-12-23T23:59:00.000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000011", - "1947-12-23T23:59:00.0000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000100", - "1947-12-23T23:59:00.00000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000101", - "1947-12-23T23:59:00.000000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000110", - "1947-12-23T23:59:00.0000000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000111", - "1947-12-23T23:59:00.00000000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00001000", - "1947-12-23T23:59:00.9Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00010011 00000001", - "1947-12-23T23:59:00.99Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11000111 00000010", - "1947-12-23T23:59:00.999Z, 11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 10011110 00001111 00000011", - "1947-12-23T23:59:00.9999Z, 11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00111110 10011100 00000100", - "1947-12-23T23:59:00.99999Z, 11110111 00010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111100 00110100 00001100 00000101", - "1947-12-23T23:59:00.999999Z, 11110111 00010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111100 00010001 01111010 00000110", - "1947-12-23T23:59:00.9999999Z, 11110111 00011001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111000 01100111 10001001 00001001 00000111", - "1947-12-23T23:59:00.99999999Z, 11110111 00011001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111000 00001111 01011110 01011111 00001000", + "1947-12-23T23:59:00.0Z, 11110111 00010001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000011", + "1947-12-23T23:59:00.00Z, 11110111 00010001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000101", + "1947-12-23T23:59:00.000Z, 11110111 00010001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000111", + "1947-12-23T23:59:00.0000Z, 11110111 00010001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00001001", + "1947-12-23T23:59:00.00000Z, 11110111 00010001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00001011", + "1947-12-23T23:59:00.000000Z, 11110111 00010001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00001101", + "1947-12-23T23:59:00.0000000Z, 11110111 00010001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00001111", + "1947-12-23T23:59:00.00000000Z, 11110111 00010001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00010001", + "1947-12-23T23:59:00.9Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000011 00001001", + "1947-12-23T23:59:00.99Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000101 01100011", + "1947-12-23T23:59:00.999Z, 11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000111 11100111 00000011", + "1947-12-23T23:59:00.9999Z, 11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00001001 00001111 00100111", + "1947-12-23T23:59:00.99999Z, 11110111 00010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00001011 10011111 10000110 00000001", + "1947-12-23T23:59:00.999999Z, 11110111 00010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00001101 00111111 01000010 00001111", + "1947-12-23T23:59:00.9999999Z, 11110111 00011001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00001111 01111111 10010110 10011000 00000000", + "1947-12-23T23:59:00.99999999Z, 11110111 00011001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00010001 11111111 11100000 11110101 00000101", + + "1947-12-23T23:59:00.9223372036854775807Z, " + // Long.MAX_VALUE + "11110111 00100001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00100111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 01111111", + + "1947-12-23T23:59:00.9223372036854775808Z, " + // Long.MAX_VALUE + 1 (unsigned) + "11110111 00100001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00100111 00000000 00000000 00000000 00000000 00000000 00000000 00000000 10000000", "1947-12-23T23:59:00.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000Z, " + - "11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 10001101", + "11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00110110 00000010", "1947-12-23T23:59:00.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000Z, " + - "11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 01101000 00000001", + "11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 10100010 00000101", "1947-12-23T23:59:00.999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999Z, " + - "11110111 10010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 " + - "11111100 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 " + - "11111111 10010100 10001001 01111001 01101100 11001110 01111000 11110010 01000000 01111101 10100110 11000111 10101000 01000110 01011001 01110001 01001101 " + - "00100000 11110101 01101110 01111010 00001100 00001001 11101111 01111111 11110011 00011110 00010100 11010111 01101000 01110111 10101100 01101100 10001110 " + - "00110010 10110111 10000010 11110010 00110110 01101000 11110010 10100111 10001101", + "11110111 10001001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00110110 00000010 11111111 11111111 11111111 11111111 11111111 11111111 " + + "11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 10011111 00110010 00110001 10001111 11001101 00011001 " + + "01001111 00011110 10101000 11001111 11110100 00011000 11010101 00101000 00101011 10101110 00001001 10100100 11011110 01001101 10001111 00100001 11100001 " + + "11111101 01101111 11011110 10000011 11100010 00011010 11101101 10001110 10010101 11001101 01010001 11100110 01010110 01010000 11011110 00000110 01001101 " + + "11111110 00010100", // Offsets "2048-01-01T01:01-23:59, 11110111 00001101 00000000 01001000 10000100 00010000 00000100 00000000",