diff --git a/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java new file mode 100644 index 0000000000..7b1ca9d4b2 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/IonEncoder_1_1.java @@ -0,0 +1,559 @@ +package com.amazon.ion.impl.bin; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IonText; +import com.amazon.ion.IonType; +import com.amazon.ion.Timestamp; +import com.amazon.ion.impl.bin.utf8.Utf8StringEncoder; +import com.amazon.ion.impl.bin.utf8.Utf8StringEncoderPool; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; + +import static com.amazon.ion.impl.bin.Ion_1_1_Constants.*; +import static java.lang.Double.doubleToRawLongBits; +import static java.lang.Float.floatToIntBits; + +/** + * Provides functions for writing various Ion values to a WriteBuffer. + * + * This class can be subsumed by IonRawBinaryWriter_1_1, when it is created. + */ +public class IonEncoder_1_1 { + + /** + * Writes an Ion Null value to the given WriteBuffer. + * @return the number of bytes written + */ + public static int writeNullValue(WriteBuffer buffer, final IonType ionType) { + if (ionType == IonType.NULL) { + buffer.writeByte(OpCodes.NULL_UNTYPED); + return 1; + } + + buffer.writeByte(OpCodes.NULL_TYPED); + switch (ionType) { + case BOOL: + buffer.writeByte((byte) 0x00); + break; + case INT: + buffer.writeByte((byte) 0x01); + break; + case FLOAT: + buffer.writeByte((byte) 0x02); + break; + case DECIMAL: + buffer.writeByte((byte) 0x03); + break; + case TIMESTAMP: + buffer.writeByte((byte) 0x04); + break; + case STRING: + buffer.writeByte((byte) 0x05); + break; + case SYMBOL: + buffer.writeByte((byte) 0x06); + break; + case BLOB: + buffer.writeByte((byte) 0x07); + break; + case CLOB: + buffer.writeByte((byte) 0x08); + break; + case LIST: + buffer.writeByte((byte) 0x09); + break; + case SEXP: + buffer.writeByte((byte) 0x0A); + break; + case STRUCT: + buffer.writeByte((byte) 0x0B); + break; + case DATAGRAM: + throw new IllegalArgumentException("Cannot write a null datagram"); + } + return 2; + } + + /** + * Writes an Ion Bool value to the given WriteBuffer. + * @return the number of bytes written + */ + public static int writeBoolValue(WriteBuffer buffer, final boolean value) { + if (value) { + buffer.writeByte(OpCodes.BOOLEAN_TRUE); + } else { + buffer.writeByte(OpCodes.BOOLEAN_FALSE); + } + return 1; + } + + /** + * Writes an Ion Integer value to the given WriteBuffer. + * @return the number of bytes written + */ + public static int writeIntValue(WriteBuffer buffer, final long value) { + if (value == 0) { + buffer.writeByte(OpCodes.INTEGER_ZERO_LENGTH); + return 1; + } + int length = WriteBuffer.fixedIntLength(value); + buffer.writeByte((byte) (OpCodes.INTEGER_ZERO_LENGTH + length)); + buffer.writeFixedInt(value); + return 1 + length; + } + + private static final BigInteger BIG_INT_LONG_MAX_VALUE = BigInteger.valueOf(Long.MAX_VALUE); + private static final BigInteger BIG_INT_LONG_MIN_VALUE = BigInteger.valueOf(Long.MIN_VALUE); + + /** + * Writes an Ion Integer value to the given WriteBuffer. + * @return the number of bytes written + */ + public static int writeIntValue(WriteBuffer buffer, final BigInteger value) { + if (value == null) { + return writeNullValue(buffer, IonType.INT); + } + if (value.compareTo(BIG_INT_LONG_MIN_VALUE) >= 0 && value.compareTo(BIG_INT_LONG_MAX_VALUE) <= 0) { + return writeIntValue(buffer, value.longValue()); + } + buffer.writeByte(OpCodes.VARIABLE_LENGTH_INTEGER); + byte[] intBytes = value.toByteArray(); + int totalBytes = 1 + intBytes.length + buffer.writeFlexUInt(intBytes.length); + for (int i = intBytes.length; i > 0; i--) { + buffer.writeByte(intBytes[i-1]); + } + return totalBytes; + } + + /** + * Writes a float to the given WriteBuffer using the Ion 1.1 encoding for Ion Floats. + * @return the number of bytes written + */ + public static int writeFloat(WriteBuffer buffer, final float value) { + // TODO: Optimization to write a 16 bit float for non-finite and possibly other values + if (value == 0.0) { + buffer.writeByte(OpCodes.FLOAT_ZERO_LENGTH); + return 1; + } else { + buffer.writeByte(OpCodes.FLOAT_32); + buffer.writeUInt32(floatToIntBits(value)); + return 5; + } + } + + /** + * Writes a double to the given WriteBuffer using the Ion 1.1 encoding for Ion Floats. + * @return the number of bytes written + */ + public static int writeFloat(WriteBuffer buffer, final double value) { + // TODO: Optimization to write a 16 bit float for non-finite and possibly other values + if (value == 0.0) { + buffer.writeByte(OpCodes.FLOAT_ZERO_LENGTH); + return 1; + } else if (!Double.isFinite(value) || value == (float) value) { + buffer.writeByte(OpCodes.FLOAT_32); + buffer.writeUInt32(floatToIntBits((float) value)); + return 5; + } else { + buffer.writeByte(OpCodes.FLOAT_64); + buffer.writeUInt64(doubleToRawLongBits(value)); + return 9; + } + } + + public static int writeDecimalValue(WriteBuffer buffer, final BigDecimal value) { + if (value == null) { + return writeNullValue(buffer, IonType.DECIMAL); + } + + int exponent = -value.scale(); + + if (BigDecimal.ZERO.compareTo(value) == 0 && !Decimal.isNegativeZero(value)) { + if (exponent == 0) { + buffer.writeByte(OpCodes.DECIMAL_ZERO_LENGTH); + return 1; + } else { + // A decimal with a coefficient of +0 is encoded using opcode 6F. + // The opcode is followed by a FlexInt representing the exponent. + buffer.writeByte(OpCodes.POSITIVE_ZERO_DECIMAL); + return 1 + buffer.writeFlexInt(exponent); + } + } + + BigInteger coefficient = value.unscaledValue(); + int numCoefficientBytes = WriteBuffer.flexIntLength(coefficient); + + int numExponentBytes = 0; + if (exponent != 0) { + numExponentBytes = WriteBuffer.fixedIntLength(exponent); + } + + int opCodeAndLengthBytes = 1; + if (numExponentBytes + numCoefficientBytes < 15) { + int opCode = OpCodes.DECIMAL_ZERO_LENGTH + numExponentBytes + numCoefficientBytes; + buffer.writeByte((byte) opCode); + } else { + // Decimal values that require more than 14 bytes can be encoded using the variable-length decimal opcode: 0xF6. + buffer.writeByte(OpCodes.VARIABLE_LENGTH_DECIMAL); + opCodeAndLengthBytes += buffer.writeFlexUInt(numExponentBytes + numCoefficientBytes); + } + buffer.writeFlexInt(coefficient); + if (exponent != 0) { + buffer.writeFixedInt(exponent); + } + + return opCodeAndLengthBytes + numCoefficientBytes + numExponentBytes; + } + + /** + * Writes a Timestamp to the given WriteBuffer using the Ion 1.1 encoding for Ion Timestamps. + * @return the number of bytes written + */ + public static int writeTimestampValue(WriteBuffer buffer, Timestamp value) { + if (value == null) { + return writeNullValue(buffer, IonType.TIMESTAMP); + } + // Timestamps may be encoded using the short form if they meet certain conditions. + // Condition 1: The year is between 1970 and 2097. + if (value.getYear() < 1970 || value.getYear() > 2097) { + return writeLongFormTimestampValue(buffer, value); + } + + // If the precision is year, month, or day, we can skip the remaining checks. + if (!value.getPrecision().includes(Timestamp.Precision.MINUTE)) { + return writeShortFormTimestampValue(buffer, value); + } + + // Condition 2: The fractional seconds are a common precision. + if (value.getZFractionalSecond() != null) { + int secondsScale = value.getZFractionalSecond().scale(); + if (secondsScale != 0 && secondsScale != 3 && secondsScale != 6 && secondsScale != 9) { + return writeLongFormTimestampValue(buffer, value); + } + } + // Condition 3: The local offset is either UTC, unknown, or falls between -14:00 to +14:00 and is divisible by 15 minutes. + Integer offset = value.getLocalOffset(); + if (offset != null && (offset < -14 * 60 || offset > 14 * 60 || offset % 15 != 0)) { + return writeLongFormTimestampValue(buffer, value); + } + return writeShortFormTimestampValue(buffer, value); + } + + /** + * Writes a short-form timestamp. + * Value cannot be null. + * If calling from outside this class, use writeTimestampValue instead. + */ + private static int writeShortFormTimestampValue(WriteBuffer buffer, Timestamp value) { + long bits = (value.getYear() - 1970L); + if (value.getPrecision() == Timestamp.Precision.YEAR) { + buffer.writeByte(OpCodes.TIMESTAMP_YEAR_PRECISION); + buffer.writeFixedIntOrUInt(bits, 1); + return 2; + } + + bits |= ((long) value.getMonth()) << S_TIMESTAMP_MONTH_BIT_OFFSET; + if (value.getPrecision() == Timestamp.Precision.MONTH) { + buffer.writeByte(OpCodes.TIMESTAMP_MONTH_PRECISION); + buffer.writeFixedIntOrUInt(bits, 2); + return 3; + } + + bits |= ((long) value.getDay()) << S_TIMESTAMP_DAY_BIT_OFFSET; + if (value.getPrecision() == Timestamp.Precision.DAY) { + buffer.writeByte(OpCodes.TIMESTAMP_DAY_PRECISION); + buffer.writeFixedIntOrUInt(bits, 2); + return 3; + } + + bits |= ((long) value.getHour()) << S_TIMESTAMP_HOUR_BIT_OFFSET; + bits |= ((long) value.getMinute()) << S_TIMESTAMP_MINUTE_BIT_OFFSET; + if (value.getLocalOffset() == null || value.getLocalOffset() == 0) { + if (value.getLocalOffset() != null) { + bits |= S_U_TIMESTAMP_UTC_FLAG; + } + + if (value.getPrecision() == Timestamp.Precision.MINUTE) { + buffer.writeByte(OpCodes.TIMESTAMP_MINUTE_PRECISION); + buffer.writeFixedIntOrUInt(bits, 4); + return 5; + } + + bits |= ((long) value.getSecond()) << S_U_TIMESTAMP_SECOND_BIT_OFFSET; + + int secondsScale = 0; + if (value.getZFractionalSecond() != null) { + secondsScale = value.getZFractionalSecond().scale(); + } + if (secondsScale != 0) { + long fractionalSeconds = value.getZFractionalSecond().unscaledValue().longValue(); + bits |= fractionalSeconds << S_U_TIMESTAMP_FRACTION_BIT_OFFSET; + } + switch (secondsScale) { + case 0: + buffer.writeByte(OpCodes.TIMESTAMP_SECOND_PRECISION); + buffer.writeFixedIntOrUInt(bits, 5); + return 6; + case 3: + buffer.writeByte(OpCodes.TIMESTAMP_MILLIS_PRECISION); + buffer.writeFixedIntOrUInt(bits, 6); + return 7; + case 6: + buffer.writeByte(OpCodes.TIMESTAMP_MICROS_PRECISION); + buffer.writeFixedIntOrUInt(bits, 7); + return 8; + case 9: + buffer.writeByte(OpCodes.TIMESTAMP_NANOS_PRECISION); + buffer.writeFixedIntOrUInt(bits, 8); + return 9; + default: + throw new IllegalStateException("This is unreachable!"); + } + } else { + long localOffset = (value.getLocalOffset().longValue() / 15) + (14 * 4); + bits |= (localOffset & LEAST_SIGNIFICANT_7_BITS) << S_O_TIMESTAMP_OFFSET_BIT_OFFSET; + + if (value.getPrecision() == Timestamp.Precision.MINUTE) { + buffer.writeByte(OpCodes.TIMESTAMP_MINUTE_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + return 6; + } + + bits |= ((long) value.getSecond()) << S_O_TIMESTAMP_SECOND_BIT_OFFSET; + + // The fractional seconds bits will be put into a separate long because we need nine bytes total + // if there are nanoseconds (which is too much for one long) and the boundary between the seconds + // and fractional seconds subfields conveniently aligns with a byte boundary. + long fractionBits = 0; + int secondsScale = 0; + if (value.getZFractionalSecond() != null) { + secondsScale = value.getZFractionalSecond().scale(); + } + if (secondsScale != 0) { + fractionBits = value.getZFractionalSecond().unscaledValue().longValue(); + } + switch (secondsScale) { + case 0: + buffer.writeByte(OpCodes.TIMESTAMP_SECOND_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + return 6; + case 3: + buffer.writeByte(OpCodes.TIMESTAMP_MILLIS_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + buffer.writeFixedIntOrUInt(fractionBits, 2); + return 8; + case 6: + buffer.writeByte(OpCodes.TIMESTAMP_MICROS_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + buffer.writeFixedIntOrUInt(fractionBits, 3); + return 9; + case 9: + buffer.writeByte(OpCodes.TIMESTAMP_NANOS_PRECISION_WITH_OFFSET); + buffer.writeFixedIntOrUInt(bits, 5); + buffer.writeFixedIntOrUInt(fractionBits, 4); + return 10; + default: + throw new IllegalStateException("This is unreachable!"); + } + } + } + + /** + * Writes a long-form timestamp. + * Value may not be null. + * Only visible for testing. If calling from outside this class, use writeTimestampValue instead. + */ + static int writeLongFormTimestampValue(WriteBuffer buffer, Timestamp value) { + buffer.writeByte(OpCodes.VARIABLE_LENGTH_TIMESTAMP); + + long bits = value.getYear(); + if (value.getPrecision() == Timestamp.Precision.YEAR) { + buffer.writeFlexUInt(2); + buffer.writeFixedIntOrUInt(bits, 2); + return 4; // OpCode + FlexUInt + 2 bytes data + } + + bits |= ((long) value.getMonth()) << L_TIMESTAMP_MONTH_BIT_OFFSET; + if (value.getPrecision() == Timestamp.Precision.MONTH) { + buffer.writeFlexUInt(3); + buffer.writeFixedIntOrUInt(bits, 3); + return 5; // OpCode + FlexUInt + 3 bytes data + } + + bits |= ((long) value.getDay()) << L_TIMESTAMP_DAY_BIT_OFFSET; + if (value.getPrecision() == Timestamp.Precision.DAY) { + buffer.writeFlexUInt(3); + buffer.writeFixedIntOrUInt(bits, 3); + return 5; // OpCode + FlexUInt + 3 bytes data + } + + bits |= ((long) value.getHour()) << L_TIMESTAMP_HOUR_BIT_OFFSET; + bits |= ((long) value.getMinute()) << L_TIMESTAMP_MINUTE_BIT_OFFSET; + long localOffsetValue = L_TIMESTAMP_UNKNOWN_OFFSET_VALUE; + if (value.getLocalOffset() != null) { + localOffsetValue = value.getLocalOffset() + (24 * 60); + } + bits |= localOffsetValue << L_TIMESTAMP_OFFSET_BIT_OFFSET; + + if (value.getPrecision() == Timestamp.Precision.MINUTE) { + buffer.writeFlexUInt(6); + buffer.writeFixedIntOrUInt(bits, 6); + return 8; // OpCode + FlexUInt + 6 bytes data + } + + + bits |= ((long) value.getSecond()) << L_TIMESTAMP_SECOND_BIT_OFFSET; + int secondsScale = 0; + if (value.getZFractionalSecond() != null) { + secondsScale = value.getZFractionalSecond().scale(); + } + if (secondsScale == 0) { + buffer.writeFlexUInt(7); + buffer.writeFixedIntOrUInt(bits, 7); + return 9; // OpCode + FlexUInt + 7 bytes data + } + + BigDecimal fractionalSeconds = value.getZFractionalSecond(); + BigInteger coefficient = fractionalSeconds.unscaledValue(); + long exponent = fractionalSeconds.scale(); + int numCoefficientBytes = WriteBuffer.flexUIntLength(coefficient); + int numExponentBytes = WriteBuffer.fixedUIntLength(exponent); + // Years-seconds data (7 bytes) + fraction coefficient + fraction exponent + int dataLength = 7 + numCoefficientBytes + numExponentBytes; + + buffer.writeFlexUInt(dataLength); + buffer.writeFixedIntOrUInt(bits, 7); + buffer.writeFlexUInt(coefficient); + buffer.writeFixedUInt(exponent); + + // OpCode + FlexUInt length + dataLength + return 1 + WriteBuffer.flexUIntLength(dataLength) + dataLength; + } + + /** + * Writes a String to the given WriteBuffer using the Ion 1.1 encoding for Ion Strings. + * @return the number of bytes written + */ + public static int writeStringValue(WriteBuffer buffer, String value) { + return writeInlineText(buffer, value, IonType.STRING, OpCodes.STRING_ZERO_LENGTH, OpCodes.VARIABLE_LENGTH_STRING); + } + + /** + * Writes an inline Symbol to the given WriteBuffer using the Ion 1.1 encoding for Ion Symbols. + * @return the number of bytes written + */ + public static int writeSymbolValue(WriteBuffer buffer, String value) { + return writeInlineText(buffer, value, IonType.SYMBOL, OpCodes.INLINE_SYMBOL_ZERO_LENGTH, OpCodes.VARIABLE_LENGTH_INLINE_SYMBOL); + } + + private static int writeInlineText(WriteBuffer buffer, String value, IonType type, byte zeroLengthOpCode, byte variableLengthOpCode) { + if (value == null) { + return writeNullValue(buffer, type); + } + + // TODO: When merging into the Ion 1.1 raw writer, keep a single instance of the Utf8StringEncoder + // instead of fetching one on every call. + Utf8StringEncoder.Result encoderResult = Utf8StringEncoderPool.getInstance().getOrCreate().encode(value); + + byte[] utf8Buffer = encoderResult.getBuffer(); + int numValueBytes = encoderResult.getEncodedLength(); + int numLengthBytes = 0; + + if (numValueBytes <= 0xF) { + buffer.writeByte((byte)(zeroLengthOpCode | numValueBytes)); + } else { + buffer.writeByte(variableLengthOpCode); + numLengthBytes = buffer.writeFlexUInt(numValueBytes); + } + buffer.writeBytes(utf8Buffer, 0, numValueBytes); + return 1 + numLengthBytes + numValueBytes; + } + + /** + * Writes an interned Symbol's address to the given WriteBuffer using the Ion 1.1 encoding for Ion Symbols. + * @return the number of bytes written + * + * TODO: Do we need to support Symbol Addresses greater than Long.MAX_VALUE? + */ + public static int writeSymbolValue(WriteBuffer buffer, long value) { + if (value < 0) { + throw new IllegalArgumentException("Symbol Address cannot be negative; was: " + value); + } else if (value < FIRST_2_BYTE_SYMBOL_ADDRESS) { + buffer.writeByte(OpCodes.SYMBOL_ADDRESS_1_BYTE); + buffer.writeFixedUInt(value); + return 2; + } else if (value < FIRST_MANY_BYTE_SYMBOL_ADDRESS) { + buffer.writeByte(OpCodes.SYMBOL_ADDRESS_2_BYTES); + buffer.writeFixedIntOrUInt(value - FIRST_2_BYTE_SYMBOL_ADDRESS, 2); + return 3; + } else { + buffer.writeByte(OpCodes.SYMBOL_ADDRESS_MANY_BYTES); + int addressBytes = buffer.writeFlexUInt(value - FIRST_MANY_BYTE_SYMBOL_ADDRESS); + return 1 + addressBytes; + } + } + + /** + * Writes a Blob to the given WriteBuffer using the Ion 1.1 encoding for Ion Blobs. + * @return the number of bytes written + */ + public static int writeBlobValue(WriteBuffer buffer, byte[] value) { + if (value == null) { + return writeNullValue(buffer, IonType.BLOB); + } + + buffer.writeByte(OpCodes.VARIABLE_LENGTH_BLOB); + int numLengthBytes = buffer.writeFlexUInt(value.length); + buffer.writeBytes(value); + return 1 + numLengthBytes + value.length; + } + + /** + * Writes a Clob to the given WriteBuffer using the Ion 1.1 encoding for Ion Clobs. + * @return the number of bytes written + */ + public static int writeClobValue(WriteBuffer buffer, byte[] value) { + if (value == null) { + return writeNullValue(buffer, IonType.CLOB); + } + + buffer.writeByte(OpCodes.VARIABLE_LENGTH_CLOB); + int numLengthBytes = buffer.writeFlexUInt(value.length); + buffer.writeBytes(value); + return 1 + numLengthBytes + value.length; + } + + // TODO: Implement FlexSym Annotations + + /** + * Writes annotations using the given symbol addresses. + */ + public static int writeAnnotations(WriteBuffer buffer, long[] annotations) { + if (annotations == null || annotations.length == 0) { + return 0; + } + if (annotations.length == 1) { + buffer.writeByte(OpCodes.ANNOTATIONS_1_SYMBOL_ADDRESS); + int numAddressBytes = buffer.writeFlexUInt(annotations[0]); + return 1 + numAddressBytes; + } else if (annotations.length == 2) { + buffer.writeByte(OpCodes.ANNOTATIONS_2_SYMBOL_ADDRESS); + int numAddressBytes = buffer.writeFlexUInt(annotations[0]); + numAddressBytes += buffer.writeFlexUInt(annotations[1]); + return 1 + numAddressBytes; + } else { + int numAddressBytes = 0; + for (long ann : annotations) { + numAddressBytes += WriteBuffer.flexUIntLength(ann); + } + buffer.writeByte(OpCodes.ANNOTATIONS_MANY_SYMBOL_ADDRESS); + int numLengthBytes = buffer.writeFlexUInt(numAddressBytes); + for (long ann : annotations) { + buffer.writeFlexUInt(ann); + } + return 1 + numLengthBytes + numAddressBytes; + } + } +} diff --git a/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java b/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java new file mode 100644 index 0000000000..8df501b349 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/Ion_1_1_Constants.java @@ -0,0 +1,39 @@ +package com.amazon.ion.impl.bin; + +/** + * Contains constants (other than OpCodes) which are generally applicable to both reading and writing binary Ion 1.1 + */ +public class Ion_1_1_Constants { + private Ion_1_1_Constants() {} + + static final int FIRST_2_BYTE_SYMBOL_ADDRESS = 256; + static final int FIRST_MANY_BYTE_SYMBOL_ADDRESS = 65792; + + //////// Timestamp Field Constants //////// + + // S_TIMESTAMP_* is applicable to all short-form timestamps + static final int S_TIMESTAMP_MONTH_BIT_OFFSET = 7; + static final int S_TIMESTAMP_DAY_BIT_OFFSET = 11; + static final int S_TIMESTAMP_HOUR_BIT_OFFSET = 16; + static final int S_TIMESTAMP_MINUTE_BIT_OFFSET = 21; + // S_U_TIMESTAMP_* is applicable to all short-form timestamps with a `U` bit + static final int S_U_TIMESTAMP_UTC_FLAG = 1 << 27; + static final int S_U_TIMESTAMP_SECOND_BIT_OFFSET = 28; + static final int S_U_TIMESTAMP_FRACTION_BIT_OFFSET = 34; + // S_O_TIMESTAMP_* is applicable to all short-form timestamps with `o` (offset) bits + static final int S_O_TIMESTAMP_OFFSET_BIT_OFFSET = 27; + static final int S_O_TIMESTAMP_SECOND_BIT_OFFSET = 34; + + // L_TIMESTAMP_* is applicable to all long-form timestamps + static final int L_TIMESTAMP_MONTH_BIT_OFFSET = 14; + static final int L_TIMESTAMP_DAY_BIT_OFFSET = 18; + static final int L_TIMESTAMP_HOUR_BIT_OFFSET = 23; + static final int L_TIMESTAMP_MINUTE_BIT_OFFSET = 28; + static final int L_TIMESTAMP_OFFSET_BIT_OFFSET = 34; + static final int L_TIMESTAMP_SECOND_BIT_OFFSET = 46; + static final int L_TIMESTAMP_UNKNOWN_OFFSET_VALUE = 0b111111111111; + + //////// Bit masks //////// + + static final long LEAST_SIGNIFICANT_7_BITS = 0b01111111L; +} diff --git a/src/com/amazon/ion/impl/bin/OpCodes.java b/src/com/amazon/ion/impl/bin/OpCodes.java new file mode 100644 index 0000000000..a84c4bc853 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/OpCodes.java @@ -0,0 +1,64 @@ +package com.amazon.ion.impl.bin; + +/** + * Utility class holding Ion 1.1 Op Codes. + */ +public class OpCodes { + private OpCodes() {} + + public static final byte INTEGER_ZERO_LENGTH = 0x50; + // 0x51-0x58 are additional lengths of integers. + // 0x59 Reserved + public static final byte FLOAT_ZERO_LENGTH = 0x5A; + public static final byte FLOAT_16 = 0x5B; + public static final byte FLOAT_32 = 0x5C; + public static final byte FLOAT_64 = 0x5D; + public static final byte BOOLEAN_TRUE = 0x5E; + public static final byte BOOLEAN_FALSE = 0x5F; + + public static final byte DECIMAL_ZERO_LENGTH = 0x60; + // 0x61-0x6E are additional lengths of decimals. + public static final byte POSITIVE_ZERO_DECIMAL = 0x6F; + + public static final byte TIMESTAMP_YEAR_PRECISION = 0x70; + public static final byte TIMESTAMP_MONTH_PRECISION = 0x71; + public static final byte TIMESTAMP_DAY_PRECISION = 0x72; + public static final byte TIMESTAMP_MINUTE_PRECISION = 0x73; + public static final byte TIMESTAMP_SECOND_PRECISION = 0x74; + public static final byte TIMESTAMP_MILLIS_PRECISION = 0x75; + public static final byte TIMESTAMP_MICROS_PRECISION = 0x76; + public static final byte TIMESTAMP_NANOS_PRECISION = 0x77; + public static final byte TIMESTAMP_MINUTE_PRECISION_WITH_OFFSET = 0x78; + public static final byte TIMESTAMP_SECOND_PRECISION_WITH_OFFSET = 0x79; + public static final byte TIMESTAMP_MILLIS_PRECISION_WITH_OFFSET = 0x7A; + public static final byte TIMESTAMP_MICROS_PRECISION_WITH_OFFSET = 0x7B; + public static final byte TIMESTAMP_NANOS_PRECISION_WITH_OFFSET = 0x7C; + // 0x7D-0x7F Reserved + + public static final byte STRING_ZERO_LENGTH = (byte) 0x80; + + public static final byte INLINE_SYMBOL_ZERO_LENGTH = (byte) 0x90; + + public static final byte SYMBOL_ADDRESS_1_BYTE = (byte) 0xE1; + public static final byte SYMBOL_ADDRESS_2_BYTES = (byte) 0xE2; + public static final byte SYMBOL_ADDRESS_MANY_BYTES = (byte) 0xE3; + public static final byte ANNOTATIONS_1_SYMBOL_ADDRESS = (byte) 0xE4; + public static final byte ANNOTATIONS_2_SYMBOL_ADDRESS = (byte) 0xE5; + public static final byte ANNOTATIONS_MANY_SYMBOL_ADDRESS = (byte) 0xE6; + public static final byte ANNOTATIONS_1_FLEX_SYM = (byte) 0xE7; + public static final byte ANNOTATIONS_2_FLEX_SYM = (byte) 0xE8; + public static final byte ANNOTATIONS_MANY_FLEX_SYM = (byte) 0xE9; + public static final byte NULL_UNTYPED = (byte) 0xEA; + public static final byte NULL_TYPED = (byte) 0xEB; + // 0xEC, 0xED NOP + // 0xEE Reserved + // 0xEF System Macro Invocation + + public static final byte VARIABLE_LENGTH_INTEGER = (byte) 0xF5; + public static final byte VARIABLE_LENGTH_DECIMAL = (byte) 0xF6; + public static final byte VARIABLE_LENGTH_TIMESTAMP = (byte) 0xF7; + public static final byte VARIABLE_LENGTH_STRING = (byte) 0xF8; + public static final byte VARIABLE_LENGTH_INLINE_SYMBOL = (byte) 0xF9; + public static final byte VARIABLE_LENGTH_BLOB = (byte) 0xFE; + public static final byte VARIABLE_LENGTH_CLOB = (byte) 0xFF; +} diff --git a/src/com/amazon/ion/impl/bin/WriteBuffer.java b/src/com/amazon/ion/impl/bin/WriteBuffer.java index 7e67af00bd..d8bf4162fc 100644 --- a/src/com/amazon/ion/impl/bin/WriteBuffer.java +++ b/src/com/amazon/ion/impl/bin/WriteBuffer.java @@ -18,6 +18,7 @@ import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; +import java.math.BigInteger; import java.util.ArrayList; import java.util.List; @@ -1360,6 +1361,64 @@ private int writeFlexIntOrUInt(final long value, final int numBytes) { return numBytes; } + public static int flexIntLength(final BigInteger value) { + return value.bitLength() / 7 + 1; + } + + public static int flexUIntLength(final BigInteger value) { + return (value.bitLength() - 1) / 7 + 1; + } + + public int writeFlexInt(final BigInteger value) { + int numBytes = flexIntLength(value); + return writeFlexIntOrUIntForBigInteger(value, numBytes); + } + + public int writeFlexUInt(final BigInteger value) { + if (value.signum() < 0) { + throw new IllegalArgumentException("Attempted to write a FlexUInt for " + value); + } + int numBytes = flexUIntLength(value); + return writeFlexIntOrUIntForBigInteger(value, numBytes); + } + + private int writeFlexIntOrUIntForBigInteger(final BigInteger value, final int numBytes) { + // TODO: Should we branch to the implementation for long if the number is small enough? + // https://github.com/amazon-ion/ion-java/issues/614 + byte[] valueBytes = value.toByteArray(); + + int i = 0; // `i` gets incremented for every byte written. + + // Start with leading zero bytes. + // If there's 1-8 total bytes, we need no leading zero-bytes. + // If there's 9-16 total bytes, we need one zero-byte + // If there's 17-24 total bytes, we need two zero-bytes, etc. + for (; i < (numBytes - 1)/8; i++) { + writeByte((byte) 0); + } + + // Write the last length bits, possibly also containing some value bits. + int remainingLengthBits = (numBytes - 1) % 8; + byte lengthPart = (byte) (0x01 << remainingLengthBits); + int valueBitOffset = remainingLengthBits + 1; + byte valuePart = (byte) (valueBytes[valueBytes.length - 1] << valueBitOffset); + writeByte((byte) (valuePart | lengthPart)); + i++; + + for (int valueByteOffset = valueBytes.length - 1; valueByteOffset > 0; valueByteOffset--) { + // Technically it's only a nibble if the bitOffset is 4, so we call it nibble-ish + byte highNibbleIsh = (byte) (valueBytes[valueByteOffset - 1] << (valueBitOffset)); + byte lowNibbleIsh = (byte) ((valueBytes[valueByteOffset] & 0xFF) >> (8 - valueBitOffset)); + writeByte((byte) (highNibbleIsh | lowNibbleIsh)); + i++; + } + if (i < numBytes) { + writeByte((byte) ((valueBytes[0]) >> (8 - valueBitOffset))); + } + + return numBytes; + } + /** Get the length of FixedInt for the provided value. */ public static int fixedIntLength(final long value) { int numMagnitudeBitsRequired; @@ -1379,7 +1438,7 @@ public static int fixedIntLength(final long value) { */ public int writeFixedInt(final long value) { int numBytes = fixedIntLength(value); - return writeFixedIntOrUInt(value, numBytes); + return _writeFixedIntOrUInt(value, numBytes); } /** Get the length of FixedUInt for the provided value. */ @@ -1395,17 +1454,40 @@ public static int fixedUIntLength(final long value) { */ public int writeFixedUInt(final long value) { if (value < 0) { - throw new IllegalArgumentException("Attempted to write a FlexUInt for " + value); + throw new IllegalArgumentException("Attempted to write a FixedUInt for " + value); } int numBytes = fixedUIntLength(value); - return writeFixedIntOrUInt(value, numBytes); + return _writeFixedIntOrUInt(value, numBytes); } /** - * Because the fixed int and fixed uint encodings are so similar, we can use this method to write either one as long - * as we provide the correct number of bytes needed to encode the value. + * Writes the bytes of a {@code long} as a {@code FixedInt} or {@code FixedUInt} using {@code numBytes} bytes. + *

+ * {@code numBytes} should be an integer from 1 to 8 inclusive. If {@code numBytes} is out of bounds, that is a + * programmer error and will result in an IllegalArgumentException. + *

+ * Because the {@code FixedInt} and {@code FixedUInt} encodings are so similar, we can use this method to write + * either one as long as we provide the correct number of bytes needed to encode the value. + *

+ * Most of the time, you should not use this method. Instead, use {@link WriteBuffer#writeFixedInt} or + * {@link WriteBuffer#writeFixedUInt}, which calculate the minimum number of required bytes to represent the value. + *

+ * You should use this method when the spec requires a {@code FixedInt} or {@code FixedUInt} of a specific + * size when it's possible that the value could fit in a smaller FixedInt or FixedUInt than the size required in + * the spec. + */ + public int writeFixedIntOrUInt(final long value, final int numBytes) { + if (0 > numBytes || numBytes > 8) { + throw new IllegalArgumentException("numBytes is out of bounds; was " + numBytes); + } + return _writeFixedIntOrUInt(value, numBytes); + } + + /** + * Because the {@code FixedInt} and {@code FixedUInt} encodings are so similar, we can use this method to write + * either one as long as we provide the correct number of bytes needed to encode the value. */ - private int writeFixedIntOrUInt(final long value, final int numBytes) { + private int _writeFixedIntOrUInt(final long value, final int numBytes) { writeByte((byte) value); if (numBytes > 1) { writeByte((byte) (value >> 8)); diff --git a/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java b/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java new file mode 100644 index 0000000000..478433660c --- /dev/null +++ b/test/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java @@ -0,0 +1,723 @@ +package com.amazon.ion.impl.bin; + +import com.amazon.ion.BitUtils; +import com.amazon.ion.Decimal; +import com.amazon.ion.IonType; +import com.amazon.ion.Timestamp; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.converter.ArgumentConversionException; +import org.junit.jupiter.params.converter.ConvertWith; +import org.junit.jupiter.params.converter.TypedArgumentConverter; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiFunction; + +public class IonEncoder_1_1Test { + + private static BlockAllocator ALLOCATOR = BlockAllocatorProviders.basicProvider().vendAllocator(11); + private WriteBuffer buf; + + @BeforeEach + public void setup() { + buf = new WriteBuffer(ALLOCATOR); + } + + private byte[] bytes() { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + buf.writeTo(out); + } catch (final IOException e) { + throw new IllegalStateException(e); + } + return out.toByteArray(); + } + + /** + * Checks that the function writes the expected bytes and returns the expected count of written bytes for the + * given input value. The expected bytes should be a string of space-separated hexadecimal pairs. + */ + private void assertWritingValue(String expectedBytes, T value, BiFunction writeOperation) { + int numBytes = writeOperation.apply(buf, value); + Assertions.assertEquals(expectedBytes, byteArrayToHex(bytes())); + Assertions.assertEquals(byteLengthFromHexString(expectedBytes), numBytes); + } + + /** + * Checks that the function writes the expected bytes and returns the expected count of written bytes for the + * given input value. The expected bytes should be a string of space-separated hexadecimal pairs. + */ + private void assertWritingValue(byte[] expectedBytes, T value, BiFunction writeOperation) { + int numBytes = writeOperation.apply(buf, value); + Assertions.assertEquals(expectedBytes, bytes()); + Assertions.assertEquals(expectedBytes.length, numBytes); + } + + /** + * Checks that the function writes the expected bytes and returns the expected count of written bytes for the + * given input value. The expectedBytes should be a string of space-separated binary octets. + */ + private void assertWritingValueWithBinary(String expectedBytes, T value, BiFunction writeOperation) { + int numBytes = writeOperation.apply(buf, value); + Assertions.assertEquals(expectedBytes, byteArrayToBitString(bytes())); + Assertions.assertEquals(byteLengthFromBitString(expectedBytes), numBytes); + } + + @ParameterizedTest + @CsvSource({ + " NULL, EA", + " BOOL, EB 00", + " INT, EB 01", + " FLOAT, EB 02", + " DECIMAL, EB 03", + "TIMESTAMP, EB 04", + " STRING, EB 05", + " SYMBOL, EB 06", + " BLOB, EB 07", + " CLOB, EB 08", + " LIST, EB 09", + " SEXP, EB 0A", + " STRUCT, EB 0B", + }) + public void testWriteNullValue(IonType value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeNullValue); + } + + @Test + public void testWriteNullValueForDatagram() { + Assertions.assertThrows(IllegalArgumentException.class, () -> IonEncoder_1_1.writeNullValue(buf, IonType.DATAGRAM)); + } + + @ParameterizedTest + @CsvSource({ + "true, 5E", + "false, 5F", + }) + public void testWriteBooleanValue(boolean value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeBoolValue); + } + + @ParameterizedTest + @CsvSource({ + " 0, 50", + " 1, 51 01", + " 17, 51 11", + " 127, 51 7F", + " 128, 52 80 00", + " 5555, 52 B3 15", + " 32767, 52 FF 7F", + " 32768, 53 00 80 00", + " 292037, 53 C5 74 04", + " 321672342, 54 96 54 2C 13", + " 64121672342, 55 96 12 F3 ED 0E", + " 1274120283167, 56 1F A4 7C A7 28 01", + " 851274120283167, 57 1F C4 8B B3 3A 06 03", + " 72624976668147840, 58 80 40 20 10 08 04 02 01", + " 9223372036854775807, 58 FF FF FF FF FF FF FF 7F", // Long.MAX_VALUE + " -1, 51 FF", + " -2, 51 FE", + " -14, 51 F2", + " -128, 51 80", + " -129, 52 7F FF", + " -944, 52 50 FC", + " -32768, 52 00 80", + " -32769, 53 FF 7F FF", + " -8388608, 53 00 00 80", + " -8388609, 54 FF FF 7F FF", + " -72624976668147841, 58 7F BF DF EF F7 FB FD FE", + "-9223372036854775808, 58 00 00 00 00 00 00 00 80", // Long.MIN_VALUE + }) + public void testWriteIntegerValue(long value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeIntValue); + } + + @ParameterizedTest + @CsvSource({ + " 0, 50", + " 1, 51 01", + " 17, 51 11", + " 127, 51 7F", + " 128, 52 80 00", + " 5555, 52 B3 15", + " 32767, 52 FF 7F", + " 32768, 53 00 80 00", + " 292037, 53 C5 74 04", + " 321672342, 54 96 54 2C 13", + " 64121672342, 55 96 12 F3 ED 0E", + " 1274120283167, 56 1F A4 7C A7 28 01", + " 851274120283167, 57 1F C4 8B B3 3A 06 03", + " 72624976668147840, 58 80 40 20 10 08 04 02 01", + " 9223372036854775807, 58 FF FF FF FF FF FF FF 7F", // Long.MAX_VALUE + " 9223372036854775808, F5 13 00 00 00 00 00 00 00 80 00", + "999999999999999999999999999999, F5 1B FF FF FF 3F EA ED 74 46 D0 9C 2C 9F 0C", + " -1, 51 FF", + " -2, 51 FE", + " -14, 51 F2", + " -128, 51 80", + " -129, 52 7F FF", + " -944, 52 50 FC", + " -32768, 52 00 80", + " -32769, 53 FF 7F FF", + " -8388608, 53 00 00 80", + " -8388609, 54 FF FF 7F FF", + " -72624976668147841, 58 7F BF DF EF F7 FB FD FE", + " -9223372036854775808, 58 00 00 00 00 00 00 00 80", // Long.MIN_VALUE + " -9223372036854775809, F5 13 FF FF FF FF FF FF FF 7F FF", + "-99999999999999999999999999999, F5 1B 01 00 00 60 35 E8 8D 92 51 F0 E1 BC FE", + }) + public void testWriteIntegerValueForBigInteger(BigInteger value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeIntValue); + } + + @Test + public void testWriteIntegerValueForNullBigInteger() { + int numBytes = IonEncoder_1_1.writeIntValue(buf, null); + Assertions.assertEquals("EB 01", byteArrayToHex(bytes())); + Assertions.assertEquals(2, numBytes); + } + + @ParameterizedTest + @CsvSource({ + " 0.0, 5A", + " 1.0, 5C 3F 80 00 00", + " 1.5, 5C 3F C0 00 00", + " 3.1415927, 5C 40 49 0F DB", + " 4.00537109375, 5C 40 80 2C 00", + " 423542.09375, 5C 48 CE CE C3", + " 3.40282347E+38, 5C 7F 7F FF FF", // Float.MAX_VALUE + " -1.0, 5C BF 80 00 00", + " -1.5, 5C BF C0 00 00", + " -3.1415927, 5C C0 49 0F DB", + " -4.00537109375, 5C C0 80 2C 00", + " -423542.09375, 5C C8 CE CE C3", + "-3.40282347E+38, 5C FF 7F FF FF", // Float.MIN_VALUE + " NaN, 5C 7F C0 00 00", + " Infinity, 5C 7F 80 00 00", + " -Infinity, 5C FF 80 00 00", + }) + public void testWriteFloatValue(float value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeFloat); + } + + @ParameterizedTest + @CsvSource({ + " 0.0, 5A", + " 1.0, 5C 3F 80 00 00", + " 1.5, 5C 3F C0 00 00", + " 3.141592653589793, 5D 40 09 21 FB 54 44 2D 18", + " 4.00537109375, 5C 40 80 2C 00", + " 4.11111111111, 5D 40 10 71 C7 1C 71 C2 39", + " 423542.09375, 5C 48 CE CE C3", + " 8236423542.09375, 5D 41 FE AE DD 97 61 80 00", + " 1.79769313486231570e+308, 5D 7F EF FF FF FF FF FF FF", // Double.MAX_VALUE + " -1.0, 5C BF 80 00 00", + " -1.5, 5C BF C0 00 00", + " -3.141592653589793, 5D C0 09 21 FB 54 44 2D 18", + " -4.00537109375, 5C C0 80 2C 00", + " -4.11111111111, 5D C0 10 71 C7 1C 71 C2 39", + " -423542.09375, 5C C8 CE CE C3", + " -8236423542.09375, 5D C1 FE AE DD 97 61 80 00", + "-1.79769313486231570e+308, 5D FF EF FF FF FF FF FF FF", // Double.MIN_VALUE + " NaN, 5C 7F C0 00 00", + " Infinity, 5C 7F 80 00 00", + " -Infinity, 5C FF 80 00 00", + }) + public void testWriteFloatValueForDouble(double value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeFloat); + } + + @ParameterizedTest + @CsvSource({ + " 0., 60", + " 0e1, 6F 03", + " 0e63, 6F 7F", + " 0e99, 6F 8E 01", + " 0.0, 6F FF", + " 0.00, 6F FD", + " 0.000, 6F FB", + " 0e-64, 6F 81", + " 0e-99, 6F 76 FE", + " -0., 61 01", + " -0e1, 62 01 01", + " -0e3, 62 01 03", + " -0e127, 62 01 7F", + " -0e199, 63 01 C7 00", + " -0e-1, 62 01 FF", + " -0e-2, 62 01 FE", + " -0e-3, 62 01 FD", + " -0e-127, 62 01 81", + " -0e-199, 63 01 39 FF", + " 0.01, 62 03 FE", + " 0.1, 62 03 FF", + " 1, 61 03", + " 1e1, 62 03 01", + " 1e2, 62 03 02", + " 1e127, 62 03 7F", + " 1e128, 63 03 80 00", + " 1e65536, 64 03 00 00 01", + " 2, 61 05", + " 7, 61 0F", + " 14, 61 1D", + " 1.0, 62 15 FF", + " 1.00, 63 92 01 FE", + " 1.27, 63 FE 01 FE", + " 3.142, 63 1A 31 FD", + " 3.14159, 64 7C 59 26 FB", + " 3.141593, 65 98 FD FE 02 FA", + " 3.141592653, 66 B0 C9 1C 68 17 F7", + " 3.14159265359, 67 E0 93 7D 56 49 12 F5", + " 3.1415926535897932, 69 80 4C 43 76 65 9E 9C 6F F0", + " 3.1415926535897932384626434, 6E 00 50 E0 DC F7 CC D6 08 48 99 92 3F 03 E7", + "3.141592653589793238462643383, F6 1F 00 E0 2D 8F A4 21 D0 E7 46 C0 87 AA 89 02 E5", + }) + public void testWriteDecimalValue(@ConvertWith(StringToDecimal.class) Decimal value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeDecimalValue); + } + + @Test + public void testWriteDecimalValueForNull() { + int numBytes = IonEncoder_1_1.writeDecimalValue(buf, null); + Assertions.assertEquals("EB 03", byteArrayToHex(bytes())); + Assertions.assertEquals(2, numBytes); + } + + // Because timestamp subfields are smeared across bytes, it's easier to reason about them in 1s and 0s + // instead of hex digits + @ParameterizedTest + @CsvSource({ + // OpCode MYYYYYYY DDDDDMMM mmmHHHHH ssssUmmm ffffffss ffffffff ffffffff ffffffff + "2023-10-15T01:00Z, 01110011 00110101 01111101 00000001 00001000", + "2023-10-15T01:59Z, 01110011 00110101 01111101 01100001 00001111", + "2023-10-15T11:22Z, 01110011 00110101 01111101 11001011 00001010", + "2023-10-15T23:00Z, 01110011 00110101 01111101 00010111 00001000", + "2023-10-15T23:59Z, 01110011 00110101 01111101 01110111 00001111", + "2023-10-15T11:22:00Z, 01110100 00110101 01111101 11001011 00001010 00000000", + "2023-10-15T11:22:33Z, 01110100 00110101 01111101 11001011 00011010 00000010", + "2023-10-15T11:22:59Z, 01110100 00110101 01111101 11001011 10111010 00000011", + "2023-10-15T11:22:33.000Z, 01110101 00110101 01111101 11001011 00011010 00000010 00000000", + "2023-10-15T11:22:33.444Z, 01110101 00110101 01111101 11001011 00011010 11110010 00000110", + "2023-10-15T11:22:33.999Z, 01110101 00110101 01111101 11001011 00011010 10011110 00001111", + "2023-10-15T11:22:33.000000Z, 01110110 00110101 01111101 11001011 00011010 00000010 00000000 00000000", + "2023-10-15T11:22:33.444555Z, 01110110 00110101 01111101 11001011 00011010 00101110 00100010 00011011", + "2023-10-15T11:22:33.999999Z, 01110110 00110101 01111101 11001011 00011010 11111110 00001000 00111101", + "2023-10-15T11:22:33.000000000Z, 01110111 00110101 01111101 11001011 00011010 00000010 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555666Z, 01110111 00110101 01111101 11001011 00011010 01001010 10000110 11111101 01101001", + "2023-10-15T11:22:33.999999999Z, 01110111 00110101 01111101 11001011 00011010 11111110 00100111 01101011 11101110", + }) + public void testWriteTimestampValueWithUtcShortForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeTimestampValue); + } + + + @ParameterizedTest + @CsvSource({ + // OpCode MYYYYYYY DDDDDMMM mmmHHHHH ssssUmmm ffffffss ffffffff ffffffff ffffffff + "1970T, 01110000 00000000", + "2023T, 01110000 00110101", + "2097T, 01110000 01111111", + "2023-01T, 01110001 10110101 00000000", + "2023-10T, 01110001 00110101 00000101", + "2023-12T, 01110001 00110101 00000110", + "2023-10-01T, 01110010 00110101 00001101", + "2023-10-15T, 01110010 00110101 01111101", + "2023-10-31T, 01110010 00110101 11111101", + "2023-10-15T01:00-00:00, 01110011 00110101 01111101 00000001 00000000", + "2023-10-15T01:59-00:00, 01110011 00110101 01111101 01100001 00000111", + "2023-10-15T11:22-00:00, 01110011 00110101 01111101 11001011 00000010", + "2023-10-15T23:00-00:00, 01110011 00110101 01111101 00010111 00000000", + "2023-10-15T23:59-00:00, 01110011 00110101 01111101 01110111 00000111", + "2023-10-15T11:22:00-00:00, 01110100 00110101 01111101 11001011 00000010 00000000", + "2023-10-15T11:22:33-00:00, 01110100 00110101 01111101 11001011 00010010 00000010", + "2023-10-15T11:22:59-00:00, 01110100 00110101 01111101 11001011 10110010 00000011", + "2023-10-15T11:22:33.000-00:00, 01110101 00110101 01111101 11001011 00010010 00000010 00000000", + "2023-10-15T11:22:33.444-00:00, 01110101 00110101 01111101 11001011 00010010 11110010 00000110", + "2023-10-15T11:22:33.999-00:00, 01110101 00110101 01111101 11001011 00010010 10011110 00001111", + "2023-10-15T11:22:33.000000-00:00, 01110110 00110101 01111101 11001011 00010010 00000010 00000000 00000000", + "2023-10-15T11:22:33.444555-00:00, 01110110 00110101 01111101 11001011 00010010 00101110 00100010 00011011", + "2023-10-15T11:22:33.999999-00:00, 01110110 00110101 01111101 11001011 00010010 11111110 00001000 00111101", + "2023-10-15T11:22:33.000000000-00:00, 01110111 00110101 01111101 11001011 00010010 00000010 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555666-00:00, 01110111 00110101 01111101 11001011 00010010 01001010 10000110 11111101 01101001", + "2023-10-15T11:22:33.999999999-00:00, 01110111 00110101 01111101 11001011 00010010 11111110 00100111 01101011 11101110", + }) + public void testWriteTimestampValueWithUnknownOffsetShortForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeTimestampValue); + } + + @ParameterizedTest + @CsvSource({ + // OpCode MYYYYYYY DDDDDMMM mmmHHHHH ooooommm ssssssoo ffffffff ffffffff ffffffff ..ffffff + "2023-10-15T01:00-14:00, 01111000 00110101 01111101 00000001 00000000 00000000", + "2023-10-15T01:00+14:00, 01111000 00110101 01111101 00000001 10000000 00000011", + "2023-10-15T01:00-01:15, 01111000 00110101 01111101 00000001 10011000 00000001", + "2023-10-15T01:00+01:15, 01111000 00110101 01111101 00000001 11101000 00000001", + "2023-10-15T01:59+01:15, 01111000 00110101 01111101 01100001 11101111 00000001", + "2023-10-15T11:22+01:15, 01111000 00110101 01111101 11001011 11101010 00000001", + "2023-10-15T23:00+01:15, 01111000 00110101 01111101 00010111 11101000 00000001", + "2023-10-15T23:59+01:15, 01111000 00110101 01111101 01110111 11101111 00000001", + "2023-10-15T11:22:00+01:15, 01111001 00110101 01111101 11001011 11101010 00000001", + "2023-10-15T11:22:33+01:15, 01111001 00110101 01111101 11001011 11101010 10000101", + "2023-10-15T11:22:59+01:15, 01111001 00110101 01111101 11001011 11101010 11101101", + "2023-10-15T11:22:33.000+01:15, 01111010 00110101 01111101 11001011 11101010 10000101 00000000 00000000", + "2023-10-15T11:22:33.444+01:15, 01111010 00110101 01111101 11001011 11101010 10000101 10111100 00000001", + "2023-10-15T11:22:33.999+01:15, 01111010 00110101 01111101 11001011 11101010 10000101 11100111 00000011", + "2023-10-15T11:22:33.000000+01:15, 01111011 00110101 01111101 11001011 11101010 10000101 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555+01:15, 01111011 00110101 01111101 11001011 11101010 10000101 10001011 11001000 00000110", + "2023-10-15T11:22:33.999999+01:15, 01111011 00110101 01111101 11001011 11101010 10000101 00111111 01000010 00001111", + "2023-10-15T11:22:33.000000000+01:15, 01111100 00110101 01111101 11001011 11101010 10000101 00000000 00000000 00000000 00000000", + "2023-10-15T11:22:33.444555666+01:15, 01111100 00110101 01111101 11001011 11101010 10000101 10010010 01100001 01111111 00011010", + "2023-10-15T11:22:33.999999999+01:15, 01111100 00110101 01111101 11001011 11101010 10000101 11111111 11001001 10011010 00111011", + + }) + public void testWriteTimestampValueWithKnownOffsetShortForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeTimestampValue); + } + + @ParameterizedTest + @CsvSource({ + // OpCode Length YYYYYYYY MMYYYYYY HDDDDDMM mmmmHHHH oooooomm ssoooooo ....ssss Coefficient+ Scale + "0001T, 11110111 00000101 00000001 00000000", + "1947T, 11110111 00000101 10011011 00000111", + "9999T, 11110111 00000101 00001111 00100111", + "1947-01T, 11110111 00000111 10011011 01000111 00000000", + "1947-12T, 11110111 00000111 10011011 00000111 00000011", + "1947-01-01T, 11110111 00000111 10011011 01000111 00000100", + "1947-12-23T, 11110111 00000111 10011011 00000111 01011111", + "1947-12-31T, 11110111 00000111 10011011 00000111 01111111", + "1947-12-23T00:00Z, 11110111 00001101 10011011 00000111 01011111 00000000 10000000 00010110", + "1947-12-23T23:59Z, 11110111 00001101 10011011 00000111 11011111 10111011 10000011 00010110", + "1947-12-23T23:59:00Z, 11110111 00001111 10011011 00000111 11011111 10111011 10000011 00010110 00000000", + "1947-12-23T23:59:59Z, 11110111 00001111 10011011 00000111 11011111 10111011 10000011 11010110 00001110", + "1947-12-23T23:59:00.0Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000001", + "1947-12-23T23:59:00.00Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000010", + "1947-12-23T23:59:00.000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000011", + "1947-12-23T23:59:00.0000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000100", + "1947-12-23T23:59:00.00000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000101", + "1947-12-23T23:59:00.000000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000110", + "1947-12-23T23:59:00.0000000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00000111", + "1947-12-23T23:59:00.00000000Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 00001000", + "1947-12-23T23:59:00.9Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00010011 00000001", + "1947-12-23T23:59:00.99Z, 11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11000111 00000010", + "1947-12-23T23:59:00.999Z, 11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 10011110 00001111 00000011", + "1947-12-23T23:59:00.9999Z, 11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00111110 10011100 00000100", + "1947-12-23T23:59:00.99999Z, 11110111 00010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111100 00110100 00001100 00000101", + "1947-12-23T23:59:00.999999Z, 11110111 00010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111100 00010001 01111010 00000110", + "1947-12-23T23:59:00.9999999Z, 11110111 00011001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111000 01100111 10001001 00001001 00000111", + "1947-12-23T23:59:00.99999999Z, 11110111 00011001 10011011 00000111 11011111 10111011 10000011 00010110 00000000 11111000 00001111 01011110 01011111 00001000", + + "1947-12-23T23:59:00.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000Z, " + + "11110111 00010011 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 10001101", + + "1947-12-23T23:59:00.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000Z, " + + "11110111 00010101 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000001 01101000 00000001", + + "1947-12-23T23:59:00.999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999Z, " + + "11110111 10010111 10011011 00000111 11011111 10111011 10000011 00010110 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 " + + "11111100 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 " + + "11111111 10010100 10001001 01111001 01101100 11001110 01111000 11110010 01000000 01111101 10100110 11000111 10101000 01000110 01011001 01110001 01001101 " + + "00100000 11110101 01101110 01111010 00001100 00001001 11101111 01111111 11110011 00011110 00010100 11010111 01101000 01110111 10101100 01101100 10001110 " + + "00110010 10110111 10000010 11110010 00110110 01101000 11110010 10100111 10001101", + + + // Offsets + "2048-01-01T01:01-23:59, 11110111 00001101 00000000 01001000 10000100 00010000 00000100 00000000", + "2048-01-01T01:01-00:02, 11110111 00001101 00000000 01001000 10000100 00010000 01111000 00010110", + "2048-01-01T01:01-00:01, 11110111 00001101 00000000 01001000 10000100 00010000 01111100 00010110", + "2048-01-01T01:01-00:00, 11110111 00001101 00000000 01001000 10000100 00010000 11111100 00111111", + "2048-01-01T01:01+00:00, 11110111 00001101 00000000 01001000 10000100 00010000 10000000 00010110", + "2048-01-01T01:01+00:01, 11110111 00001101 00000000 01001000 10000100 00010000 10000100 00010110", + "2048-01-01T01:01+00:02, 11110111 00001101 00000000 01001000 10000100 00010000 10001000 00010110", + "2048-01-01T01:01+23:59, 11110111 00001101 00000000 01001000 10000100 00010000 11111100 00101100", + }) + public void testWriteTimestampValueLongForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeLongFormTimestampValue); + } + + @ParameterizedTest + @CsvSource({ + // Long form because it's out of the year range + "0001T, 11110111 00000101 00000001 00000000", + "9999T, 11110111 00000101 00001111 00100111", + + // Long form because the offset is too high/low + "2048-01-01T01:01+14:15, 11110111 00001101 00000000 01001000 10000100 00010000 11011100 00100011", + "2048-01-01T01:01-14:15, 11110111 00001101 00000000 01001000 10000100 00010000 00100100 00001001", + + // Long form because the offset is not a multiple of 15 + "2048-01-01T01:01+00:01, 11110111 00001101 00000000 01001000 10000100 00010000 10000100 00010110", + + // Long form because the fractional seconds are millis, micros, or nanos + "2023-12-31T23:59:00.0Z, 11110111 00010011 11100111 00000111 11111111 10111011 10000011 00010110 00000000 00000001 00000001", + }) + public void testWriteTimestampDelegatesCorrectlyToLongForm(@ConvertWith(StringToTimestamp.class) Timestamp value, String expectedBytes) { + assertWritingValueWithBinary(expectedBytes, value, IonEncoder_1_1::writeTimestampValue); + } + + @Test + public void testWriteTimestampValueForNullTimestamp() { + int numBytes = IonEncoder_1_1.writeTimestampValue(buf, null); + Assertions.assertEquals("EB 04", byteArrayToHex(bytes())); + Assertions.assertEquals(2, numBytes); + } + + @ParameterizedTest + @CsvSource({ + "'', 80", + "'a', 81 61", + "'ab', 82 61 62", + "'abc', 83 61 62 63", + "'fourteen bytes', 8E 66 6F 75 72 74 65 65 6E 20 62 79 74 65 73", + "'this has sixteen', F8 21 74 68 69 73 20 68 61 73 20 73 69 78 74 65 65 6E", + "'variable length encoding', F8 31 76 61 72 69 61 62 6C 65 20 6C 65 6E 67 74 68 20 65 6E 63 6F 64 69 6E 67", + }) + public void testWriteStringValue(String value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeStringValue); + } + + @Test + public void testWriteStringValueForNull() { + int numBytes = IonEncoder_1_1.writeStringValue(buf, null); + Assertions.assertEquals("EB 05", byteArrayToHex(bytes())); + Assertions.assertEquals(2, numBytes); + } + + @ParameterizedTest + @CsvSource({ + "'', 90", + "'a', 91 61", + "'ab', 92 61 62", + "'abc', 93 61 62 63", + "'fourteen bytes', 9E 66 6F 75 72 74 65 65 6E 20 62 79 74 65 73", + "'this has sixteen', F9 21 74 68 69 73 20 68 61 73 20 73 69 78 74 65 65 6E", + "'variable length encoding', F9 31 76 61 72 69 61 62 6C 65 20 6C 65 6E 67 74 68 20 65 6E 63 6F 64 69 6E 67", + }) + public void testWriteSymbolValue(String value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeSymbolValue); + } + + @ParameterizedTest + @CsvSource({ + "0, E1 00", + "1, E1 01", + "255, E1 FF", + "256, E2 00 00", + "257, E2 01 00", + "512, E2 00 01", + "513, E2 01 01", + "65535, E2 FF FE", + "65791, E2 FF FF", + "65792, E3 01", + "65793, E3 03", + "65919, E3 FF", + "65920, E3 02 02", + "9223372036854775807, E3 00 FF FD FD FF FF FF FF FF" + }) + public void testWriteSymbolValue(long value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeSymbolValue); + } + + @Test + public void testWriteSymbolValueForNull() { + int numBytes = IonEncoder_1_1.writeSymbolValue(buf, null); + Assertions.assertEquals("EB 06", byteArrayToHex(bytes())); + Assertions.assertEquals(2, numBytes); + } + + @ParameterizedTest + @CsvSource({ + "'', FE 01", // + "20, FE 03 20", + "49 20 61 70 70 6C 61 75 64 20 79 6F 75 72 20 63 75 72 69 6F 73 69 74 79, " + + "FE 31 49 20 61 70 70 6C 61 75 64 20 79 6F 75 72 20 63 75 72 69 6F 73 69 74 79" + }) + public void testWriteBlobValue(@ConvertWith(HexStringToByteArray.class) byte[] value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeBlobValue); + } + + @Test + public void testWriteBlobValueForNull() { + int numBytes = IonEncoder_1_1.writeBlobValue(buf, null); + Assertions.assertEquals("EB 07", byteArrayToHex(bytes())); + Assertions.assertEquals(2, numBytes); + } + + @ParameterizedTest + @CsvSource({ + "'', FF 01", + "20, FF 03 20", + "49 20 61 70 70 6C 61 75 64 20 79 6F 75 72 20 63 75 72 69 6F 73 69 74 79, " + + "FF 31 49 20 61 70 70 6C 61 75 64 20 79 6F 75 72 20 63 75 72 69 6F 73 69 74 79" + }) + public void testWriteClobValue(@ConvertWith(HexStringToByteArray.class) byte[] value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeClobValue); + } + + @Test + public void testWriteClobValueForNull() { + int numBytes = IonEncoder_1_1.writeClobValue(buf, null); + Assertions.assertEquals("EB 08", byteArrayToHex(bytes())); + Assertions.assertEquals(2, numBytes); + } + + @ParameterizedTest + @CsvSource({ + " '', ''", // Empty array of annotations + " $0, E4 01", + " $10, E4 15", + " $256, E4 02 04", + " $10 $11, E5 15 17", + " $256 $257, E5 02 04 06 04", + " $10 $11 $12, E6 07 15 17 19", + "$256 $257 $258, E6 0D 02 04 06 04 0A 04", + }) + public void testWriteAnnotations(@ConvertWith(SymbolIdsToLongArray.class) long[] value, String expectedBytes) { + assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeAnnotations); + } + + @Test + public void testWriteAnnotationsForNull() { + int numBytes = IonEncoder_1_1.writeAnnotations(buf, null); + Assertions.assertEquals("", byteArrayToHex(bytes())); + Assertions.assertEquals(0, numBytes); + } + + /** + * Utility method to make it easier to write test cases that assert specific sequences of bytes. + */ + private static String byteArrayToHex(byte[] bytes) { + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02X ", b)); + } + return sb.toString().trim(); + } + + /** + * Determines the number of bytes needed to represent a series of hexadecimal digits. + */ + private static int byteLengthFromHexString(String hexString) { + return (hexString.replaceAll("[^\\dA-F]", "").length()) / 2; + } + + /** + * Converts a byte array to a string of bits, such as "00110110 10001001". + * The purpose of this method is to make it easier to read and write test assertions. + */ + private static String byteArrayToBitString(byte[] bytes) { + StringBuilder s = new StringBuilder(); + for (byte aByte : bytes) { + for (int bit = 7; bit >= 0; bit--) { + if (((0x01 << bit) & aByte) != 0) { + s.append("1"); + } else { + s.append("0"); + } + } + s.append(" "); + } + return s.toString().trim(); + } + + /** + * Determines the number of bytes needed to represent a series of hexadecimal digits. + */ + private static int byteLengthFromBitString(String bitString) { + return (bitString.replaceAll("[^01]", "").length()) / 8; + } + + /** + * Converts a String to a Timestamp for a @Parameterized test + */ + static class StringToTimestamp extends TypedArgumentConverter { + protected StringToTimestamp() { + super(String.class, Timestamp.class); + } + + @Override + protected Timestamp convert(String source) throws ArgumentConversionException { + if (source == null) return null; + return Timestamp.valueOf(source); + } + } + + /** + * Converts a String to a Decimal for a @Parameterized test + */ + static class StringToDecimal extends TypedArgumentConverter { + protected StringToDecimal() { + super(String.class, Decimal.class); + } + + @Override + protected Decimal convert(String source) throws ArgumentConversionException { + if (source == null) return null; + return Decimal.valueOf(source); + } + } + + /** + * Converts a Hex String to a Byte Array for a @Parameterized test + */ + static class HexStringToByteArray extends TypedArgumentConverter { + + private static final CharsetEncoder ASCII_ENCODER = StandardCharsets.US_ASCII.newEncoder(); + + protected HexStringToByteArray() { + super(String.class, byte[].class); + } + + @Override + protected byte[] convert(String source) throws ArgumentConversionException { + if (source == null) return null; + if (source.trim().isEmpty()) return new byte[0]; + String[] octets = source.split(" "); + byte[] result = new byte[octets.length]; + for (int i = 0; i < octets.length; i++) { + if (octets[i].length() == 1) { + char c = octets[i].charAt(0); + if (!ASCII_ENCODER.canEncode(c)) { + throw new IllegalArgumentException("Cannot convert non-ascii character: " + c); + } + result[i] = (byte) c; + } else { + result[i] = (byte) Integer.parseInt(octets[i], 16); + } + } + return result; + } + } + + /** + * Converts a String of symbol ids to a long[] for a @Parameterized test + */ + static class SymbolIdsToLongArray extends TypedArgumentConverter { + protected SymbolIdsToLongArray() { + super(String.class, long[].class); + } + + @Override + protected long[] convert(String source) throws ArgumentConversionException { + if (source == null) return null; + int size = (int) source.chars().filter(i -> i == '$').count(); + String[] sids = source.split("\\$"); + long[] result = new long[size]; + int i = 0; + for (String sid : sids) { + if (sid.isEmpty()) continue; + result[i] = Long.parseLong(sid.trim()); + i++; + } + return result; + } + } +} diff --git a/test/com/amazon/ion/impl/bin/WriteBufferTest.java b/test/com/amazon/ion/impl/bin/WriteBufferTest.java index 10d49f7b14..1a8e83b80f 100644 --- a/test/com/amazon/ion/impl/bin/WriteBufferTest.java +++ b/test/com/amazon/ion/impl/bin/WriteBufferTest.java @@ -24,6 +24,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.math.BigInteger; import java.util.Arrays; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Assertions; @@ -1411,6 +1412,116 @@ public void testWriteFlexUIntForNegativeNumber() { Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFlexUInt(-1)); } + @ParameterizedTest + @CsvSource({ + " 0, 00000001", + " 1, 00000011", + " 2, 00000101", + " 3, 00000111", + " 4, 00001001", + " 5, 00001011", + " 14, 00011101", + " 63, 01111111", + " 64, 00000010 00000001", + " 729, 01100110 00001011", + " 8191, 11111110 01111111", + " 8192, 00000100 00000000 00000001", + " 1048575, 11111100 11111111 01111111", + " 1048576, 00001000 00000000 00000000 00000001", + " 134217727, 11111000 11111111 11111111 01111111", + " 134217728, 00010000 00000000 00000000 00000000 00000001", + " 17179869184, 00100000 00000000 00000000 00000000 00000000 00000001", + " 2199023255552, 01000000 00000000 00000000 00000000 00000000 00000000 00000001", + " 281474976710655, 11000000 11111111 11111111 11111111 11111111 11111111 01111111", + " 281474976710656, 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001", + " 36028797018963967, 10000000 11111111 11111111 11111111 11111111 11111111 11111111 01111111", + " 36028797018963968, 00000000 00000001 00000000 00000000 00000000 00000000 00000000 00000000 00000001", + // Different one-bits in every byte, making it easy to see if any bytes are out of order + " 72624976668147840, 00000000 00000001 10000001 01000000 00100000 00010000 00001000 00000100 00000010", + " 4611686018427387903, 00000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111 01111111", + " 4611686018427387904, 00000000 00000010 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001", + // Long.MAX_VALUE + " 9223372036854775807, 00000000 11111110 11111111 11111111 11111111 11111111 11111111 11111111 11111111 00000001", + " 9223372036854775808, 00000000 00000010 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000010", + " -1, 11111111", + " -2, 11111101", + " -3, 11111011", + " -14, 11100101", + " -64, 10000001", + " -65, 11111110 11111110", + " -729, 10011110 11110100", + " -8192, 00000010 10000000", + " -8193, 11111100 11111111 11111110", + " -1048576, 00000100 00000000 10000000", + " -1048577, 11111000 11111111 11111111 11111110", + " -134217728, 00001000 00000000 00000000 10000000", + " -134217729, 11110000 11111111 11111111 11111111 11111110", + " -17179869184, 00010000 00000000 00000000 00000000 10000000", + " -17179869185, 11100000 11111111 11111111 11111111 11111111 11111110", + " -281474976710656, 01000000 00000000 00000000 00000000 00000000 00000000 10000000", + " -281474976710657, 10000000 11111111 11111111 11111111 11111111 11111111 11111111 11111110", + " -36028797018963968, 10000000 00000000 00000000 00000000 00000000 00000000 00000000 10000000", + " -36028797018963969, 00000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111110", + // Different zero-bits in every byte, making it easy to see if any bytes are out of order + " -72624976668147841, 00000000 11111111 01111110 10111111 11011111 11101111 11110111 11111011 11111101", + "-4611686018427387904, 00000000 00000001 00000000 00000000 00000000 00000000 00000000 00000000 10000000", + "-4611686018427387905, 00000000 11111110 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111110", + // Long.MIN_VALUE + "-9223372036854775808, 00000000 00000010 00000000 00000000 00000000 00000000 00000000 00000000 00000000 11111110", + "-9223372036854775809, 00000000 11111110 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111101", + + }) + public void testWriteFlexIntForBigInteger(String value, String expectedBits) { + int numBytes = buf.writeFlexInt(new BigInteger(value)); + String actualBits = byteArrayToBitString(bytes()); + Assertions.assertEquals(expectedBits, actualBits); + Assertions.assertEquals((expectedBits.length() + 1)/9, numBytes); + } + + @ParameterizedTest + @CsvSource({ + " 0, 00000001", + " 1, 00000011", + " 2, 00000101", + " 3, 00000111", + " 4, 00001001", + " 5, 00001011", + " 14, 00011101", + " 63, 01111111", + " 64, 10000001", + " 127, 11111111", + " 128, 00000010 00000010", + " 729, 01100110 00001011", + " 16383, 11111110 11111111", + " 16384, 00000100 00000000 00000010", + " 2097151, 11111100 11111111 11111111", + " 2097152, 00001000 00000000 00000000 00000010", + " 268435455, 11111000 11111111 11111111 11111111", + " 268435456, 00010000 00000000 00000000 00000000 00000010", + " 34359738368, 00100000 00000000 00000000 00000000 00000000 00000010", + " 4398046511104, 01000000 00000000 00000000 00000000 00000000 00000000 00000010", + " 562949953421311, 11000000 11111111 11111111 11111111 11111111 11111111 11111111", + " 562949953421312, 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000010", + " 72057594037927935, 10000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111", + " 72057594037927936, 00000000 00000001 00000000 00000000 00000000 00000000 00000000 00000000 00000010", + // Different one-bits in every byte, making it easy to see if any bytes are out of order + " 72624976668147840, 00000000 00000001 10000001 01000000 00100000 00010000 00001000 00000100 00000010", + // Long.MAX_VALUE + "9223372036854775807, 00000000 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111", + "9223372036854775808, 00000000 00000010 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000010", + }) + public void testWriteFlexUIntForBigInteger(String value, String expectedBits) { + int numBytes = buf.writeFlexUInt(new BigInteger(value)); + String actualBits = byteArrayToBitString(bytes()); + Assertions.assertEquals(expectedBits, actualBits); + Assertions.assertEquals((expectedBits.length() + 1)/9, numBytes); + } + + @Test + public void testWriteFlexUIntForNegativeBigInteger() { + Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFlexUInt(BigInteger.ONE.negate())); + } + @ParameterizedTest @CsvSource({ " 0, 00000000", @@ -1508,6 +1619,37 @@ public void testWriteFixedUIntForNegativeNumber() { Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFixedUInt(-1)); } + @ParameterizedTest + @CsvSource({ + " 0, 1, 00000000", + " 0, 2, 00000000 00000000", + " 0, 8, 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000", + " 1, 1, 00000001", + " 1, 2, 00000001 00000000", + " 1, 8, 00000001 00000000 00000000 00000000 00000000 00000000 00000000 00000000", + " 255, 1, 11111111", + " 255, 2, 11111111 00000000", + " 255, 3, 11111111 00000000 00000000", + " -1, 1, 11111111", + " -1, 2, 11111111 11111111", + " -1, 8, 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111", + // Long.MIN_VALUE and Long.MAX_VALUE + " 9223372036854775807, 8, 11111111 11111111 11111111 11111111 11111111 11111111 11111111 01111111", + "-9223372036854775808, 8, 00000000 00000000 00000000 00000000 00000000 00000000 00000000 10000000", + }) + public void testWriteFixedIntOrUInt(long value, int numBytes, String expectedBits) { + int actualNumBytes = buf.writeFixedIntOrUInt(value, numBytes); + String actualBits = byteArrayToBitString(bytes()); + Assertions.assertEquals(expectedBits, actualBits); + Assertions.assertEquals(numBytes, actualNumBytes); + } + + @Test + public void testWriteFixedIntOrUIntThrowsExceptionWhenNumBytesIsOutOfBounds() { + Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFixedIntOrUInt(0, -1)); + Assertions.assertThrows(IllegalArgumentException.class, () -> buf.writeFixedIntOrUInt(0, 9)); + } + /** * Converts a byte array to a string of bits, such as "00110110 10001001". * The purpose of this method is to make it easier to read and write test assertions.