Skip to content

Commit

Permalink
Adds support for reading binary Ion 1.1 decimals.
Browse files Browse the repository at this point in the history
  • Loading branch information
@tgregg
tgregg committed Nov 21, 2023
1 parent 53ee450 commit 4b6aa34
Show file tree
Hide file tree
Showing 4 changed files with 175 additions and 9 deletions.
60 changes: 57 additions & 3 deletions src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,7 @@ private boolean classifyInteger_1_0() {
}

/**
* Reads a FlexUInt into an int. After this method returns, `peekIndex` points to the first byte after the end of
* Reads a FlexUInt into a long. After this method returns, `peekIndex` points to the first byte after the end of
* the FlexUInt.
* @return the value.
*/
Expand All @@ -473,6 +473,25 @@ long readFlexUInt_1_1() {
return result;
}

/**
* Reads a FlexInt into a long. After this method returns, `peekIndex` points to the first byte after the end of
* the FlexInt.
* @return the value.
*/
long readFlexInt_1_1() {
int currentByte = buffer[(int)(peekIndex++)] & SINGLE_BYTE_MASK;
byte length = (byte) (Integer.numberOfTrailingZeros(currentByte) + 1);
long result = currentByte >>> length;
for (byte i = 1; i < length; i++) {
result |= ((long) (buffer[(int) (peekIndex++)] & SINGLE_BYTE_MASK) << (8 * i - length));
}
if (buffer[(int) peekIndex - 1] < 0) {
// Sign extension.
result |= ~(-1 >>> Long.numberOfLeadingZeros(result));
}
return result;
}

private int readVarSym_1_1(Marker marker) {
throw new UnsupportedOperationException();
}
Expand Down Expand Up @@ -536,12 +555,47 @@ private BigInteger readFixedIntOrFixedUIntAsBigInteger_1_1(int length) {
return value;
}

/**
* Reads into a BigDecimal the decimal value that begins at `peekIndex` and ends at `valueMarker.endIndex`.
* @return the value.
*/
private BigDecimal readBigDecimal_1_1() {
throw new UnsupportedOperationException();
int scale = (int) -readFlexInt_1_1();
BigDecimal value;
int length = (int) (valueMarker.endIndex - peekIndex);
if (length <= LONG_SIZE_IN_BYTES) {
// No need to allocate a BigInteger to hold the coefficient.
value = BigDecimal.valueOf(readFixedInt_1_1(), scale);
} else {
// The coefficient may overflow a long, so a BigInteger is required.
value = new BigDecimal(readFixedIntOrFixedUIntAsBigInteger_1_1(length), scale);
}
return value;
}

/**
* Reads into a Decimal the decimal value that begins at `peekIndex` and ends at `valueMarker.endIndex`.
* @return the value.
*/
private Decimal readDecimal_1_1() {
throw new UnsupportedOperationException();
int scale = (int) -readFlexInt_1_1();
BigInteger coefficient;
int length = (int) (valueMarker.endIndex - peekIndex);
if (length > 0) {
// NOTE: there is a BigInteger.valueOf(long unscaledValue, int scale) factory method that avoids allocating
// a BigInteger for coefficients that fit in a long. See its use in readBigDecimal() above. Unfortunately,
// it is not possible to use this for Decimal because the necessary BigDecimal constructor is
// package-private. If a compatible BigDecimal constructor is added in a future JDK revision, a
// corresponding factory method should be added to Decimal to enable this optimization.
coefficient = readFixedIntOrFixedUIntAsBigInteger_1_1(length);
if (coefficient.signum() == 0) {
return Decimal.negativeZero(scale);
}
}
else {
coefficient = BigInteger.ZERO;
}
return Decimal.valueOf(coefficient, scale);
}

/**
Expand Down
4 changes: 1 addition & 3 deletions src/main/java/com/amazon/ion/impl/IonTypeID.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,6 @@ private IonTypeID(byte id, int minorVersion) {
macroId = -1;
variableLength =
(upperNibble == 0xF && lowerNibble >= 0x4) // Variable length, all types.
|| id == POSITIVE_ZERO_DECIMAL
|| id == ANNOTATIONS_MANY_SYMBOL_ADDRESS
|| id == ANNOTATIONS_MANY_FLEX_SYM
|| id == VARIABLE_LENGTH_NOP;
Expand Down Expand Up @@ -354,8 +353,7 @@ private IonTypeID(byte id, int minorVersion) {
length = 9;
break;
}
} else if (type != IonType.DECIMAL || lowerNibble != 0xF) {
// Negative-zero coefficient decimals are always variable-length.
} else {
length = lowerNibble;
}
}
Expand Down
2 changes: 0 additions & 2 deletions src/main/java/com/amazon/ion/impl/bin/OpCodes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ private OpCodes() {}
public static final byte BOOLEAN_FALSE = 0x5F;

public static final byte DECIMAL_ZERO_LENGTH = 0x60;
// 0x61-0x6E are additional lengths of decimals.
public static final byte POSITIVE_ZERO_DECIMAL = 0x6F;

public static final byte TIMESTAMP_YEAR_PRECISION = 0x70;
public static final byte TIMESTAMP_MONTH_PRECISION = 0x71;
Expand Down
118 changes: 117 additions & 1 deletion src/test/java/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package com.amazon.ion.impl;

import com.amazon.ion.BufferConfiguration;
import com.amazon.ion.Decimal;
import com.amazon.ion.IntegerSize;
import com.amazon.ion.IonBufferConfiguration;
import com.amazon.ion.IonDatagram;
Expand All @@ -21,6 +22,7 @@
import com.amazon.ion.SymbolToken;
import com.amazon.ion.SystemSymbols;
import com.amazon.ion.TestUtils;
import com.amazon.ion.Timestamp;
import com.amazon.ion.UnknownSymbolException;
import com.amazon.ion.impl.bin._Private_IonManagedBinaryWriterBuilder;
import com.amazon.ion.impl.bin._Private_IonManagedWriter;
Expand Down Expand Up @@ -51,6 +53,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.zip.GZIPInputStream;

import static com.amazon.ion.BitUtils.bytes;
Expand Down Expand Up @@ -404,6 +407,13 @@ static ExpectationProvider<IonReaderContinuableTopLevelBinary> decimalValue(BigD
));
}

static ExpectationProvider<IonReaderContinuableTopLevelBinary> bigDecimalValue(BigDecimal expectedValue) {
return consumer -> consumer.accept(new Expectation<>(
String.format("bigDecimal(%s)", expectedValue),
reader -> assertEquals(expectedValue, reader.bigDecimalValue())
));
}

static ExpectationProvider<IonReaderContinuableTopLevelBinary> timestampValue(Timestamp expectedValue) {
return consumer -> consumer.accept(new Expectation<>(
String.format("timestamp(%s)", expectedValue),
Expand Down Expand Up @@ -3862,6 +3872,112 @@ public void readDoubleValue(double expectedValue, String inputBytes) throws Exce
assertDoubleCorrectlyParsed(false, expectedValue, inputBytes);
}

/**
* Checks that the reader reads the expected Decimal or BigDecimal from the given input bits.
*/
private void assertDecimalCorrectlyParsed(
boolean constructFromBytes,
BigDecimal expectedValue,
String inputBytes,
Function<BigDecimal, ExpectationProvider<IonReaderContinuableTopLevelBinary>> expectationProviderFunction
) throws Exception {
reader = readerForIon11(hexStringToByteArray(inputBytes), constructFromBytes);
assertSequence(
next(IonType.DECIMAL), expectationProviderFunction.apply(expectedValue),
next(null)
);
closeAndCount();
}

@ParameterizedTest
@CsvSource({
" 0., 60",
" 0e1, 61 03",
" 0e63, 61 7F",
" 0e64, 62 02 01",
" 0e99, 62 8E 01",
" 0.0, 61 FF",
" 0.00, 61 FD",
" 0.000, 61 FB",
" 0e-64, 61 81",
" 0e-99, 62 76 FE",
" -0., 62 01 00",
" -0e1, 62 03 00",
" -0e3, 62 07 00",
" -0e63, 62 7F 00",
" -0e199, 63 1E 03 00",
" -0e-1, 62 FF 00",
" -0e-2, 62 FD 00",
" -0e-3, 62 FB 00",
" -0e-63, 62 83 00",
" -0e-64, 62 81 00",
" -0e-65, 63 FE FE 00",
" -0e-199, 63 E6 FC 00",
" 0.01, 62 FD 01",
" 0.1, 62 FF 01",
" 1, 62 01 01",
" 1e1, 62 03 01",
" 1e2, 62 05 01",
" 1e63, 62 7F 01",
" 1e64, 63 02 01 01",
" 1e65536, 64 04 00 08 01",
" 2, 62 01 02",
" 7, 62 01 07",
" 14, 62 01 0E",
" 14, 63 02 00 0E", // overpadded exponent
" 14, 64 01 0E 00 00", // Overpadded coefficient
" 14, 65 02 00 0E 00 00", // Overpadded coefficient and exponent
" 1.0, 62 FF 0A",
" 1.00, 62 FD 64",
" 1.27, 62 FD 7F",
" 1.28, 63 FD 80 00",
" 3.142, 63 FB 46 0C",
" 3.14159, 64 F7 2F CB 04",
" 3.1415927, 65 F3 77 5E DF 01",
" 3.141592653, 66 EF 4D E6 40 BB 00",
" 3.141592653590, 67 E9 16 9F 83 75 DB 02",
" 3.14159265358979323, 69 DF FB A0 9E F6 2F 1E 5C 04",
" 3.1415926535897932384626, 6B D5 72 49 64 CC AF EF 8F 0F A7 06",
" 3.141592653589793238462643383, 6D CB B7 3C 92 86 40 9F 1B 01 1F AA 26 0A",
" 3.14159265358979323846264338327950, 6F C1 8E 29 E5 E3 56 D5 DF C5 10 8F 55 3F 7D 0F",
"3.141592653589793238462643383279503, F6 21 BF 8F 9F F3 E6 64 55 BE BA A7 96 57 79 E4 9A 00",
})
public void readDecimalValue(@ConvertWith(TestUtils.StringToDecimal.class) Decimal expectedValue, String inputBytes) throws Exception {
assertDecimalCorrectlyParsed(true, expectedValue, inputBytes, IonReaderContinuableTopLevelBinaryTest::decimalValue);
assertDecimalCorrectlyParsed(false, expectedValue, inputBytes, IonReaderContinuableTopLevelBinaryTest::decimalValue);
assertDecimalCorrectlyParsed(true, expectedValue, inputBytes, IonReaderContinuableTopLevelBinaryTest::bigDecimalValue);
assertDecimalCorrectlyParsed(false, expectedValue, inputBytes, IonReaderContinuableTopLevelBinaryTest::bigDecimalValue);
}

@ParameterizedTest
@CsvSource({
" 0., F6 01",
" 0e99, F6 05 8E 01",
" 0.0, F6 03 FF",
" 0.00, F6 03 FD",
" 0e-99, F6 05 76 FE",
" -0., F6 05 01 00",
" -0e199, F6 07 1E 03 00",
" -0e-1, F6 05 FF 00",
" -0e-65, F6 07 FE FE 00",
" 0.01, F6 05 FD 01",
" 1, F6 05 01 01",
" 1e65536, F6 09 04 00 08 01",
" 1.0, F6 05 FF 0A",
" 1.28, F6 07 FD 80 00",
" 3.141592653590, F6 0F E9 16 9F 83 75 DB 02",
" 3.14159265358979323, F6 13 DF FB A0 9E F6 2F 1E 5C 04",
" 3.1415926535897932384626, F6 17 D5 72 49 64 CC AF EF 8F 0F A7 06",
" 3.141592653589793238462643383, F6 1B CB B7 3C 92 86 40 9F 1B 01 1F AA 26 0A",
" 3.14159265358979323846264338327950, F6 1F C1 8E 29 E5 E3 56 D5 DF C5 10 8F 55 3F 7D 0F",
})
public void readDecimalValueFromVariableLengthEncoding(@ConvertWith(TestUtils.StringToDecimal.class) Decimal expectedValue, String inputBytes) throws Exception {
assertDecimalCorrectlyParsed(true, expectedValue, inputBytes, IonReaderContinuableTopLevelBinaryTest::decimalValue);
assertDecimalCorrectlyParsed(false, expectedValue, inputBytes, IonReaderContinuableTopLevelBinaryTest::decimalValue);
assertDecimalCorrectlyParsed(true, expectedValue, inputBytes, IonReaderContinuableTopLevelBinaryTest::bigDecimalValue);
assertDecimalCorrectlyParsed(false, expectedValue, inputBytes, IonReaderContinuableTopLevelBinaryTest::bigDecimalValue);
}

/**
* Checks that the reader reads the expected timestamp value from the given input bits.
*/
Expand Down Expand Up @@ -4028,7 +4144,7 @@ public void readTimestampValueWithKnownOffsetShortForm(@ConvertWith(StringToTime
"2048-01-01T01:01+00:02, 11110111 00001101 00000000 01001000 10000100 00010000 10001000 00010110",
"2048-01-01T01:01+23:59, 11110111 00001101 00000000 01001000 10000100 00010000 11111100 00101100",
})
public void testWriteTimestampValueLongForm(@ConvertWith(StringToTimestamp.class) Timestamp expectedValue, String inputBits) throws Exception {
public void readTimestampValueLongForm(@ConvertWith(StringToTimestamp.class) Timestamp expectedValue, String inputBits) throws Exception {
assertIonTimestampCorrectlyParsed(true, expectedValue, inputBits);
assertIonTimestampCorrectlyParsed(false, expectedValue, inputBits);
}
Expand Down

0 comments on commit 4b6aa34

Please sign in to comment.