Skip to content

Commit

Permalink
Adds support for all scalar types to IonRawBinaryWriter_1_1 (#667)
Browse files Browse the repository at this point in the history
  • Loading branch information
@popematt
popematt authored Dec 11, 2023
1 parent b92edd7 commit f74e7e0
Show file tree
Hide file tree
Showing 6 changed files with 280 additions and 106 deletions.
4 changes: 0 additions & 4 deletions src/main/java/com/amazon/ion/impl/IonWriter_1_1.kt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.amazon.ion.impl

import com.amazon.ion.Decimal
import com.amazon.ion.IonType
import com.amazon.ion.Timestamp
import java.math.BigDecimal
Expand Down Expand Up @@ -149,16 +148,13 @@ interface IonWriter_1_1 {

fun writeBool(value: Boolean)

fun writeInt(value: Byte)
fun writeInt(value: Int)
fun writeInt(value: Long)
fun writeInt(value: BigInteger)

fun writeFloat(value: Float)
fun writeFloat(value: Double)

fun writeDecimal(value: BigDecimal)
fun writeDecimal(value: Decimal)

/**
* TODO: Consider adding a function for writing a timestamp that doesn't require creating a [Timestamp] instance, so
Expand Down
45 changes: 17 additions & 28 deletions src/main/java/com/amazon/ion/impl/bin/IonEncoder_1_1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import com.amazon.ion.IonType;
import com.amazon.ion.Timestamp;
import com.amazon.ion.impl.bin.utf8.Utf8StringEncoder;
import com.amazon.ion.impl.bin.utf8.Utf8StringEncoderPool;

import java.math.BigDecimal;
import java.math.BigInteger;
Expand Down Expand Up @@ -129,7 +128,7 @@ public static int writeIntValue(WriteBuffer buffer, final BigInteger value) {
* Writes a float to the given WriteBuffer using the Ion 1.1 encoding for Ion Floats.
* @return the number of bytes written
*/
public static int writeFloat(WriteBuffer buffer, final float value) {
public static int writeFloatValue(WriteBuffer buffer, final float value) {
// TODO: Optimization to write a 16 bit float for non-finite and possibly other values
if (value == 0.0) {
buffer.writeByte(OpCodes.FLOAT_ZERO_LENGTH);
Expand All @@ -145,7 +144,7 @@ public static int writeFloat(WriteBuffer buffer, final float value) {
* Writes a double to the given WriteBuffer using the Ion 1.1 encoding for Ion Floats.
* @return the number of bytes written
*/
public static int writeFloat(WriteBuffer buffer, final double value) {
public static int writeFloatValue(WriteBuffer buffer, final double value) {
// TODO: Optimization to write a 16 bit float for non-finite and possibly other values
if (value == 0.0) {
buffer.writeByte(OpCodes.FLOAT_ZERO_LENGTH);
Expand Down Expand Up @@ -199,10 +198,8 @@ public static int writeDecimalValue(WriteBuffer buffer, final BigDecimal value)
if (numCoefficientBytes > 0) {
if (coefficientBytes != null) {
buffer.writeFixedIntOrUInt(coefficientBytes);
} else if (numCoefficientBytes == 1){
buffer.writeByte((byte) 0);
} else {
throw new IllegalStateException("Unreachable! coefficientBytes should not be null when numCoefficientBytes > 1");
buffer.writeByte((byte) 0);
}
}

Expand Down Expand Up @@ -446,29 +443,25 @@ static int writeLongFormTimestampValue(WriteBuffer buffer, Timestamp value) {
* Writes a String to the given WriteBuffer using the Ion 1.1 encoding for Ion Strings.
* @return the number of bytes written
*/
public static int writeStringValue(WriteBuffer buffer, String value) {
public static int writeStringValue(WriteBuffer buffer, Utf8StringEncoder.Result value) {
return writeInlineText(buffer, value, IonType.STRING, OpCodes.STRING_ZERO_LENGTH, OpCodes.VARIABLE_LENGTH_STRING);
}

/**
* Writes an inline Symbol to the given WriteBuffer using the Ion 1.1 encoding for Ion Symbols.
* @return the number of bytes written
*/
public static int writeSymbolValue(WriteBuffer buffer, String value) {
public static int writeSymbolValue(WriteBuffer buffer, Utf8StringEncoder.Result value) {
return writeInlineText(buffer, value, IonType.SYMBOL, OpCodes.INLINE_SYMBOL_ZERO_LENGTH, OpCodes.VARIABLE_LENGTH_INLINE_SYMBOL);
}

private static int writeInlineText(WriteBuffer buffer, String value, IonType type, byte zeroLengthOpCode, byte variableLengthOpCode) {
private static int writeInlineText(WriteBuffer buffer, Utf8StringEncoder.Result value, IonType type, byte zeroLengthOpCode, byte variableLengthOpCode) {
if (value == null) {
return writeNullValue(buffer, type);
}

// TODO: When merging into the Ion 1.1 raw writer, keep a single instance of the Utf8StringEncoder
// instead of fetching one on every call.
Utf8StringEncoder.Result encoderResult = Utf8StringEncoderPool.getInstance().getOrCreate().encode(value);

byte[] utf8Buffer = encoderResult.getBuffer();
int numValueBytes = encoderResult.getEncodedLength();
byte[] utf8Buffer = value.getBuffer();
int numValueBytes = value.getEncodedLength();
int numLengthBytes = 0;

if (numValueBytes <= 0xF) {
Expand All @@ -484,10 +477,8 @@ private static int writeInlineText(WriteBuffer buffer, String value, IonType typ
/**
* Writes an interned Symbol's address to the given WriteBuffer using the Ion 1.1 encoding for Ion Symbols.
* @return the number of bytes written
*
* TODO: Do we need to support Symbol Addresses greater than Long.MAX_VALUE?
*/
public static int writeSymbolValue(WriteBuffer buffer, long value) {
public static int writeSymbolValue(WriteBuffer buffer, int value) {
if (value < 0) {
throw new IllegalArgumentException("Symbol Address cannot be negative; was: " + value);
} else if (value < FIRST_2_BYTE_SYMBOL_ADDRESS) {
Expand All @@ -509,34 +500,32 @@ public static int writeSymbolValue(WriteBuffer buffer, long value) {
* Writes a Blob to the given WriteBuffer using the Ion 1.1 encoding for Ion Blobs.
* @return the number of bytes written
*/
public static int writeBlobValue(WriteBuffer buffer, byte[] value) {
public static int writeBlobValue(WriteBuffer buffer, byte[] value, int start, int length) {
if (value == null) {
return writeNullValue(buffer, IonType.BLOB);
}

buffer.writeByte(OpCodes.VARIABLE_LENGTH_BLOB);
int numLengthBytes = buffer.writeFlexUInt(value.length);
buffer.writeBytes(value);
return 1 + numLengthBytes + value.length;
int numLengthBytes = buffer.writeFlexUInt(length);
buffer.writeBytes(value, start, length);
return 1 + numLengthBytes + length;
}

/**
* Writes a Clob to the given WriteBuffer using the Ion 1.1 encoding for Ion Clobs.
* @return the number of bytes written
*/
public static int writeClobValue(WriteBuffer buffer, byte[] value) {
public static int writeClobValue(WriteBuffer buffer, byte[] value, int start, int length) {
if (value == null) {
return writeNullValue(buffer, IonType.CLOB);
}

buffer.writeByte(OpCodes.VARIABLE_LENGTH_CLOB);
int numLengthBytes = buffer.writeFlexUInt(value.length);
buffer.writeBytes(value);
return 1 + numLengthBytes + value.length;
int numLengthBytes = buffer.writeFlexUInt(length);
buffer.writeBytes(value, start, length);
return 1 + numLengthBytes + length;
}

// TODO: Implement FlexSym Annotations

/**
* Writes annotations using the given symbol addresses.
*/
Expand Down
85 changes: 25 additions & 60 deletions src/main/java/com/amazon/ion/impl/bin/IonRawBinaryWriter_1_1.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ package com.amazon.ion.impl.bin

import com.amazon.ion.*
import com.amazon.ion.impl.*
import com.amazon.ion.impl.bin.IonEncoder_1_1.*
import com.amazon.ion.impl.bin.IonRawBinaryWriter_1_1.ContainerType.*
import com.amazon.ion.impl.bin.IonRawBinaryWriter_1_1.ContainerType.List
import com.amazon.ion.impl.bin.Ion_1_1_Constants.*
import com.amazon.ion.impl.bin.utf8.*
import com.amazon.ion.util.*
Expand Down Expand Up @@ -299,23 +301,20 @@ class IonRawBinaryWriter_1_1 internal constructor(
}

/**
* Helper function for finishing scalar values. Similar concerns for containers are handled in [stepOut].
* Helper function for writing scalar values that builds on [openValue] and also includes updating
* the length of the current container.
*
* @param valueWriterExpression should be a function that writes the scalar value to the buffer, and
* returns the number of bytes that were written.
*/
private inline fun closeScalar(valueWriterExpression: () -> Int) {
private inline fun writeScalar(valueWriterExpression: () -> Int) = openValue {
val numBytesWritten = valueWriterExpression()
currentContainer.length += numBytesWritten
}

/**
* Helper function for writing scalar values that composes both [openValue] and [closeScalar].
*/
private inline fun writeScalar(valueWriterExpression: () -> Int) {
openValue { closeScalar(valueWriterExpression) }
}

override fun writeFieldName(sid: Int) {
confirm(currentContainer.type == Struct) { "Can only write a field name inside of a struct." }
if (sid == 0 && !currentContainer.usesFlexSym) switchToFlexSym()
if (sid == 0 && !currentContainer.usesFlexSym) switchCurrentStructToFlexSym()

currentContainer.length += if (currentContainer.usesFlexSym) {
buffer.writeFlexSym(sid)
Expand All @@ -327,13 +326,13 @@ class IonRawBinaryWriter_1_1 internal constructor(

override fun writeFieldName(text: CharSequence) {
confirm(currentContainer.type == Struct) { "Can only write a field name inside of a struct." }
if (!currentContainer.usesFlexSym) switchToFlexSym()
if (!currentContainer.usesFlexSym) switchCurrentStructToFlexSym()

currentContainer.length += buffer.writeFlexSym(utf8StringEncoder.encode(text.toString()))
hasFieldName = true
}

private fun switchToFlexSym() {
private fun switchCurrentStructToFlexSym() {
// To switch, we need to insert the sid-to-flexsym switch marker, unless...
// if no fields have been written yet, we can just switch the op code of the struct.
if (currentContainer.length == 0L) {
Expand All @@ -345,71 +344,37 @@ class IonRawBinaryWriter_1_1 internal constructor(
currentContainer.usesFlexSym = true
}

override fun writeNull() = writeScalar { IonEncoder_1_1.writeNullValue(buffer, IonType.NULL) }
override fun writeNull() = writeScalar { writeNullValue(buffer, IonType.NULL) }

override fun writeNull(type: IonType) = writeScalar { IonEncoder_1_1.writeNullValue(buffer, type) }
override fun writeNull(type: IonType) = writeScalar { writeNullValue(buffer, type) }

override fun writeBool(value: Boolean) = writeScalar { IonEncoder_1_1.writeBoolValue(buffer, value) }
override fun writeBool(value: Boolean) = writeScalar { writeBoolValue(buffer, value) }

override fun writeInt(value: Byte) {
TODO("Not yet implemented")
}
override fun writeInt(value: Long) = writeScalar { writeIntValue(buffer, value) }

override fun writeInt(value: Int) {
TODO("Not yet implemented")
}
override fun writeInt(value: BigInteger) = writeScalar { writeIntValue(buffer, value) }

override fun writeInt(value: Long) {
TODO("Not yet implemented")
}
override fun writeFloat(value: Float) = writeScalar { writeFloatValue(buffer, value) }

override fun writeInt(value: BigInteger) {
TODO("Not yet implemented")
}
override fun writeFloat(value: Double) = writeScalar { writeFloatValue(buffer, value) }

override fun writeFloat(value: Float) {
TODO("Not yet implemented")
}
override fun writeDecimal(value: BigDecimal) = writeScalar { writeDecimalValue(buffer, value) }

override fun writeFloat(value: Double) {
TODO("Not yet implemented")
}

override fun writeDecimal(value: BigDecimal) {
TODO("Not yet implemented")
}

override fun writeDecimal(value: Decimal) {
TODO("Not yet implemented")
}

override fun writeTimestamp(value: Timestamp) {
TODO("Not yet implemented")
}
override fun writeTimestamp(value: Timestamp) = writeScalar { writeTimestampValue(buffer, value) }

override fun writeTimestamp(value: Instant) {
TODO("Not yet implemented")
}

override fun writeSymbol(id: Int) {
TODO("Not yet implemented")
}
override fun writeSymbol(id: Int) = writeScalar { writeSymbolValue(buffer, id) }

override fun writeSymbol(text: CharSequence) {
TODO("Not yet implemented")
}
override fun writeSymbol(text: CharSequence) = writeScalar { writeSymbolValue(buffer, utf8StringEncoder.encode(text.toString())) }

override fun writeString(value: CharSequence) {
TODO("Not yet implemented")
}
override fun writeString(value: CharSequence) = writeScalar { writeStringValue(buffer, utf8StringEncoder.encode(value.toString())) }

override fun writeBlob(value: ByteArray, start: Int, length: Int) {
TODO("Not yet implemented")
}
override fun writeBlob(value: ByteArray, start: Int, length: Int) = writeScalar { writeBlobValue(buffer, value, start, length) }

override fun writeClob(value: ByteArray, start: Int, length: Int) {
TODO("Not yet implemented")
}
override fun writeClob(value: ByteArray, start: Int, length: Int) = writeScalar { writeClobValue(buffer, value, start, length) }

override fun stepInList(delimited: Boolean) {
openValue {
Expand Down
2 changes: 2 additions & 0 deletions src/test/java/com/amazon/ion/impl/bin/FlexIntTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ class FlexIntTest {
" 1048576, 00001000 00000000 00000000 00000001",
" 134217727, 11111000 11111111 11111111 01111111",
" 134217728, 00010000 00000000 00000000 00000000 00000001",
" ${Int.MAX_VALUE}, 11110000 11111111 11111111 11111111 00001111",
" 17179869184, 00100000 00000000 00000000 00000000 00000000 00000001",
" 2199023255552, 01000000 00000000 00000000 00000000 00000000 00000000 00000001",
" 281474976710655, 11000000 11111111 11111111 11111111 11111111 11111111 01111111",
Expand Down Expand Up @@ -94,6 +95,7 @@ class FlexIntTest {
" 2097152, 00001000 00000000 00000000 00000010",
" 268435455, 11111000 11111111 11111111 11111111",
" 268435456, 00010000 00000000 00000000 00000000 00000010",
" ${Int.MAX_VALUE}, 11110000 11111111 11111111 11111111 00001111",
" 34359738368, 00100000 00000000 00000000 00000000 00000000 00000010",
" 4398046511104, 01000000 00000000 00000000 00000000 00000000 00000000 00000010",
" 562949953421311, 11000000 11111111 11111111 11111111 11111111 11111111 11111111",
Expand Down
24 changes: 14 additions & 10 deletions src/test/java/com/amazon/ion/impl/bin/IonEncoder_1_1Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import com.amazon.ion.Decimal;
import com.amazon.ion.IonType;
import com.amazon.ion.Timestamp;
import com.amazon.ion.impl.bin.utf8.Utf8StringEncoder;
import com.amazon.ion.impl.bin.utf8.Utf8StringEncoderPool;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -207,7 +209,7 @@ public void testWriteIntegerValueForNullBigInteger() {
" -Infinity, 5C FF 80 00 00",
})
public void testWriteFloatValue(float value, String expectedBytes) {
assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeFloat);
assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeFloatValue);
}

@ParameterizedTest
Expand All @@ -234,7 +236,7 @@ public void testWriteFloatValue(float value, String expectedBytes) {
" -Infinity, 5C FF 80 00 00",
})
public void testWriteFloatValueForDouble(double value, String expectedBytes) {
assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeFloat);
assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeFloatValue);
}

@ParameterizedTest
Expand Down Expand Up @@ -488,7 +490,8 @@ public void testWriteTimestampValueForNullTimestamp() {
"'variable length encoding', F8 31 76 61 72 69 61 62 6C 65 20 6C 65 6E 67 74 68 20 65 6E 63 6F 64 69 6E 67",
})
public void testWriteStringValue(String value, String expectedBytes) {
assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeStringValue);
Utf8StringEncoder.Result result = Utf8StringEncoderPool.getInstance().getOrCreate().encode(value);
assertWritingValue(expectedBytes, result, IonEncoder_1_1::writeStringValue);
}

@Test
Expand All @@ -509,7 +512,8 @@ public void testWriteStringValueForNull() {
"'variable length encoding', F9 31 76 61 72 69 61 62 6C 65 20 6C 65 6E 67 74 68 20 65 6E 63 6F 64 69 6E 67",
})
public void testWriteSymbolValue(String value, String expectedBytes) {
assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeSymbolValue);
Utf8StringEncoder.Result result = Utf8StringEncoderPool.getInstance().getOrCreate().encode(value);
assertWritingValue(expectedBytes, result, IonEncoder_1_1::writeSymbolValue);
}

@ParameterizedTest
Expand All @@ -527,9 +531,9 @@ public void testWriteSymbolValue(String value, String expectedBytes) {
"65793, E3 03",
"65919, E3 FF",
"65920, E3 02 02",
"9223372036854775807, E3 00 FF FD FD FF FF FF FF FF"
"2147483647 , E3 F0 DF DF FF 0F"
})
public void testWriteSymbolValue(long value, String expectedBytes) {
public void testWriteSymbolValue(int value, String expectedBytes) {
assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeSymbolValue);
}

Expand All @@ -548,12 +552,12 @@ public void testWriteSymbolValueForNull() {
"FE 31 49 20 61 70 70 6C 61 75 64 20 79 6F 75 72 20 63 75 72 69 6F 73 69 74 79"
})
public void testWriteBlobValue(@ConvertWith(HexStringToByteArray.class) byte[] value, String expectedBytes) {
assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeBlobValue);
assertWritingValue(expectedBytes, value, (buffer, bytes) -> IonEncoder_1_1.writeBlobValue(buffer, bytes, 0, bytes.length));
}

@Test
public void testWriteBlobValueForNull() {
int numBytes = IonEncoder_1_1.writeBlobValue(buf, null);
int numBytes = IonEncoder_1_1.writeBlobValue(buf, null, 0, 0);
Assertions.assertEquals("EB 07", byteArrayToHex(bytes()));
Assertions.assertEquals(2, numBytes);
}
Expand All @@ -566,12 +570,12 @@ public void testWriteBlobValueForNull() {
"FF 31 49 20 61 70 70 6C 61 75 64 20 79 6F 75 72 20 63 75 72 69 6F 73 69 74 79"
})
public void testWriteClobValue(@ConvertWith(HexStringToByteArray.class) byte[] value, String expectedBytes) {
assertWritingValue(expectedBytes, value, IonEncoder_1_1::writeClobValue);
assertWritingValue(expectedBytes, value, (buffer, bytes) -> IonEncoder_1_1.writeClobValue(buffer, bytes, 0, bytes.length));
}

@Test
public void testWriteClobValueForNull() {
int numBytes = IonEncoder_1_1.writeClobValue(buf, null);
int numBytes = IonEncoder_1_1.writeClobValue(buf, null, 0, 0);
Assertions.assertEquals("EB 08", byteArrayToHex(bytes()));
Assertions.assertEquals(2, numBytes);
}
Expand Down
Loading

0 comments on commit f74e7e0

Please sign in to comment.