Adds support for writing Ion 1.1 sexps and structs (#665)

src/main/java/com/amazon/ion/impl/IonTypeID.java

@@ -236,7 +236,7 @@ private IonTypeID(byte id, int minorVersion) {
                        (upperNibble == 0xD && lowerNibble >= 0x2)
                     || id == DELIMITED_STRUCT
                     || id == VARIABLE_LENGTH_INLINE_SYMBOL
-                    || id == VARIABLE_LENGTH_STRUCT_WITH_FLEXSYMS
+                    || id == VARIABLE_LENGTH_STRUCT_WITH_FLEX_SYMS
                     || id == ANNOTATIONS_1_FLEX_SYM
                     || id == ANNOTATIONS_2_FLEX_SYM
                     || id == ANNOTATIONS_MANY_FLEX_SYM
@@ -292,7 +292,7 @@ private IonTypeID(byte id, int minorVersion) {
                         if (id == DELIMITED_END_MARKER) {
                             type = null;
                             length = 0;
-                        } else if (id == DELIMITED_STRUCT || id == VARIABLE_LENGTH_STRUCT_WITH_SIDS || id == VARIABLE_LENGTH_STRUCT_WITH_FLEXSYMS) {
+                        } else if (id == DELIMITED_STRUCT || id == VARIABLE_LENGTH_STRUCT_WITH_SIDS || id == VARIABLE_LENGTH_STRUCT_WITH_FLEX_SYMS) {
                             type = IonType.STRUCT;
                         } else if (id == VARIABLE_LENGTH_INTEGER) {
                             type = IonType.INT;

src/main/java/com/amazon/ion/impl/IonWriter_1_1.kt

@@ -114,11 +114,10 @@ interface IonWriter_1_1 {
     /**
      * Steps into a Struct.
      *
-     * The [delimited] and [useFlexSym] parameters are suggestions. Implementations may ignore these parameters if they
-     * are not relevant for that particular implementation. All implementations must document their specific behavior
-     * for this method.
+     * The [delimited] parameter is a suggestion. Implementations may ignore it if it is not relevant for that
+     * particular implementation. All implementations must document their specific behavior for this method.
      */
-    fun stepInStruct(delimited: Boolean, useFlexSym: Boolean)
+    fun stepInStruct(delimited: Boolean)
 
     /**
      * Steps into a stream.

src/main/java/com/amazon/ion/impl/bin/FlexInt.kt

@@ -13,6 +13,11 @@ import java.math.BigInteger
  */
 object FlexInt {
 
+    /**
+     * A byte representing zero, encoded as a FlexInt (or FlexUInt).
+     */
+    const val ZERO: Byte = 0x01
+
     /** Determine the length of FlexUInt for the provided value.  */
     @JvmStatic
     fun flexUIntLength(value: Long): Int {

src/main/java/com/amazon/ion/impl/bin/IonRawBinaryWriter_1_1.kt

@@ -5,6 +5,7 @@ package com.amazon.ion.impl.bin
 import com.amazon.ion.*
 import com.amazon.ion.impl.*
 import com.amazon.ion.impl.bin.IonRawBinaryWriter_1_1.ContainerType.*
+import com.amazon.ion.impl.bin.Ion_1_1_Constants.*
 import com.amazon.ion.impl.bin.utf8.*
 import com.amazon.ion.util.*
 import java.io.ByteArrayOutputStream
@@ -43,15 +44,20 @@ class IonRawBinaryWriter_1_1 internal constructor(
     private data class ContainerInfo(
         var type: ContainerType? = null,
         var isDelimited: Boolean = false,
+        var usesFlexSym: Boolean = false,
         var position: Long = -1,
         var length: Long = 0,
         // TODO: Test if performance is better with an Object Reference or an index into the PatchPoint queue.
         var patchPoint: PatchPoint? = null,
     ) {
-        fun clear() {
-            type = null
-            isDelimited = false
-            position = -1
+        /**
+         * Clears this [ContainerInfo] of old data and initializes it with the given new data.
+         */
+        fun reset(type: ContainerType? = null, position: Long = -1, isDelimited: Boolean = false) {
+            this.type = type
+            this.isDelimited = isDelimited
+            this.position = position
+            usesFlexSym = false
             length = 0
             patchPoint = null
         }
@@ -88,7 +94,7 @@ class IonRawBinaryWriter_1_1 internal constructor(
     private val patchPoints = _Private_RecyclingQueue(512) { PatchPoint() }
     private val containerStack = _Private_RecyclingStack(8) { ContainerInfo() }
 
-    private var currentContainer: ContainerInfo = containerStack.push { it.type = Top }
+    private var currentContainer: ContainerInfo = containerStack.push { it.reset(Top, 0L) }
 
     override fun finish() {
         if (closed) return
@@ -210,6 +216,8 @@ class IonRawBinaryWriter_1_1 internal constructor(
      */
     private inline fun openValue(valueWriterExpression: () -> Unit) {
 
+        if (isInStruct()) confirm(hasFieldName) { "Values in a struct must have a field name." }
+
         // Start at 1, assuming there's an annotations OpCode byte.
         // We'll clear this if there are no annotations.
         var annotationsTotalLength = 1L
@@ -265,11 +273,7 @@ class IonRawBinaryWriter_1_1 internal constructor(
      * Writes 3 or more annotations for SIDs or FlexSyms
      */
     private fun writeManyAnnotations(): Long {
-        currentContainer = containerStack.push {
-            it.clear()
-            it.type = Annotations
-            it.position = buffer.position()
-        }
+        currentContainer = containerStack.push { it.reset(Annotations, position = buffer.position()) }
         if (annotationFlexSymFlag == FLEX_SYMS_REQUIRED) {
             buffer.writeByte(OpCodes.ANNOTATIONS_MANY_FLEX_SYM)
             buffer.reserve(ANNOTATIONS_LENGTH_PREFIX_ALLOCATION_SIZE)
@@ -310,11 +314,35 @@ class IonRawBinaryWriter_1_1 internal constructor(
     }
 
     override fun writeFieldName(sid: Int) {
-        TODO("Not implemented yet.")
+        confirm(currentContainer.type == Struct) { "Can only write a field name inside of a struct." }
+        if (sid == 0 && !currentContainer.usesFlexSym) switchToFlexSym()
+
+        currentContainer.length += if (currentContainer.usesFlexSym) {
+            buffer.writeFlexSym(sid)
+        } else {
+            buffer.writeFlexUInt(sid)
+        }
+        hasFieldName = true
     }
 
     override fun writeFieldName(text: CharSequence) {
-        TODO("Not implemented yet.")
+        confirm(currentContainer.type == Struct) { "Can only write a field name inside of a struct." }
+        if (!currentContainer.usesFlexSym) switchToFlexSym()
+
+        currentContainer.length += buffer.writeFlexSym(utf8StringEncoder.encode(text.toString()))
+        hasFieldName = true
+    }
+
+    private fun switchToFlexSym() {
+        // To switch, we need to insert the sid-to-flexsym switch marker, unless...
+        // if no fields have been written yet, we can just switch the op code of the struct.
+        if (currentContainer.length == 0L) {
+            buffer.writeUInt8At(currentContainer.position, OpCodes.VARIABLE_LENGTH_STRUCT_WITH_FLEX_SYMS.toLong())
+        } else {
+            buffer.writeByte(SID_TO_FLEX_SYM_SWITCH_MARKER)
+            currentContainer.length += 1
+        }
+        currentContainer.usesFlexSym = true
     }
 
     override fun writeNull() = writeScalar { IonEncoder_1_1.writeNullValue(buffer, IonType.NULL) }
@@ -385,12 +413,7 @@ class IonRawBinaryWriter_1_1 internal constructor(
 
     override fun stepInList(delimited: Boolean) {
         openValue {
-            currentContainer = containerStack.push {
-                it.clear()
-                it.type = List
-                it.position = buffer.position()
-                it.isDelimited = delimited
-            }
+            currentContainer = containerStack.push { it.reset(List, buffer.position(), delimited) }
             if (delimited) {
                 buffer.writeByte(OpCodes.DELIMITED_LIST)
             } else {
@@ -401,11 +424,28 @@ class IonRawBinaryWriter_1_1 internal constructor(
     }
 
     override fun stepInSExp(delimited: Boolean) {
-        TODO("Not yet implemented")
+        openValue {
+            currentContainer = containerStack.push { it.reset(SExp, buffer.position(), delimited) }
+            if (delimited) {
+                buffer.writeByte(OpCodes.DELIMITED_SEXP)
+            } else {
+                buffer.writeByte(OpCodes.VARIABLE_LENGTH_SEXP)
+                buffer.reserve(lengthPrefixPreallocation)
+            }
+        }
     }
 
-    override fun stepInStruct(delimited: Boolean, useFlexSym: Boolean) {
-        TODO("Not yet implemented")
+    override fun stepInStruct(delimited: Boolean) {
+        openValue {
+            currentContainer = containerStack.push { it.reset(Struct, buffer.position(), delimited) }
+            if (delimited) {
+                buffer.writeByte(OpCodes.DELIMITED_STRUCT)
+                currentContainer.usesFlexSym = true
+            } else {
+                buffer.writeByte(OpCodes.VARIABLE_LENGTH_STRUCT_WITH_SIDS)
+                buffer.reserve(lengthPrefixPreallocation)
+            }
+        }
     }
 
     override fun stepInEExp(name: CharSequence) {
@@ -430,24 +470,32 @@ class IonRawBinaryWriter_1_1 internal constructor(
         // Write closing delimiter if we're in a delimited container.
         // Update length prefix if we're in a prefixed container.
         if (currentContainer.isDelimited) {
+            if (isInStruct()) {
+                // Need a 0 FlexInt before the end delimiter
+                buffer.writeByte(FlexInt.ZERO)
+                thisContainerTotalLength += 1
+            }
             thisContainerTotalLength += 1 // For the end marker
             buffer.writeByte(OpCodes.DELIMITED_END_MARKER)
         } else {
             // currentContainer.type is non-null for any initialized ContainerInfo
             when (currentContainer.type.assumeNotNull()) {
-                List -> {
-                    // TODO: Possibly extract this so it can be reused for the other length-prefixed container types
+                List, SExp, Struct -> {
                     val contentLength = currentContainer.length
-                    if (contentLength <= 0xF) {
+                    if (contentLength <= 0xF && !currentContainer.usesFlexSym) {
                         // Clean up any unused space that was pre-allocated.
                         buffer.shiftBytesLeft(currentContainer.length.toInt(), lengthPrefixPreallocation)
-                        buffer.writeUInt8At(currentContainer.position, OpCodes.LIST_ZERO_LENGTH + contentLength)
+                        val zeroLengthOpCode = when (currentContainer.type) {
+                            List -> OpCodes.LIST_ZERO_LENGTH
+                            SExp -> OpCodes.SEXP_ZERO_LENGTH
+                            Struct -> OpCodes.STRUCT_SID_ZERO_LENGTH
+                            else -> TODO("Unreachable")
+                        }
+                        buffer.writeUInt8At(currentContainer.position, zeroLengthOpCode + contentLength)
                     } else {
                         thisContainerTotalLength += writeCurrentContainerLength(lengthPrefixPreallocation)
                     }
                 }
-                SExp -> TODO()
-                Struct -> TODO()
                 Macro -> TODO()
                 Stream -> TODO()
                 Annotations -> TODO("Unreachable.")

src/main/java/com/amazon/ion/impl/bin/Ion_1_1_Constants.java

@@ -13,6 +13,9 @@ private Ion_1_1_Constants() {}
 
     static final int FIRST_2_BYTE_SYMBOL_ADDRESS = 256;
     static final int FIRST_MANY_BYTE_SYMBOL_ADDRESS = 65792;
+
+    static final byte SID_TO_FLEX_SYM_SWITCH_MARKER = 0;
+
     public static final int MAX_NANOSECONDS = 999999999;
     public static final int NANOSECOND_SCALE = 9;
     public static final int MAX_MICROSECONDS = 999999;

src/main/java/com/amazon/ion/impl/bin/OpCodes.java

@@ -34,10 +34,23 @@ private OpCodes() {}
     // 0x7D-0x7F Reserved
 
     public static final byte STRING_ZERO_LENGTH = (byte) 0x80;
+    // 0x81-0x8F are additional lengths of strings.
 
     public static final byte INLINE_SYMBOL_ZERO_LENGTH = (byte) 0x90;
+    // 0x91-0x9F are additional lengths of symbols.
 
     public static final byte LIST_ZERO_LENGTH = (byte) 0xA0;
+    // 0xA1-0xAF are additional lengths of lists.
+
+    public static final byte SEXP_ZERO_LENGTH = (byte) 0xB0;
+    // 0xB1-0xBF are additional lengths of sexps.
+
+    public static final byte STRUCT_SID_ZERO_LENGTH = (byte) 0xC0;
+    // 0xC1 Reserved
+    // 0xC2-0xCF are additional lengths of structs.
+
+    // 0xD0-0xDF
+    // See https://github.com/amazon-ion/ion-docs/issues/292
 
     public static final byte SYMBOL_ADDRESS_1_BYTE = (byte) 0xE1;
     public static final byte SYMBOL_ADDRESS_2_BYTES = (byte) 0xE2;
@@ -68,7 +81,7 @@ private OpCodes() {}
     public static final byte VARIABLE_LENGTH_LIST = (byte) 0xFA;
     public static final byte VARIABLE_LENGTH_SEXP = (byte) 0xFB;
     public static final byte VARIABLE_LENGTH_STRUCT_WITH_SIDS = (byte) 0xFC;
-    public static final byte VARIABLE_LENGTH_STRUCT_WITH_FLEXSYMS = (byte) 0xFD;
+    public static final byte VARIABLE_LENGTH_STRUCT_WITH_FLEX_SYMS = (byte) 0xFD;
     public static final byte VARIABLE_LENGTH_BLOB = (byte) 0xFE;
     public static final byte VARIABLE_LENGTH_CLOB = (byte) 0xFF;
 }

src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java

@@ -1529,7 +1529,7 @@ public int writeFlexSym(int sid) {
         if (sid != 0) {
             return writeFlexInt(sid);
         } else {
-            writeByte((byte) 0x01);
+            writeByte(FlexInt.ZERO);
             writeByte((byte) 0x90);
             return 2;
         }
@@ -1540,7 +1540,7 @@ public int writeFlexSym(int sid) {
      */
     public int writeFlexSym(Utf8StringEncoder.Result text) {
     if (text.getEncodedLength() == 0) {
-        writeByte((byte) 0x01);
+        writeByte(FlexInt.ZERO);
         writeByte((byte) 0x80);
         return 2;
     } else {