From fea45ffbf454e3eca4b8ac09f7d27a64e4925760 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Thu, 12 Dec 2024 13:24:18 -0800 Subject: [PATCH 01/10] Everything is awesome! --- .../ion/impl/bin/IonManagedWriter_1_1.kt | 2 +- .../ion/impl/macro/EExpressionArgsReader.java | 9 +- .../com/amazon/ion/impl/macro/Expression.kt | 35 +- .../amazon/ion/impl/macro/MacroEvaluator.kt | 1474 +++++++++-------- .../impl/macro/MacroEvaluatorAsIonReader.kt | 14 +- .../com/amazon/ion/impl/macro/SystemMacro.kt | 67 +- .../ion/conformance/ConformanceTestRunner.kt | 11 +- .../ion/impl/IonRawTextWriterTest_1_1.kt | 3 + .../ion/impl/macro/MacroEvaluatorTest.kt | 68 +- 9 files changed, 974 insertions(+), 709 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt index 7848944fc..bd6bfcbad 100644 --- a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt @@ -628,7 +628,7 @@ internal class IonManagedWriter_1_1( is Expression.MacroInvocation -> { val invokedMacro = expression.macro if (invokedMacro is SystemMacro) { - stepInTdlSystemMacroInvocation(invokedMacro.systemSymbol) + stepInTdlSystemMacroInvocation(invokedMacro.systemSymbol!!) } else { val invokedAddress = macroTable[invokedMacro] ?: newMacros[invokedMacro] diff --git a/src/main/java/com/amazon/ion/impl/macro/EExpressionArgsReader.java b/src/main/java/com/amazon/ion/impl/macro/EExpressionArgsReader.java index 0549b0ef9..53f6fe861 100644 --- a/src/main/java/com/amazon/ion/impl/macro/EExpressionArgsReader.java +++ b/src/main/java/com/amazon/ion/impl/macro/EExpressionArgsReader.java @@ -231,15 +231,16 @@ private void readStreamAsExpressionGroup( * @param expressions receives the expressions as they are materialized. */ protected void readValueAsExpression(boolean isImplicitRest, List expressions) { - if (isMacroInvocation()) { + if (isImplicitRest && !isContainerAnExpressionGroup()) { + readStreamAsExpressionGroup(expressions); + return; + } else if (isMacroInvocation()) { collectEExpressionArgs(expressions); // TODO avoid recursion return; } IonType type = reader.encodingType(); List annotations = getAnnotations(); - if (isImplicitRest && !isContainerAnExpressionGroup()) { - readStreamAsExpressionGroup(expressions); - } else if (IonType.isContainer(type)) { + if (IonType.isContainer(type) && !reader.isNullValue()) { readContainerValueAsExpression(type, annotations, expressions); } else { readScalarValueAsExpression(type, annotations, expressions); diff --git a/src/main/java/com/amazon/ion/impl/macro/Expression.kt b/src/main/java/com/amazon/ion/impl/macro/Expression.kt index 555214f7b..7e3d95d1b 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Expression.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Expression.kt @@ -52,7 +52,11 @@ sealed interface Expression { * These expressions are the only ones that may be the output from the macro evaluator. * All [DataModelExpression]s are also valid to use as [TemplateBodyExpression]s and [EExpressionBodyExpression]s. */ - sealed interface DataModelExpression : Expression, EExpressionBodyExpression, TemplateBodyExpression + sealed interface DataModelExpression : Expression, EExpressionBodyExpression, TemplateBodyExpression, ExpansionOutputExpression + + sealed interface ExpansionOutputExpressionOrContinue + + sealed interface ExpansionOutputExpression : ExpansionOutputExpressionOrContinue /** * Interface for expressions that are _values_ in the Ion data model. @@ -65,12 +69,22 @@ sealed interface Expression { } /** Expressions that represent Ion container types */ - sealed interface DataModelContainer : HasStartAndEnd, DataModelValue + sealed interface DataModelContainer : HasStartAndEnd, DataModelValue { + val isConstructedFromMacro: Boolean + } + + data object ContinueExpansion : ExpansionOutputExpressionOrContinue + data object EndOfExpansion : ExpansionOutputExpression + + // TODO: See if we can remove this + data object EndOfContainer : ExpansionOutputExpression // , DataModelExpression /** * A temporary placeholder that is used only while a macro or e-expression is partially compiled. + * + * TODO: See if we can get rid of this by e.g. using nulls during macro compilation. */ - object Placeholder : TemplateBodyExpression, EExpressionBodyExpression + data object Placeholder : TemplateBodyExpression, EExpressionBodyExpression /** * A group of expressions that form the argument for one macro parameter. @@ -180,10 +194,11 @@ sealed interface Expression { * @property selfIndex the index of the first expression of the list (i.e. this instance) * @property endExclusive the index of the last expression contained in the list */ - data class ListValue( + data class ListValue @JvmOverloads constructor( override val annotations: List = emptyList(), override val selfIndex: Int, - override val endExclusive: Int + override val endExclusive: Int, + override val isConstructedFromMacro: Boolean = false, ) : DataModelContainer { override val type: IonType get() = IonType.LIST override fun withAnnotations(annotations: List) = copy(annotations = annotations) @@ -192,10 +207,11 @@ sealed interface Expression { /** * An Ion SExp that could contain variables or macro invocations. */ - data class SExpValue( + data class SExpValue @JvmOverloads constructor( override val annotations: List = emptyList(), override val selfIndex: Int, - override val endExclusive: Int + override val endExclusive: Int, + override val isConstructedFromMacro: Boolean = false, ) : DataModelContainer { override val type: IonType get() = IonType.SEXP override fun withAnnotations(annotations: List) = copy(annotations = annotations) @@ -204,11 +220,12 @@ sealed interface Expression { /** * An Ion Struct that could contain variables or macro invocations. */ - data class StructValue( + data class StructValue @JvmOverloads constructor( override val annotations: List = emptyList(), override val selfIndex: Int, override val endExclusive: Int, - val templateStructIndex: Map> + val templateStructIndex: Map> = emptyMap(), + override val isConstructedFromMacro: Boolean = false, ) : DataModelContainer { override val type: IonType get() = IonType.STRUCT override fun withAnnotations(annotations: List) = copy(annotations = annotations) diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index 008890994..41cbf8e1d 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -4,11 +4,39 @@ package com.amazon.ion.impl.macro import com.amazon.ion.* import com.amazon.ion.impl._Private_RecyclingStack -import com.amazon.ion.impl._Private_Utils.newSymbolToken +import com.amazon.ion.impl._Private_Utils +import com.amazon.ion.impl._Private_Utils.* import com.amazon.ion.impl.macro.Expression.* +import com.amazon.ion.impl.macro.MacroEvaluator.ExpanderKind.* import java.io.ByteArrayOutputStream +import java.lang.StringBuilder import java.math.BigDecimal import java.math.BigInteger +import java.util.IdentityHashMap + +private fun getExpanderKindForSystemMacro(systemMacro: SystemMacro) = when (systemMacro) { + SystemMacro.Annotate -> Annotate + SystemMacro.MakeString -> MakeString + SystemMacro.MakeSymbol -> MakeSymbol + SystemMacro.MakeDecimal -> MakeDecimal + SystemMacro.Repeat -> Repeat + SystemMacro.Sum -> Sum + SystemMacro.Delta -> Delta + SystemMacro.MakeBlob -> MakeBlob + SystemMacro.Flatten -> Flatten + SystemMacro.FlattenStruct -> FlattenStruct + SystemMacro.MakeTimestamp -> MakeTimestamp + SystemMacro.MakeFieldNameAndValue -> MakeFieldNameAndValue + SystemMacro.IfNone -> IfNone + SystemMacro.IfSome -> IfSome + SystemMacro.IfSingle -> IfSingle + SystemMacro.IfMulti -> IfMulti + else -> if (systemMacro.body != null) { + throw IllegalStateException("SystemMacro ${systemMacro.name} should be using its template body.") + } else { + TODO("Not implemented yet: ${systemMacro.name}") + } +} /** * Evaluates an EExpression from a List of [EExpressionBodyExpression] and the [TemplateBodyExpression]s @@ -25,780 +53,896 @@ import java.math.BigInteger */ class MacroEvaluator { - /** - * Implementations must update [ExpansionInfo.i] in order for [ExpansionInfo.hasNext] to work properly. - */ - private fun interface Expander { - fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression + // TODO: + data class ContainerInfo(var type: Type = Type.Uninitialized, private var _expansion: Expansion? = null) { + enum class Type { TopLevel, List, Sexp, Struct, Uninitialized } - /** - * Read the expanded values from one argument, returning exactly one value. - * Throws an exception if there is not exactly one expanded value. - */ - fun readExactlyOneExpandedArgumentValue(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, argName: String): DataModelExpression { - return readZeroOrOneExpandedArgumentValues(expansionInfo, macroEvaluator, argName) - ?: throw IonException("Argument $argName expanded to nothing.") + fun release() { + _expansion?.release() + _expansion = null + type = Type.Uninitialized } - /** - * Read the expanded values from one argument, returning zero or one values. - * Throws an exception if there is more than one expanded value. - */ - fun readZeroOrOneExpandedArgumentValues(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, argName: String): DataModelExpression? { - var value: DataModelExpression? = null - readExpandedArgumentValues(expansionInfo, macroEvaluator) { - if (value == null) { - value = it - } else { - throw IonException("Too many values for argument $argName") - } - true // Continue expansion + var expansion: Expansion + get() = _expansion!! + set(value) { _expansion = value } + } + + private val expansionPool = Pool { pool -> Expansion(pool) } + private val containerStack = _Private_RecyclingStack(8) { ContainerInfo() } + private var currentExpr: DataModelExpression? = null + + fun getArguments(): List { + return containerStack.iterator().next().expansion.expressions + } + + /** + * Initialize the macro evaluator with an E-Expression. + */ + fun initExpansion(encodingExpressions: List) { + containerStack.push { ci -> + ci.type = ContainerInfo.Type.TopLevel + ci.expansion = expansionPool.acquire { + it.initExpansion(Stream, encodingExpressions, 0, encodingExpressions.size, Environment.EMPTY) } - return value } + } - /** - * Reads the expanded values from one argument. - * - * The callback should return true to continue the expansion or false to abandon the expansion early. - */ - fun readExpandedArgumentValues(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, callback: (DataModelExpression) -> Boolean) { - val i = expansionInfo.i - expansionInfo.nextSourceExpression() - - macroEvaluator.pushExpansion( - expansionKind = ExpansionKind.Values, - argsStartInclusive = i, - // There can only be one top-level expression for an argument (it's either a value, macro, or - // expression group) so we can set the end to one more than the start. - argsEndExclusive = i + 1, - environment = expansionInfo.environment ?: Environment.EMPTY, - expressions = expansionInfo.expressions!!, - ) - - val depth = macroEvaluator.expansionStack.size() - var expr = macroEvaluator.expandNext(depth) - var continueExpansion: Boolean - while (expr != null) { - continueExpansion = callback(expr) - if (!continueExpansion) break - expr = macroEvaluator.expandNext(depth) - } - // Step back out to the original depth (in case we exited the expansion early) - while (macroEvaluator.expansionStack.size() > depth) { - macroEvaluator.expansionStack.pop() + /** + * Evaluate the macro expansion until the next [DataModelExpression] can be returned. + * Returns null if at the end of a container or at the end of the expansion. + */ + fun expandNext(): ExpansionOutputExpression? { + currentExpr = null + while (currentExpr == null && !containerStack.isEmpty()) { + val currentContainer = containerStack.peek() + + val nextExpansionOutput = currentContainer.expansion.produceNext() + when (nextExpansionOutput) { + is DataModelExpression -> currentExpr = nextExpansionOutput + EndOfExpansion -> { + // TODO: Do we need to release anything? + // TODO: Is there a better way to do this? + if (currentContainer.type == ContainerInfo.Type.TopLevel) { + currentContainer.release() + containerStack.pop() + } + return null + } + EndOfContainer -> { + return null + } } } + return currentExpr + } - /** - * Reads the first expanded value from one argument. - * - * Does not perform any sort of cardinality check, and leaves the evaluator stepped into the level of the - * returned expression. Returns null if the argument expansion produces no values. - */ - fun readFirstExpandedArgumentValue(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): DataModelExpression? { - val i = expansionInfo.i - expansionInfo.nextSourceExpression() - - macroEvaluator.pushExpansion( - expansionKind = ExpansionKind.Values, - argsStartInclusive = i, - // There can only be one top-level expression for an argument (it's either a value, macro, or - // expression group) so we can set the end to one more than the start. - argsEndExclusive = i + 1, - environment = expansionInfo.environment ?: Environment.EMPTY, - expressions = expansionInfo.expressions!!, - ) - - val depth = macroEvaluator.expansionStack.size() - return macroEvaluator.expandNext(depth) - } + /** + * Steps out of the current [DataModelContainer]. + */ + fun stepOut() { + // step out of anything we find until we have stepped out of a container. + if (containerStack.size() <= 1) throw IonException("Nothing to step out of.") + val popped = containerStack.pop() + popped.release() } - private object SimpleExpander : Expander { - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - return expansionInfo.nextSourceExpression() + /** + * Steps in to the current [DataModelContainer]. + * Throws [IonException] if not positioned on a container. + */ + fun stepIn() { + val expression = requireNotNull(currentExpr) { "Not positioned on a value" } + if (expression is DataModelContainer) { + val currentContainer = containerStack.peek() + if (expression.isConstructedFromMacro) { + val currentTop = currentContainer.expansion.top() + } else { + containerStack.push { ci -> + ci.type = when (expression.type) { + IonType.LIST -> ContainerInfo.Type.List + IonType.SEXP -> ContainerInfo.Type.Sexp + IonType.STRUCT -> ContainerInfo.Type.Struct + else -> TODO("Unreachable") + } + ci.expansion = expansionPool.acquire { + val topExpansion = currentContainer.expansion.top() + it.initExpansion( + expanderKind = Stream, + expressions = topExpansion.expressions, + startInclusive = expression.startInclusive, + endExclusive = expression.endExclusive, + environment = topExpansion.environment!!, + ) + } + } + } + currentExpr = null + } else { + throw IonException("Not positioned on a container.") } } - private object AnnotateExpander : Expander { - // TODO: Handle edge cases mentioned in https://github.com/amazon-ion/ion-docs/issues/347 - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - val annotations = mutableListOf() - - readExpandedArgumentValues(expansionInfo, macroEvaluator) { - when (it) { - is StringValue -> annotations.add(newSymbolToken(it.value)) - is SymbolValue -> annotations.add(it.value) - is DataModelValue -> throw IonException("Annotation arguments must be string or symbol; found: ${it.type}") - is FieldName -> TODO("Unreachable. Must encounter a StructValue first.") - } + enum class ExpanderKind { + Uninitialized { + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + throw IllegalStateException("ExpansionInfo not initialized.") } + }, + Empty { + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue = EndOfExpansion + }, + Stream { + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + val self = expansion + + // If there's a delegate, we'll try that first. + val delegate = self.expansionDelegate + if (delegate != null) { + val result = delegate.produceNext() + return when (result) { + is DataModelExpression -> result + // TODO: figure out some way to stick on this... or maybe it's not necessary. + // Test this by attempting to go beyond the end of containers. + EndOfContainer -> EndOfContainer + EndOfExpansion -> { + delegate.release() + self.expansionDelegate = null + ContinueExpansion + } + } + } - val valueToAnnotate = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.Annotate.signature[1].variableName) + if (self.i >= self.endExclusive) { + expansion.expanderKind = Empty + return ContinueExpansion + } - // It cannot be a FieldName expression because we haven't stepped into a struct, so it must be DataModelValue - valueToAnnotate as DataModelValue - // Combine the annotations - annotations.addAll(valueToAnnotate.annotations) - return valueToAnnotate.withAnnotations(annotations) - } - } + val next = self.expressions[self.i] + self.i++ + if (next is HasStartAndEnd) self.i = next.endExclusive + + return when (next) { + is DataModelExpression -> next + is EExpression -> { + val macro = next.macro + val argIndices = macro.calculateArgumentIndices( + encodingExpressions = expansion.expressions, + argsStartInclusive = next.startInclusive, + argsEndExclusive = next.endExclusive, + ) + val newEnvironment = self.environment.createChild(self.expressions, argIndices) + if (macro.body != null) { + self.expansionDelegate = self.expansionPool.acquire { new -> + new.initExpansion( + expanderKind = Stream, + expressions = macro.body!!, + startInclusive = 0, + endExclusive = macro.body!!.size, + environment = newEnvironment, + ) + } + } else { + val expanderKind = getExpanderKindForSystemMacro(macro as SystemMacro) + self.expansionDelegate = self.expansionPool.acquire { new -> + new.initExpansion( + expanderKind = expanderKind, + expressions = emptyList(), + startInclusive = 0, + endExclusive = 0, + environment = newEnvironment, + ) + } + } + ContinueExpansion + } + is MacroInvocation -> { + // TODO: Verify if this is correct + val macro = next.macro + val argIndices = macro.calculateArgumentIndices( + encodingExpressions = expansion.expressions, + argsStartInclusive = next.startInclusive, + argsEndExclusive = next.endExclusive, + ) + val newEnvironment = self.environment.createChild(self.expressions, argIndices) + if (macro.body != null) { + self.expansionDelegate = self.expansionPool.acquire { new -> + new.initExpansion( + expanderKind = Stream, + expressions = macro.body!!, + startInclusive = 0, + endExclusive = macro.body!!.size, + environment = newEnvironment, + ) + } + } else { + val expanderKind = getExpanderKindForSystemMacro(macro as SystemMacro) + self.expansionDelegate = self.expansionPool.acquire { new -> + new.initExpansion( + expanderKind = expanderKind, + expressions = emptyList(), + startInclusive = 0, + endExclusive = 0, + environment = newEnvironment, + ) + } + } + ContinueExpansion + } + is ExpressionGroup -> { + self.expansionDelegate = self.expansionPool.acquire { new -> + new.initExpansion( + expanderKind = Stream, + expressions = self.expressions, + startInclusive = next.startInclusive, + endExclusive = next.endExclusive, + environment = self.environment, + ) + } + ContinueExpansion + } + + is VariableRef -> { + self.expansionDelegate = self.readArgument(next) + ContinueExpansion + } - private object MakeStringExpander : Expander { - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - val sb = StringBuilder() - readExpandedArgumentValues(expansionInfo, macroEvaluator) { - when (it) { - is StringValue -> sb.append(it.value) - is SymbolValue -> sb.append(it.value.assumeText()) - is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}") - is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.") + Placeholder -> TODO("Unreachable") } - true // continue expansion } - return StringValue(value = sb.toString()) - } - } + }, + OneValuedStream { + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + if (expansion.additionalState != 1) { + return when (val firstValue = Stream.produceNext(expansion)) { + is DataModelExpression -> { + expansion.additionalState = 1 + firstValue + } + ContinueExpansion -> ContinueExpansion + EndOfExpansion -> throw IonException("Expected one value, found 0") + EndOfContainer -> TODO("Unused?") + } + } else { + return when (val secondValue = Stream.produceNext(expansion)) { + is DataModelExpression -> throw IonException("Expected one value, found multiple") + ContinueExpansion -> ContinueExpansion + EndOfExpansion -> secondValue + EndOfContainer -> TODO("Unused?") + } + } + } + }, + IfNone { + private val ARG_TO_TEST = VariableRef(0) + private val TRUE_BRANCH = VariableRef(1) + private val FALSE_BRANCH = VariableRef(2) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + val testArg = expansion.readArgument(ARG_TO_TEST) + var n = 0 + while (n < 2) { + if (testArg.produceNext() is EndOfExpansion) break + n++ + } - private object MakeSymbolExpander : Expander { - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - val sb = StringBuilder() - readExpandedArgumentValues(expansionInfo, macroEvaluator) { - when (it) { - is StringValue -> sb.append(it.value) - is SymbolValue -> sb.append(it.value.assumeText()) - is DataModelValue -> throw IonException("Invalid argument type for 'make_symbol': ${it.type}") - is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.") + val branch = if (n > 0) FALSE_BRANCH else TRUE_BRANCH + val branchExpansion = expansion.readArgument(branch) + expansion.reInitializeFrom(branchExpansion) + branchExpansion.release() + testArg.release() + return ContinueExpansion + } + }, + IfSome { + private val ARG_TO_TEST = VariableRef(0) + private val TRUE_BRANCH = VariableRef(1) + private val FALSE_BRANCH = VariableRef(2) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + val testArg = expansion.readArgument(ARG_TO_TEST) + var n = 0 + while (n < 2) { + if (testArg.produceNext() is EndOfExpansion) break + n++ } - true // continue expansion + testArg.release() + + val branch = if (n > 0) TRUE_BRANCH else FALSE_BRANCH + val branchExpansion = expansion.readArgument(branch) + expansion.reInitializeFrom(branchExpansion) + branchExpansion.release() + return ContinueExpansion } - return SymbolValue(value = newSymbolToken(sb.toString())) - } - } + }, + IfSingle { + private val ARG_TO_TEST = VariableRef(0) + private val TRUE_BRANCH = VariableRef(1) + private val FALSE_BRANCH = VariableRef(2) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + val testArg = expansion.readArgument(ARG_TO_TEST) + var n = 0 + while (n < 2) { + if (testArg.produceNext() is EndOfExpansion) break + n++ + } + testArg.release() - private object MakeBlobExpander : Expander { - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - // TODO: See if we can create a `ByteArrayView` or similar class based on the principles of a Persistent - // Collection in order to minimize copying (and therefore allocation). - val baos = ByteArrayOutputStream() - readExpandedArgumentValues(expansionInfo, macroEvaluator) { - when (it) { - is LobValue -> baos.write(it.value) - is DataModelValue -> throw IonException("Invalid argument type for 'make_blob': ${it.type}") - is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.") + val branch = if (n == 1) TRUE_BRANCH else FALSE_BRANCH + val branchExpansion = expansion.readArgument(branch) + expansion.reInitializeFrom(branchExpansion) + branchExpansion.release() + return ContinueExpansion + } + }, + IfMulti { + private val ARG_TO_TEST = VariableRef(0) + private val TRUE_BRANCH = VariableRef(1) + private val FALSE_BRANCH = VariableRef(2) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + val testArg = expansion.readArgument(ARG_TO_TEST) + var n = 0 + while (n < 2) { + if (testArg.produceNext() is EndOfExpansion) break + n++ } - true // continue expansion + testArg.release() + + val branch = if (n > 1) TRUE_BRANCH else FALSE_BRANCH + val branchExpansion = expansion.readArgument(branch) + expansion.reInitializeFrom(branchExpansion) + branchExpansion.release() + return ContinueExpansion } - return BlobValue(value = baos.toByteArray()) - } - } + }, - private object MakeDecimalExpander : Expander { - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - val coefficient = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeDecimal.signature[0].variableName) - .let { it as? IntValue } - ?.bigIntegerValue - ?: throw IonException("Coefficient must be an integer") - val exponent = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeDecimal.signature[1].variableName) - .let { it as? IntValue } - ?.bigIntegerValue - ?: throw IonException("Exponent must be an integer") - - return DecimalValue(value = BigDecimal(coefficient, -1 * exponent.intValueExact())) - } - } + Annotate { + + private val ANNOTATIONS_ARG = VariableRef(0) + private val VALUE_TO_ANNOTATE_ARG = VariableRef(1) - private object MakeTimestampExpander : Expander { - private fun readOptionalIntArg( - signatureIndex: Int, - expansionInfo: ExpansionInfo, - macroEvaluator: MacroEvaluator - ): Int? { - if (expansionInfo.i == expansionInfo.endExclusive) return null - val parameterName = SystemMacro.MakeTimestamp.signature[signatureIndex].variableName - val arg = readZeroOrOneExpandedArgumentValues(expansionInfo, macroEvaluator, parameterName) - return arg?.let { - it as? IntValue ?: throw IonException("$parameterName must be an integer") - it.longValue.toInt() + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + val annotations = expansion.readArgument(ANNOTATIONS_ARG).map { + when (it) { + is StringValue -> _Private_Utils.newSymbolToken(it.value) + is SymbolValue -> it.value + is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}") + else -> TODO("Unreachable without stepping in to a container") + } + } + + val valueToAnnotateExpansion = expansion.readArgument(VALUE_TO_ANNOTATE_ARG) + + val annotatedExpression = valueToAnnotateExpansion.produceNext().let { + it as? DataModelValue ?: throw IonException("Required at least one value.") + it.withAnnotations(annotations + it.annotations) + } + // Tail-recursion-like optimization + expansion.reInitializeFrom(valueToAnnotateExpansion) + expansion.expanderKind = OneValuedStream + return annotatedExpression } - } + }, + MakeString { + private val STRINGS_ARG = VariableRef(0) - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - val year = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeTimestamp.signature[0].variableName) - .let { it as? IntValue ?: throw IonException("year must be an integer") } - .longValue.toInt() - val month = readOptionalIntArg(1, expansionInfo, macroEvaluator) - val day = readOptionalIntArg(2, expansionInfo, macroEvaluator) - val hour = readOptionalIntArg(3, expansionInfo, macroEvaluator) - val minute = readOptionalIntArg(4, expansionInfo, macroEvaluator) - val second = if (expansionInfo.i == expansionInfo.endExclusive) { - null - } else when (val arg = readZeroOrOneExpandedArgumentValues(expansionInfo, macroEvaluator, SystemMacro.MakeTimestamp.signature[5].variableName)) { - null -> null - is DecimalValue -> arg.value - is IntValue -> arg.longValue.toBigDecimal() - else -> throw IonException("second must be a decimal") + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + val sb = StringBuilder() + expansion.readArgument(STRINGS_ARG).forEach { + when (it) { + is StringValue -> sb.append(it.value) + is SymbolValue -> sb.append(it.value.assumeText()) + is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}") + is FieldName -> TODO("Unreachable.") + } + } + expansion.expanderKind = Empty + return StringValue(value = sb.toString()) } - val offsetMinutes = readOptionalIntArg(6, expansionInfo, macroEvaluator) - - try { - val ts = if (second != null) { - month ?: throw IonException("make_timestamp: month is required when second is present") - day ?: throw IonException("make_timestamp: day is required when second is present") - hour ?: throw IonException("make_timestamp: hour is required when second is present") - minute ?: throw IonException("make_timestamp: minute is required when second is present") - Timestamp.forSecond(year, month, day, hour, minute, second, offsetMinutes) - } else if (minute != null) { - month ?: throw IonException("make_timestamp: month is required when minute is present") - day ?: throw IonException("make_timestamp: day is required when minute is present") - hour ?: throw IonException("make_timestamp: hour is required when minute is present") - Timestamp.forMinute(year, month, day, hour, minute, offsetMinutes) - } else if (hour != null) { - throw IonException("make_timestamp: minute is required when hour is present") - } else { - if (offsetMinutes != null) throw IonException("make_timestamp: offset_minutes is prohibited when hours and minute are not present") - if (day != null) { - month ?: throw IonException("make_timestamp: month is required when day is present") - Timestamp.forDay(year, month, day) - } else if (month != null) { - Timestamp.forMonth(year, month) - } else { - Timestamp.forYear(year) + }, + MakeSymbol { + private val STRINGS_ARG = VariableRef(0) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + if (expansion.additionalState != null) return EndOfExpansion + expansion.additionalState = Unit + + val sb = StringBuilder() + expansion.readArgument(STRINGS_ARG).forEach { + when (it) { + is StringValue -> sb.append(it.value) + is SymbolValue -> sb.append(it.value.assumeText()) + is DataModelValue -> throw IonException("Invalid argument type for 'make_symbol': ${it.type}") + is FieldName -> TODO("Unreachable.") } } - return TimestampValue(value = ts) - } catch (e: IllegalArgumentException) { - throw IonException(e.message) + return SymbolValue(value = _Private_Utils.newSymbolToken(sb.toString())) } - } - } + }, + MakeBlob { + private val LOB_ARG = VariableRef(0) - private object SumExpander : Expander { - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - val a = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, "a") - val b = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, "b") - if (a !is IntValue || b !is IntValue) throw IonException("operands of sum must be integers") - // TODO: Use LongIntValue when possible. - return BigIntValue(value = a.bigIntegerValue + b.bigIntegerValue) - } - } + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + // TODO: Optimize to see if we can create a Byte "view" over the existing byte arrays. + if (expansion.additionalState != null) return EndOfExpansion + expansion.additionalState = Unit - private object DeltaExpander : Expander { - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - // TODO: Optimize to use Long and only fallback to BigInteger if needed. - // TODO: Optimize for lazy evaluation - if (expansionInfo.additionalState == null) { - val position = expansionInfo.i - var runningTotal = BigInteger.ZERO - val values = ArrayDeque() - readExpandedArgumentValues(expansionInfo, macroEvaluator) { + val baos = ByteArrayOutputStream() + expansion.readArgument(LOB_ARG).forEach { when (it) { - is IntValue -> { - runningTotal += it.bigIntegerValue - values += runningTotal - } - is DataModelValue -> throw IonException("Invalid argument type for 'delta': ${it.type}") - is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.") + is LobValue -> baos.write(it.value) + is DataModelValue -> throw IonException("Invalid argument type for 'make_blob': ${it.type}") + is FieldName -> TODO("Unreachable.") + } + } + return BlobValue(value = baos.toByteArray()) + } + }, + MakeDecimal { + private val COEFFICIENT_ARG = VariableRef(0) + private val EXPONENT_ARG = VariableRef(1) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + if (expansion.additionalState != null) return EndOfExpansion + expansion.additionalState = Unit + + val coefficient = expansion.readExactlyOneArgument(COEFFICIENT_ARG).bigIntegerValue + val exponent = expansion.readExactlyOneArgument(EXPONENT_ARG).bigIntegerValue + return DecimalValue(value = BigDecimal(coefficient, -1 * exponent.intValueExact())) + } + }, + MakeTimestamp { + private val YEAR_ARG = VariableRef(0) + private val MONTH_ARG = VariableRef(1) + private val DAY_ARG = VariableRef(2) + private val HOUR_ARG = VariableRef(3) + private val MINUTE_ARG = VariableRef(4) + private val SECOND_ARG = VariableRef(5) + private val OFFSET_ARG = VariableRef(6) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + val year = expansion.readExactlyOneArgument(YEAR_ARG).longValue.toInt() + val month = expansion.readZeroOrOneArgument(MONTH_ARG)?.longValue?.toInt() + val day = expansion.readZeroOrOneArgument(DAY_ARG)?.longValue?.toInt() + val hour = expansion.readZeroOrOneArgument(HOUR_ARG)?.longValue?.toInt() + val minute = expansion.readZeroOrOneArgument(MINUTE_ARG)?.longValue?.toInt() + val second = expansion.readZeroOrOneArgument(SECOND_ARG)?.let { + when (it) { + is DecimalValue -> it.value + is IntValue -> it.longValue.toBigDecimal() + else -> throw IonException("second must be an integer or decimal") } - true // continue expansion } - if (values.isEmpty()) { - // Return fake, empty expression group - return ExpressionGroup(position, position) + val offsetMinutes = expansion.readZeroOrOneArgument(OFFSET_ARG)?.longValue?.toInt() + + try { + val ts = if (second != null) { + month ?: throw IonException("make_timestamp: month is required when second is present") + day ?: throw IonException("make_timestamp: day is required when second is present") + hour ?: throw IonException("make_timestamp: hour is required when second is present") + minute ?: throw IonException("make_timestamp: minute is required when second is present") + Timestamp.forSecond(year, month, day, hour, minute, second, offsetMinutes) + } else if (minute != null) { + month ?: throw IonException("make_timestamp: month is required when minute is present") + day ?: throw IonException("make_timestamp: day is required when minute is present") + hour ?: throw IonException("make_timestamp: hour is required when minute is present") + Timestamp.forMinute(year, month, day, hour, minute, offsetMinutes) + } else if (hour != null) { + throw IonException("make_timestamp: minute is required when hour is present") + } else { + if (offsetMinutes != null) throw IonException("make_timestamp: offset_minutes is prohibited when hours and minute are not present") + if (day != null) { + month ?: throw IonException("make_timestamp: month is required when day is present") + Timestamp.forDay(year, month, day) + } else if (month != null) { + Timestamp.forMonth(year, month) + } else { + Timestamp.forYear(year) + } + } + expansion.expanderKind = Empty + return TimestampValue(value = ts) + } catch (e: IllegalArgumentException) { + throw IonException(e.message) + } + } + }, + MakeFieldNameAndValue { + private val FIELD_NAME = VariableRef(0) + private val FIELD_VALUE = VariableRef(1) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + val fieldName = expansion.readExactlyOneArgument(FIELD_NAME) + val fieldNameExpression = when (fieldName) { + is SymbolValue -> FieldName(fieldName.value) + is StringValue -> FieldName(newSymbolToken(fieldName.value)) } - expansionInfo.additionalState = values - expansionInfo.i = position + expansion.readExactlyOneArgument(FIELD_VALUE) + + val valueExpansion = expansion.readArgument(FIELD_VALUE) + + expansion.reInitializeFrom(valueExpansion) + expansion.expanderKind = OneValuedStream + return fieldNameExpression } + }, + + FlattenStruct { + private val STRUCTS = VariableRef(0) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + var argumentExpansion: Expansion? = expansion.additionalState as Expansion? + if (argumentExpansion == null) { + argumentExpansion = expansion.readArgument(STRUCTS) + expansion.additionalState = argumentExpansion + } + + val currentChildExpansion = expansion.expansionDelegate - val valueQueue = expansionInfo.additionalState as ArrayDeque - val nextValue = valueQueue.removeFirst() - if (valueQueue.isEmpty()) { - expansionInfo.i = expansionInfo.endExclusive + return when (val next = currentChildExpansion?.produceNext()) { + is DataModelExpression -> next + EndOfContainer -> TODO("I think this is unused!") + EndOfExpansion -> { + expansion.expansionDelegate!!.release() + expansion.expansionDelegate = null + ContinueExpansion + } + // Only possible if expansionDelegate is null + null -> when (val nextSequence = argumentExpansion.produceNext()) { + is StructValue -> { + expansion.expansionDelegate = expansion.expansionPool.acquire { child -> + child.initExpansion( + expanderKind = Stream, + expressions = argumentExpansion.top().expressions, + startInclusive = nextSequence.startInclusive, + endExclusive = nextSequence.endExclusive, + environment = argumentExpansion.top().environment, + ) + } + ContinueExpansion + } + EndOfExpansion -> EndOfExpansion + is DataModelExpression -> throw IonException("invalid argument; make_struct expects structs") + EndOfContainer -> TODO("Unreachable") + } + } } - return BigIntValue(value = nextValue) - } - } + }, - private enum class IfExpander(private val minInclusive: Int, private val maxExclusive: Int) : Expander { - IF_NONE(0, 1), - IF_SOME(1, -1), - IF_SINGLE(1, 2), - IF_MULTI(2, -1), - ; + Flatten { + private val SEQUENCES = VariableRef(0) - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - var n = 0 - readExpandedArgumentValues(expansionInfo, macroEvaluator) { - n++ - // If there's no max, then we'll only continue the expansion if we haven't yet reached the min - // If there is a max, then we'll continue the expansion until we reach the max - if (maxExclusive < 0) n < minInclusive else n < maxExclusive + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + var argumentExpansion: Expansion? = expansion.additionalState as Expansion? + if (argumentExpansion == null) { + argumentExpansion = expansion.readArgument(SEQUENCES) + expansion.additionalState = argumentExpansion + } + + val currentChildExpansion = expansion.expansionDelegate + + return when (val next = currentChildExpansion?.produceNext()) { + is DataModelExpression -> next + EndOfContainer -> TODO("I think this is unused!") + EndOfExpansion -> { + expansion.expansionDelegate!!.release() + expansion.expansionDelegate = null + ContinueExpansion + } + // Only possible if expansionDelegate is null + null -> when (val nextSequence = argumentExpansion.produceNext()) { + is StructValue -> throw IonException("invalid argument; flatten expects sequences") + is DataModelContainer -> { + expansion.expansionDelegate = expansion.expansionPool.acquire { child -> + child.initExpansion( + expanderKind = Stream, + expressions = argumentExpansion.top().expressions, + startInclusive = nextSequence.startInclusive, + endExclusive = nextSequence.endExclusive, + environment = argumentExpansion.top().environment, + ) + } + ContinueExpansion + } + EndOfExpansion -> EndOfExpansion + is DataModelExpression -> throw IonException("invalid argument; flatten expects sequences") + EndOfContainer -> TODO("Unreachable") + } + } } - val isConditionTrue = n >= minInclusive && (maxExclusive < 0 || n < maxExclusive) - // Save the current expansion index. This is the index of the "true" expression - val trueExpressionPosition = expansionInfo.i - // Now we are positioned on the "false" expression - expansionInfo.nextSourceExpression() - if (isConditionTrue) { - // If the condition is true, we can set the EXCLUSIVE END of this expansion to the position of the - // "false" expression, and then we reset the current index to the position of the "true" expression. - expansionInfo.endExclusive = expansionInfo.i - expansionInfo.i = trueExpressionPosition + }, + Sum { + private val ARG_A = VariableRef(0) + private val ARG_B = VariableRef(1) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + if (expansion.additionalState != null) return EndOfExpansion + expansion.additionalState = Unit + + val a = expansion.readExactlyOneArgument(ARG_A).bigIntegerValue + val b = expansion.readExactlyOneArgument(ARG_B).bigIntegerValue + return BigIntValue(value = a + b) } - return expansionInfo.nextSourceExpression() - } - } + }, + Delta { + private val ARGS = VariableRef(0) + + // Initial value = 0 + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + // TODO: Optimize to use LongIntValue when possible + var delegate = expansion.expansionDelegate + val runningTotal = expansion.additionalState as? BigInteger ?: BigInteger.ZERO + if (delegate == null) { + delegate = expansion.readArgument(ARGS) + expansion.expansionDelegate = delegate + } - private object RepeatExpander : Expander { - /** - * Initializes the counter of the number of iterations remaining. - * [ExpansionInfo.additionalState] is the number of iterations remaining. Once initialized, it is always `Int`. - */ - private fun init(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Int { - val nExpression = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, "n") - var iterationsRemaining = when (nExpression) { - is LongIntValue -> nExpression.value.toInt() - is BigIntValue -> { - if (nExpression.value.bitLength() >= Int.SIZE_BITS) { - throw IonException("ion-java does not support repeats of more than ${Int.MAX_VALUE}") + when (val nextExpandedArg = delegate.produceNext()) { + is IntValue -> { + val nextDelta = nextExpandedArg.bigIntegerValue + val nextOutput = runningTotal + nextDelta + expansion.additionalState = nextOutput + return BigIntValue(value = nextOutput) } - nExpression.value.intValueExact() + EndOfExpansion -> return nextExpandedArg + is DataModelValue -> throw IonException("delta arguments must be integers") + is FieldName, EndOfContainer -> TODO("Unreachable") } - else -> throw IonException("The first argument of repeat must be a non-negative integer") } - if (iterationsRemaining < 0) { - throw IonException("The first argument of repeat must be a non-negative integer") - } - // Decrement because we're starting the first iteration right away. - iterationsRemaining-- - expansionInfo.additionalState = iterationsRemaining - return iterationsRemaining - } + }, + Repeat { + private val COUNT_ARG = VariableRef(0) + private val THING_TO_REPEAT = VariableRef(1) + + override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + var n = expansion.additionalState as Long? + if (n == null) { + n = expansion.readExactlyOneArgument(COUNT_ARG).longValue + if (n < 0) throw IonException("invalid argument; 'n' must be non-negative") + expansion.additionalState = n + } - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - val repeatsRemainingAfterTheCurrentOne = expansionInfo.additionalState as? Int - ?: init(expansionInfo, macroEvaluator) + if (expansion.expansionDelegate == null) { + if (n > 0) { + expansion.expansionDelegate = expansion.readArgument(THING_TO_REPEAT) + expansion.additionalState = n - 1 + } else { + return EndOfExpansion + } + } - if (repeatsRemainingAfterTheCurrentOne < 0) { - expansionInfo.nextSourceExpression() - return ExpressionGroup(0, 0) + val repeated = expansion.expansionDelegate!! + return when (val maybeNext = repeated.produceNext()) { + is DataModelExpression, EndOfContainer -> maybeNext + EndOfExpansion -> { + expansion.expansionDelegate!!.release() + expansion.expansionDelegate = null + ContinueExpansion + } + } } + }, + ; - val repeatedExpressionIndex = expansionInfo.i - val next = readFirstExpandedArgumentValue(expansionInfo, macroEvaluator) - next ?: return ExpressionGroup(0, 0) - if (repeatsRemainingAfterTheCurrentOne > 0) { - expansionInfo.additionalState = repeatsRemainingAfterTheCurrentOne - 1 - expansionInfo.i = repeatedExpressionIndex + abstract fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue + + protected fun Expansion.readArgument(variableRef: VariableRef): Expansion { + val argIndex = environment.argumentIndices[variableRef.signatureIndex] + if (argIndex < 0) { + // Argument was elided. + return expansionPool.acquire { it.expanderKind = Empty } + } + val firstArgExpression = environment.arguments[argIndex] + + return expansionPool.acquire { new -> + new.initExpansion( + expanderKind = Stream, + expressions = environment.arguments, + startInclusive = if (firstArgExpression is ExpressionGroup) firstArgExpression.startInclusive else argIndex, + endExclusive = if (firstArgExpression is HasStartAndEnd) firstArgExpression.endExclusive else argIndex + 1, + environment = environment.parentEnvironment!! + ) } - return next } - } - private object MakeFieldExpander : Expander { - // This is wrong! - override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - /** - * Uses [ExpansionInfo.additionalState] to track whether the expansion is on the field name or value. - * If unset, reads the field name. If set to 0, reads the field value. - */ - return when (expansionInfo.additionalState) { - // First time, get the field name - null -> { - val fieldName = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, "field_name") - val fieldNameExpression = when (fieldName) { - is SymbolValue -> FieldName(fieldName.value) - else -> throw IonException("the first argument of make_field must expand to exactly one symbol value") - } - expansionInfo.additionalState = 0 - fieldNameExpression + protected inline fun Expansion.forEach(action: (DataModelExpression) -> Unit) { + while (true) { + when (val next = produceNext()) { + EndOfContainer, EndOfExpansion -> return + is DataModelExpression -> action(next) } - 0 -> { - val value = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, "value") - expansionInfo.additionalState = 1 - value - } - else -> throw IllegalStateException("Unreachable") } } - } - private enum class ExpansionKind(val expander: Expander) { - Container(SimpleExpander), - TemplateBody(SimpleExpander), - Values(SimpleExpander), - Annotate(AnnotateExpander), - MakeString(MakeStringExpander), - MakeSymbol(MakeSymbolExpander), - MakeBlob(MakeBlobExpander), - MakeDecimal(MakeDecimalExpander), - MakeTimestamp(MakeTimestampExpander), - MakeField(MakeFieldExpander), - Sum(SumExpander), - Delta(DeltaExpander), - IfNone(IfExpander.IF_NONE), - IfSome(IfExpander.IF_SOME), - IfSingle(IfExpander.IF_SINGLE), - IfMulti(IfExpander.IF_MULTI), - Repeat(RepeatExpander), - ; + protected inline fun Expansion.map(action: (DataModelExpression) -> T): List { + val result = mutableListOf() + while (true) { + when (val next = produceNext()) { + EndOfContainer, EndOfExpansion -> return result + is DataModelExpression -> result.add(action(next)) + } + } + } - companion object { - @JvmStatic - fun forSystemMacro(macro: SystemMacro): ExpansionKind { - return when (macro) { - SystemMacro.IfNone -> IfNone - SystemMacro.IfSome -> IfSome - SystemMacro.IfSingle -> IfSingle - SystemMacro.IfMulti -> IfMulti - SystemMacro.None -> Values // "none" takes no args, so we can treat it as an empty "values" expansion - SystemMacro.Values -> Values - SystemMacro.Annotate -> Annotate - SystemMacro.MakeString -> MakeString - SystemMacro.MakeSymbol -> MakeSymbol - SystemMacro.MakeDecimal -> MakeDecimal - SystemMacro.MakeTimestamp -> MakeTimestamp - SystemMacro.MakeBlob -> MakeBlob - SystemMacro.MakeField -> MakeField - SystemMacro.Repeat -> Repeat - SystemMacro.Sum -> Sum - SystemMacro.Delta -> Delta - else -> if (macro.body != null) { - TemplateBody + protected inline fun Expansion.readZeroOrOneArgument(variableRef: VariableRef): T? { + val argExpansion = readArgument(variableRef) + var argValue: T? = null + while (true) { + when (val it = argExpansion.produceNext()) { + is T -> if (argValue == null) { + argValue = it } else { - TODO("System macro ${macro.macroName} needs either a template body or a hard-coded expander.") + throw IonException("invalid argument; too many values") } + is DataModelValue -> throw IonException("invalid argument; found ${it.type}") + EndOfExpansion -> break + EndOfContainer, + is FieldName -> TODO("Unreachable without stepping into a container") } } + return argValue + } + + protected inline fun Expansion.readExactlyOneArgument(variableRef: VariableRef): T { + return readZeroOrOneArgument(variableRef) ?: throw IonException("invalid argument; no value when one is expected") } } - private inner class ExpansionInfo : Iterator { - /** The [ExpansionKind]. */ - @JvmField var expansionKind: ExpansionKind = ExpansionKind.Values - /** - * The evaluation [Environment]—i.e. variable bindings. - */ - @JvmField var environment: Environment? = null + class Expansion( + @JvmField val expansionPool: Pool, + + @JvmField var expanderKind: ExpanderKind = Uninitialized, /** * The [Expression]s being expanded. This MUST be the original list, not a sublist because * (a) we don't want to be allocating new sublists all the time, and (b) the * start and end indices of the expressions may be incorrect if a sublist is taken. */ - @JvmField var expressions: List? = null - // /** Start of [expressions] that are applicable for this [ExpansionInfo] */ - // TODO: Do we actually need this for anything other than debugging? - // @JvmField var startInclusive: Int = 0 - /** End of [expressions] that are applicable for this [ExpansionInfo] */ - @JvmField var endExclusive: Int = 0 + @JvmField var expressions: List = emptyList(), /** Current position within [expressions] of this expansion */ - @JvmField var i: Int = 0 - + @JvmField var i: Int = 0, + /** End of [expressions] that are applicable for this [ExpansionInfo] */ + @JvmField var endExclusive: Int = 0, /** - * Field for storing any additional state required in an expander. - * - * TODO: Once all system macros are implemented, see if we can make this an int instead - * - * There is currently some lost space in ExpansionInfo. We can add one more `additionalState` field without - * actually increasing the object size. + * The evaluation [Environment]—i.e. variable bindings. */ - @JvmField - var additionalState: Any? = null + @JvmField var environment: Environment = Environment.EMPTY, + // TODO: Should this be "additional state"? + @JvmField var expansionDelegate: Expansion? = null, + @JvmField var additionalState: Any? = null, + ) { + fun top(): Expansion = expansionDelegate?.top() ?: this - /** Checks if this expansion can produce any more expressions */ - override fun hasNext(): Boolean = i < endExclusive + fun release() { + expanderKind = Uninitialized + additionalState = null + expansionDelegate?.release() + expansionPool.take(this) + } - /** Returns the next expression from this expansion */ - override fun next(): Expression { - return expansionKind.expander.nextExpression(this, this@MacroEvaluator) + fun initExpansion( + expanderKind: ExpanderKind, + expressions: List, + startInclusive: Int, + endExclusive: Int, + environment: Environment, + ) { + this.expanderKind = expanderKind + this.expressions = expressions + this.i = startInclusive + this.endExclusive = endExclusive + this.environment = environment + additionalState = null + expansionDelegate = null } - /** - * Returns the next expression from the input expressions ([expressions]) of this Expansion. - * This is intended for use in [Expander] implementations. - */ - fun nextSourceExpression(): Expression { - val next = expressions!![i] - i++ - if (next is HasStartAndEnd) i = next.endExclusive - return next + fun reInitializeFrom(other: Expansion) { + this.expanderKind = other.expanderKind + this.expressions = other.expressions + this.i = other.i + this.endExclusive = other.endExclusive + this.expansionDelegate = other.expansionDelegate + this.additionalState = other.additionalState + } + + fun produceNext(): ExpansionOutputExpression { + while (true) { + val next = expanderKind.produceNext(this) + if (next is ExpansionOutputExpression) return next + // Implied: + // if (next is ContinueExpansion) continue + } } override fun toString() = """ |ExpansionInfo( - | expansionKind: $expansionKind, + | expansionKind: $expanderKind, | environment: $environment, | expressions: [ - | ${expressions!!.joinToString(",\n| ") { it.toString() } } + | ${expressions.joinToString(",\n| ") { it.toString() } } | ], | endExclusive: $endExclusive, | i: $i, + | child: ${expansionDelegate?.expanderKind} + | additionalState: $additionalState, |) """.trimMargin() } - private val expansionStack = _Private_RecyclingStack(8) { ExpansionInfo() } - - private var currentExpr: DataModelExpression? = null - - /** - * Initialize the macro evaluator with an E-Expression. - */ - fun initExpansion(encodingExpressions: List) { - // Pretend that the whole thing is a "values" expansion so that we don't have to care about what - // the first expression actually is. - pushExpansion(ExpansionKind.Values, 0, encodingExpressions.size, Environment.EMPTY, encodingExpressions) - } - /** - * Returns the e-expression argument expressions that this MacroEvaluator would evaluate. - */ - fun getArguments(): List { - return expansionStack.peek().expressions!! - } - - /** - * Evaluate the macro expansion until the next [DataModelExpression] can be returned. - * Returns null if at the end of a container or at the end of the expansion. - */ - fun expandNext(): DataModelExpression? { - return expandNext(-1) - } - - /** - * Evaluate the macro expansion until the next [DataModelExpression] can be returned. - * Returns null if at the end of a container or at the end of the expansion. + * Suitable for single-threaded use only. * - * Treats [minDepth] as the minimum expansion depth allowed—i.e. it will not step out any further than - * [minDepth]. This is used for built-in macros when they need to delegate something to the macro evaluator - * but don't want the macro evaluator to step out beyond the invoking built-in macro. + * TODO: Clean up the debugging parts. */ - private fun expandNext(minDepth: Int): DataModelExpression? { - - /* ==== Evaluation Algorithm ==== - 01 | Check the top expansion in the expansion stack - 02 | If there is none, return null (macro expansion is over) - 03 | If there is one, but it has no more expressions... - 04 | If the expansion kind is a data-model container type, return null (user needs to step out) - 05 | If the expansion kind is not a data-model container type, automatically step out - 06 | If there is one, and it has more expressions... - 07 | If it is a scalar, return that - 08 | If it is a container, return that (user needs to step in) - 09 | If it is a variable, using parent Environment, push variable ExpansionInfo onto the stack and goto 1 - 10 | If it is an expression group, using current Environment, push expression group ExpansionInfo onto the stack and goto 1 - 11 | If it is a macro invocation, create updated Environment, push ExpansionInfo onto stack, and goto 1 - 12 | If it is an e-expression, using empty Environment, push ExpansionInfo onto stack and goto 1 - */ - - currentExpr = null - while (!expansionStack.isEmpty) { - if (!expansionStack.peek().hasNext()) { - if (expansionStack.peek().expansionKind == ExpansionKind.Container) { - // End of container. User needs to step out. - // TODO: Do we need something to distinguish End-Of-Expansion from End-Of-Container? - return null - } else { - // End of a macro invocation or something else that is not part of the data model, - // so we seamlessly close this out and continue with the parent expansion. - if (expansionStack.size() > minDepth) { - expansionStack.pop() - continue - } else { - // End of expansion for something internal. - return null - } - } - } - when (val currentExpr = expansionStack.peek().next()) { - Placeholder -> TODO("unreachable") - is MacroInvocation -> pushTdlMacroExpansion(currentExpr) - is EExpression -> pushEExpressionExpansion(currentExpr) - is VariableRef -> pushVariableExpansion(currentExpr) - is ExpressionGroup -> pushExpressionGroup(currentExpr) - is DataModelExpression -> { - this.currentExpr = currentExpr - break - } - } + class Pool(private val objectFactory: (Pool) -> T) { + private val availableElements = ArrayList(32) + private val allElements = IdentityHashMap(32) + private var acquireCount = 0 + private var releaseCount = 0 + fun acquire(init: (T) -> Unit): T { + val element = availableElements.removeLastOrNull() ?: objectFactory(this) + element.apply(init) + allElements[element] = 1 + // println("Pool(a=${++acquireCount},r=$releaseCount)") + if (acquireCount - releaseCount > 1000) throw IllegalStateException("Probable runtime stack overflow or memory leak") + return element } - return currentExpr - } - - /** - * Steps out of the current [DataModelContainer]. - */ - fun stepOut() { - // step out of anything we find until we have stepped out of a container. - while (expansionStack.pop()?.expansionKind != ExpansionKind.Container) { - if (expansionStack.isEmpty) throw IonException("Nothing to step out of.") + fun take(t: T) { + check(allElements[t] != 0) { "Double return!" } + if (allElements[t] == 1) { + availableElements.add(t) + allElements[t] = 0 + } + // println("Pool(a=$acquireCount,r=${++releaseCount})") } } +} - /** - * Steps in to the current [DataModelContainer]. - * Throws [IonException] if not positioned on a container. - */ - fun stepIn() { - val expression = requireNotNull(currentExpr) { "Not positioned on a value" } - expression as? DataModelContainer ?: throw IonException("Not positioned on a container.") - val currentExpansion = expansionStack.peek() - pushExpansion(ExpansionKind.Container, expression.startInclusive, expression.endExclusive, currentExpansion.environment!!, currentExpansion.expressions!!) - } - - /** - * Push a variable onto the expansion stack. - * - * Variables are a little bit different from other expansions. There is only one (top) expression - * in a variable expansion. It can be another variable, a value, a macro invocation, or an expression group. - * Furthermore, the current environment becomes the "source expressions" for the expansion, and the - * parent of the current environment becomes the environment in which the variable is expanded (thus - * maintaining the proper scope of variables). - */ - private fun pushVariableExpansion(expression: VariableRef) { - val currentEnvironment = expansionStack.peek().environment ?: Environment.EMPTY - val argumentExpressionIndex = currentEnvironment.argumentIndices[expression.signatureIndex] - - // Argument was elided; don't push anything so that we skip the empty expansion - if (argumentExpressionIndex < 0) return - - pushExpansion( - expansionKind = ExpansionKind.Values, - argsStartInclusive = argumentExpressionIndex, - // There can only be one expression for an argument. It's either a value, macro, or expression group. - argsEndExclusive = argumentExpressionIndex + 1, - environment = currentEnvironment.parentEnvironment ?: Environment.EMPTY, - expressions = currentEnvironment.arguments - ) - } - - private fun pushExpressionGroup(expr: ExpressionGroup) { - val currentExpansion = expansionStack.peek() - pushExpansion(ExpansionKind.Values, expr.startInclusive, expr.endExclusive, currentExpansion.environment!!, currentExpansion.expressions!!) - } - - /** - * Push a macro from a TDL macro invocation, found in the current expansion, to the expansion stack - */ - private fun pushTdlMacroExpansion(expression: MacroInvocation) { - val currentExpansion = expansionStack.peek() - pushMacro( - macro = expression.macro, - argsStartInclusive = expression.startInclusive, - argsEndExclusive = expression.endExclusive, - currentExpansion.environment!!, - encodingExpressions = currentExpansion.expressions!!, - ) - } - - /** - * Push a macro from the e-expression [expression] onto the expansionStack, handling concerns such as - * looking up the macro reference, setting up the environment, etc. - */ - private fun pushEExpressionExpansion(expression: EExpression) { - val currentExpansion = expansionStack.peek() - pushMacro( - macro = expression.macro, - argsStartInclusive = expression.startInclusive, - argsEndExclusive = expression.endExclusive, - environment = Environment.EMPTY, - encodingExpressions = currentExpansion.expressions!!, - ) - } - - /** - * Pushes a macro invocation to the expansionStack - */ - private fun pushMacro( - macro: Macro, - argsStartInclusive: Int, - argsEndExclusive: Int, - environment: Environment, - encodingExpressions: List, - ) { - val argIndices = calculateArgumentIndices(macro, encodingExpressions, argsStartInclusive, argsEndExclusive) - val templateBody = macro.body - if (templateBody == null) { - // If there's no template body, it must be a system macro. - macro as SystemMacro - val kind = ExpansionKind.forSystemMacro(macro) - pushExpansion(kind, argsStartInclusive, argsEndExclusive, environment, encodingExpressions) +/** + * Given a [Macro] (or more specifically, its signature), calculates the position of each of its arguments + * in [encodingExpressions]. The result is a list that can be used to map from a parameter's + * signature index to the encoding expression index. Any trailing, optional arguments that are + * elided have a value of -1. + * + * This function also validates that the correct number of parameters are present. If there are + * too many parameters or too few parameters, this will throw [IonException]. + */ +private fun Macro.calculateArgumentIndices( + encodingExpressions: List, + argsStartInclusive: Int, + argsEndExclusive: Int +): List { + // TODO: For TDL macro invocations, see if we can calculate this during the "compile" step. + var numArgs = 0 + val argsIndices = IntArray(signature.size) + var currentArgIndex = argsStartInclusive + + for (p in signature) { + if (currentArgIndex >= argsEndExclusive) { + if (!p.cardinality.canBeVoid) throw IonException("No value provided for parameter ${p.variableName}") + // Elided rest parameter. + argsIndices[numArgs] = -1 } else { - pushExpansion( - ExpansionKind.TemplateBody, - argsStartInclusive = 0, - argsEndExclusive = templateBody.size, - expressions = templateBody, - environment = environment.createChild(encodingExpressions, argIndices) - ) - } - } - - /** - * Pushes an expansion to the expansion stack. - */ - private fun pushExpansion( - expansionKind: ExpansionKind, - argsStartInclusive: Int, - argsEndExclusive: Int, - environment: Environment, - expressions: List, - ) { - expansionStack.push { - it.expansionKind = expansionKind - it.environment = environment - it.expressions = expressions - it.i = argsStartInclusive - it.endExclusive = argsEndExclusive - it.additionalState = null - } - } - - /** - * Given a [Macro] (or more specifically, its signature), calculates the position of each of its arguments - * in [encodingExpressions]. The result is a list that can be used to map from a parameter's - * signature index to the encoding expression index. Any trailing, optional arguments that are - * elided have a value of -1. - * - * This function also validates that the correct number of parameters are present. If there are - * too many parameters or too few parameters, this will throw [IonException]. - */ - private fun calculateArgumentIndices( - macro: Macro, - encodingExpressions: List, - argsStartInclusive: Int, - argsEndExclusive: Int - ): List { - // TODO: For TDL macro invocations, see if we can calculate this during the "compile" step. - var numArgs = 0 - val argsIndices = IntArray(macro.signature.size) - var currentArgIndex = argsStartInclusive - for (p in macro.signature) { - if (currentArgIndex >= argsEndExclusive) { - if (!p.cardinality.canBeVoid) throw IonException("No value provided for parameter ${p.variableName}") - // Elided rest parameter. - argsIndices[numArgs] = -1 - } else { - argsIndices[numArgs] = currentArgIndex - currentArgIndex = when (val expr = encodingExpressions[currentArgIndex]) { - is HasStartAndEnd -> expr.endExclusive - else -> currentArgIndex + 1 - } - } - numArgs++ - } - while (currentArgIndex < argsEndExclusive) { + argsIndices[numArgs] = currentArgIndex currentArgIndex = when (val expr = encodingExpressions[currentArgIndex]) { is HasStartAndEnd -> expr.endExclusive else -> currentArgIndex + 1 } - numArgs++ } - if (numArgs > macro.signature.size) { - throw IonException("Too many arguments. Expected ${macro.signature.size}, but found $numArgs") + numArgs++ + } + while (currentArgIndex < argsEndExclusive) { + currentArgIndex = when (val expr = encodingExpressions[currentArgIndex]) { + is HasStartAndEnd -> expr.endExclusive + else -> currentArgIndex + 1 } - return argsIndices.toList() + numArgs++ + } + if (numArgs > signature.size) { + throw IonException("Too many arguments. Expected ${signature.size}, but found $numArgs") } + return argsIndices.toList() } diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt index 4945e5c47..30d355c1d 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt @@ -35,10 +35,22 @@ class MacroEvaluatorAsIonReader( private fun queueNext() { queuedValueExpression = null while (queuedValueExpression == null) { - val nextCandidate = evaluator.expandNext() ?: return + val nextCandidate = evaluator.expandNext() when (nextCandidate) { + null -> { + queuedFieldName = null + return + } is Expression.FieldName -> queuedFieldName = nextCandidate is Expression.DataModelValue -> queuedValueExpression = nextCandidate + Expression.EndOfContainer -> { + queuedFieldName = null + return + } + Expression.EndOfExpansion -> { + queuedFieldName = null + return + } } } } diff --git a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt index eaa7e89dc..4b0d3e37c 100644 --- a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt @@ -15,7 +15,7 @@ import com.amazon.ion.impl.macro.ParameterFactory.zeroToManyTagged */ enum class SystemMacro( val id: Byte, - val systemSymbol: SystemSymbols_1_1, + val systemSymbol: SystemSymbols_1_1?, override val signature: List, override val body: List? = null ) : Macro { @@ -26,9 +26,13 @@ enum class SystemMacro( IfSingle(-1, IF_SINGLE, listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), IfMulti(-1, IF_MULTI, listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), + // Unnameable, unaddressable macros used for the internals of certain other system macros + FlattenStruct(-1, systemSymbol = null, listOf(zeroToManyTagged("structs"))), + MakeFieldNameAndValue(-1, systemSymbol = null, listOf(exactlyOneTagged("fieldName"), exactlyOneTagged("value"))), + // The real macros - None(0, NONE, emptyList()), - Values(1, VALUES, listOf(zeroToManyTagged("values"))), + Values(1, VALUES, listOf(zeroToManyTagged("values")), templateBody { variable(0) }), + None(0, NONE, emptyList(), templateBody { macro(Values) { expressionGroup { } } }), Default( 2, DEFAULT, listOf(zeroToManyTagged("expr"), zeroToManyTagged("default_expr")), templateBody { @@ -37,7 +41,7 @@ enum class SystemMacro( ), Meta(3, META, listOf(zeroToManyTagged("values")), templateBody { macro(None) {} }), Repeat(4, REPEAT, listOf(exactlyOneTagged("n"), zeroToManyTagged("value"))), - Flatten(5, FLATTEN, listOf(zeroToManyTagged("values")), null), // TODO: flatten + Flatten(5, FLATTEN, listOf(zeroToManyTagged("values"))), Delta(6, DELTA, listOf(zeroToManyTagged("deltas"))), Sum(7, SUM, listOf(exactlyOneTagged("a"), exactlyOneTagged("b"))), @@ -58,16 +62,49 @@ enum class SystemMacro( ) ), MakeBlob(13, MAKE_BLOB, listOf(zeroToManyTagged("bytes"))), - MakeList(14, MAKE_LIST, listOf(zeroToManyTagged("sequences")), null), // TODO: make_list - MakeSExp(15, MAKE_SEXP, listOf(zeroToManyTagged("sequences")), null), // TODO: make_sexp + MakeList( + 14, MAKE_LIST, listOf(zeroToManyTagged("sequences")), templateBody { + list { + macro(Flatten) { + variable(0) + } + } + } + ), + MakeSExp( + 15, MAKE_SEXP, listOf(zeroToManyTagged("sequences")), templateBody { + sexp { + macro(Flatten) { + variable(0) + } + } + } + ), + MakeField( - 16, MAKE_FIELD, - listOf( - Macro.Parameter("field_name", Macro.ParameterEncoding.FlexSym, Macro.ParameterCardinality.ExactlyOne), exactlyOneTagged("value") - ) + 16, MAKE_FIELD, listOf(exactlyOneTagged("fieldName"), exactlyOneTagged("value")), + templateBody { + struct { + macro(MakeFieldNameAndValue) { + variable(0) + variable(1) + } + } + } ), - MakeStruct(17, MAKE_STRUCT, listOf(zeroToManyTagged("structs")), null), // TODO: make_struct - ParseIon(18, PARSE_ION, listOf(zeroToManyTagged("data")), null), // TODO: parse_ion + + MakeStruct( + 17, MAKE_STRUCT, listOf(zeroToManyTagged("structs")), + templateBody { + struct { + macro(FlattenStruct) { + variable(0) + } + } + } + ), + ParseIon(18, PARSE_ION, listOf(zeroToManyTagged("data"))), // TODO: parse_ion + /** * ```ion @@ -222,7 +259,7 @@ enum class SystemMacro( ), ; - val macroName: String get() = this.systemSymbol.text + val macroName: String get() = this.systemSymbol?.text ?: throw IllegalStateException("Attempt to get name for unaddressable macro $name") override val dependencies: List get() = body @@ -233,7 +270,9 @@ enum class SystemMacro( companion object : MacroTable { - private val MACROS_BY_NAME: Map = SystemMacro.entries.associateBy { it.macroName } + private val MACROS_BY_NAME: Map = SystemMacro.entries + .filter { it.systemSymbol != null } + .associateBy { it.macroName } // TODO: Once all of the macros are implemented, replace this with an array as in SystemSymbols_1_1 private val MACROS_BY_ID: Map = SystemMacro.entries diff --git a/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt b/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt index a96f8f8f0..108d2234c 100644 --- a/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt +++ b/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt @@ -99,15 +99,8 @@ abstract class ConformanceTestRunner( // TODO: Not implemented yet "subnormal f16" in completeTestName -> false - "conformance/system_macros/" in file.absolutePath -> when { - file.endsWith("parse_ion.ion") || - file.endsWith("make_list.ion") || - file.endsWith("make_sexp.ion") || - file.endsWith("make_field.ion") || - file.endsWith("flatten.ion") || - file.endsWith("make_struct.ion") -> false - else -> true - } + "conformance/system_macros/parse_ion.ion" in file.absolutePath -> false + // Some of these are failing because // - Ion Java doesn't support the Ion 1.1 system symbol table yet // - The tokens `$ion_1_0` and `'$ion_1_0'` are never user values. diff --git a/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt b/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt index 66b15cbd6..19124b691 100644 --- a/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt +++ b/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt @@ -5,6 +5,7 @@ package com.amazon.ion.impl import com.amazon.ion.* import com.amazon.ion.impl.macro.* import com.amazon.ion.system.* +import java.lang.AssertionError import java.math.BigDecimal import java.math.BigInteger import org.junit.jupiter.api.Assertions.assertEquals @@ -15,6 +16,7 @@ import org.junit.jupiter.api.assertThrows import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.CsvSource import org.junit.jupiter.params.provider.EnumSource +import org.opentest4j.TestAbortedException class IonRawTextWriterTest_1_1 { @@ -712,6 +714,7 @@ class IonRawTextWriterTest_1_1 { @ParameterizedTest @EnumSource(SystemMacro::class) fun `write system macro E-expression by name`(systemMacro: SystemMacro) { + if (systemMacro.systemSymbol == null) throw TestAbortedException("Skip this test for unaddressable macros") assertWriterOutputEquals("(:\$ion::${systemMacro.macroName})") { stepInEExp(systemMacro) stepOut() diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt index 4e7c07112..f993bf5a3 100644 --- a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt +++ b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt @@ -55,6 +55,47 @@ class MacroEvaluatorTest { val evaluator = MacroEvaluator() + @Test + fun `data with just scalars`() { + evaluator.initExpansion { + int(1) + int(2) + int(3) + } + + evaluator.assertExpansion("1 2 3") + } + + @Test + fun `data with a list`() { + evaluator.initExpansion { + list { + int(1) + list { + int(2) + int(3) + } + string("a") + } + } + evaluator.assertExpansion("""[1, [2, 3], "a"] """) + } + + @Test + fun `data with a struct`() { + + evaluator.initExpansion { + struct { + fieldName("a") + int(1) + fieldName("b") + int(2) + } + } + + evaluator.assertExpansion("{a:1, b:2}") + } + @Test fun `the 'none' system macro`() { // Given: @@ -244,7 +285,8 @@ class MacroEvaluatorTest { } } - assertEquals(BoolValue(emptyList(), true), evaluator.expandNext()) + val actual = evaluator.expandNext() + assertEquals(BoolValue(emptyList(), true), actual) assertEquals(null, evaluator.expandNext()) } @@ -328,20 +370,24 @@ class MacroEvaluatorTest { fun `invoke values with scalars`() { // Given: // When: - // (:values 1 "a") + // (:values 1 2 3 "a") // Then: - // 1 "a" + // 1 2 3 "a" evaluator.initExpansion { eexp(Values) { expressionGroup { int(1) + int(2) + int(3) string("a") } } } assertEquals(LongIntValue(emptyList(), 1), evaluator.expandNext()) + assertEquals(LongIntValue(emptyList(), 2), evaluator.expandNext()) + assertEquals(LongIntValue(emptyList(), 3), evaluator.expandNext()) assertEquals(StringValue(emptyList(), "a"), evaluator.expandNext()) assertEquals(null, evaluator.expandNext()) } @@ -386,7 +432,9 @@ class MacroEvaluatorTest { } evaluator.initExpansion { - eexp(voidableIdentityMacro) {} + eexp(voidableIdentityMacro) { + expressionGroup { } + } } assertEquals(null, evaluator.expandNext()) @@ -560,9 +608,13 @@ class MacroEvaluatorTest { } } + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() assertEquals(FieldName(value = newSymbolToken("foo")), evaluator.expandNext()) assertEquals(LongIntValue(value = 1), evaluator.expandNext()) assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) } @Test @@ -1003,11 +1055,15 @@ class MacroEvaluatorTest { evaluator.initExpansion { eexp(Repeat) { - int(2) - int(0) + int(4) + expressionGroup { + int(0) + } } } + assertEquals(LongIntValue(value = 0), evaluator.expandNext()) + assertEquals(LongIntValue(value = 0), evaluator.expandNext()) assertEquals(LongIntValue(value = 0), evaluator.expandNext()) assertEquals(LongIntValue(value = 0), evaluator.expandNext()) assertEquals(null, evaluator.expandNext()) From fbef69b3dd07ed84499129d61cd1cc2eaa93e69e Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Mon, 16 Dec 2024 15:28:52 -0800 Subject: [PATCH 02/10] Working rewrite of MacroEvaluator --- ion-tests | 2 +- .../com/amazon/ion/impl/macro/Environment.kt | 19 +- .../com/amazon/ion/impl/macro/Expression.kt | 39 +- .../ion/impl/macro/ExpressionBuilderDsl.kt | 4 +- .../amazon/ion/impl/macro/MacroCompiler.kt | 2 +- .../amazon/ion/impl/macro/MacroEvaluator.kt | 843 +++++++++--------- .../impl/macro/MacroEvaluatorAsIonReader.kt | 4 - .../com/amazon/ion/impl/macro/SystemMacro.kt | 21 +- .../java/com/amazon/ion/util/Assumptions.kt | 2 + .../ion/conformance/ConformanceTestRunner.kt | 6 + .../com/amazon/ion/conformance/structure.kt | 4 +- .../ion/impl/IonRawTextWriterTest_1_1.kt | 1 - .../ion/impl/macro/MacroEvaluatorTest.kt | 2 +- 13 files changed, 479 insertions(+), 470 deletions(-) diff --git a/ion-tests b/ion-tests index c2aca0161..89a6fd4c9 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit c2aca0161515ba8b153c0d949c882705306cf67e +Subproject commit 89a6fd4c9a1eae3b90457ea77758dc8cb8219a26 diff --git a/src/main/java/com/amazon/ion/impl/macro/Environment.kt b/src/main/java/com/amazon/ion/impl/macro/Environment.kt index dfbd05f75..8b0478e4c 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Environment.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Environment.kt @@ -18,13 +18,26 @@ data class Environment private constructor( val arguments: List, // TODO: Replace with IntArray val argumentIndices: List, + val argumentsByName: Map, val parentEnvironment: Environment?, ) { - fun createChild(arguments: List, argumentIndices: List) = Environment(arguments, argumentIndices, this) + fun createChild(arguments: List, argumentIndices: List, byName: Map) = Environment(arguments, argumentIndices, byName, this) + + override fun toString() = """ + |Environment( + | argumentIndices: $argumentIndices, + | argumentsByName: [${argumentsByName.map { (name, index) -> "\n| $name -> $index" }.joinToString() } + | ], + | argumentExpressions: [${arguments.mapIndexed { index, expression -> "\n| $index. $expression" }.joinToString() } + | ], + | parent: ${parentEnvironment.toString().lines().joinToString("\n| ")}, + |) + """.trimMargin() + companion object { @JvmStatic - val EMPTY = Environment(emptyList(), emptyList(), null) + val EMPTY = Environment(emptyList(), emptyList(), emptyMap(), null) @JvmStatic - fun create(arguments: List, argumentIndices: List) = Environment(arguments, argumentIndices, null) + fun create(arguments: List, argumentIndices: List, byName: Map) = Environment(arguments, argumentIndices, byName, null) } } diff --git a/src/main/java/com/amazon/ion/impl/macro/Expression.kt b/src/main/java/com/amazon/ion/impl/macro/Expression.kt index 7e3d95d1b..12d5d27c4 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Expression.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Expression.kt @@ -54,10 +54,20 @@ sealed interface Expression { */ sealed interface DataModelExpression : Expression, EExpressionBodyExpression, TemplateBodyExpression, ExpansionOutputExpression + /** Output of a macro expansion (internal to the macro evaluator) */ sealed interface ExpansionOutputExpressionOrContinue - + /** Output of the macro evaluator */ sealed interface ExpansionOutputExpression : ExpansionOutputExpressionOrContinue + /** + * Indicates to the macro evaluator that the current expansion did not produce a value this time, but it may + * produce more expressions. The macro evaluator should request another expression from that macro. + */ + data object ContinueExpansion : ExpansionOutputExpressionOrContinue + + /** Signals the end of an expansion in the macro evaluator. */ + data object EndOfExpansion : ExpansionOutputExpression + /** * Interface for expressions that are _values_ in the Ion data model. */ @@ -69,15 +79,7 @@ sealed interface Expression { } /** Expressions that represent Ion container types */ - sealed interface DataModelContainer : HasStartAndEnd, DataModelValue { - val isConstructedFromMacro: Boolean - } - - data object ContinueExpansion : ExpansionOutputExpressionOrContinue - data object EndOfExpansion : ExpansionOutputExpression - - // TODO: See if we can remove this - data object EndOfContainer : ExpansionOutputExpression // , DataModelExpression + sealed interface DataModelContainer : HasStartAndEnd, DataModelValue /** * A temporary placeholder that is used only while a macro or e-expression is partially compiled. @@ -198,7 +200,6 @@ sealed interface Expression { override val annotations: List = emptyList(), override val selfIndex: Int, override val endExclusive: Int, - override val isConstructedFromMacro: Boolean = false, ) : DataModelContainer { override val type: IonType get() = IonType.LIST override fun withAnnotations(annotations: List) = copy(annotations = annotations) @@ -211,7 +212,6 @@ sealed interface Expression { override val annotations: List = emptyList(), override val selfIndex: Int, override val endExclusive: Int, - override val isConstructedFromMacro: Boolean = false, ) : DataModelContainer { override val type: IonType get() = IonType.SEXP override fun withAnnotations(annotations: List) = copy(annotations = annotations) @@ -225,7 +225,6 @@ sealed interface Expression { override val selfIndex: Int, override val endExclusive: Int, val templateStructIndex: Map> = emptyMap(), - override val isConstructedFromMacro: Boolean = false, ) : DataModelContainer { override val type: IonType get() = IonType.STRUCT override fun withAnnotations(annotations: List) = copy(annotations = annotations) @@ -236,23 +235,27 @@ sealed interface Expression { /** * A reference to a variable that needs to be expanded. */ - data class VariableRef(val signatureIndex: Int) : TemplateBodyExpression + data class VariableRef @JvmOverloads constructor(val signatureIndex: Int, val parameter: Macro.Parameter? = null) : TemplateBodyExpression + + sealed interface InvokableExpression : HasStartAndEnd, Expression { + val macro: Macro + } /** * A macro invocation that needs to be expanded. */ data class MacroInvocation( - val macro: Macro, + override val macro: Macro, override val selfIndex: Int, override val endExclusive: Int - ) : TemplateBodyExpression, HasStartAndEnd + ) : TemplateBodyExpression, HasStartAndEnd, InvokableExpression /** * An e-expression that needs to be expanded. */ data class EExpression( - val macro: Macro, + override val macro: Macro, override val selfIndex: Int, override val endExclusive: Int - ) : EExpressionBodyExpression, HasStartAndEnd + ) : EExpressionBodyExpression, HasStartAndEnd, InvokableExpression } diff --git a/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt b/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt index 7a1756361..b67565a66 100644 --- a/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt +++ b/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt @@ -57,7 +57,7 @@ internal interface DataModelDsl : ValuesDsl { @ExpressionBuilderDslMarker internal interface TemplateDsl : ValuesDsl { fun macro(macro: Macro, arguments: InvocationBody.() -> Unit) - fun variable(signatureIndex: Int) + fun variable(signatureIndex: Int, parameter: Macro.Parameter? = null) fun list(content: TemplateDsl.() -> Unit) fun sexp(content: TemplateDsl.() -> Unit) fun struct(content: Fields.() -> Unit) @@ -186,7 +186,7 @@ internal sealed class ExpressionBuilderDsl : ValuesDsl, ValuesDsl.Fields { override fun list(content: TemplateDsl.() -> Unit) = containerWithAnnotations(content, ::ListValue) override fun sexp(content: TemplateDsl.() -> Unit) = containerWithAnnotations(content, ::SExpValue) override fun struct(content: TemplateDsl.Fields.() -> Unit) = containerWithAnnotations(content, ::newStruct) - override fun variable(signatureIndex: Int) { expressions.add(VariableRef(signatureIndex)) } + override fun variable(signatureIndex: Int, parameter: Macro.Parameter?) { expressions.add(VariableRef(signatureIndex, parameter)) } override fun macro(macro: Macro, arguments: TemplateDsl.InvocationBody.() -> Unit) = container(arguments) { start, end -> MacroInvocation(macro, start, end) } override fun expressionGroup(content: TemplateDsl.() -> Unit) = container(content, ::ExpressionGroup) } diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt b/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt index f64da15a6..3f67e9146 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt @@ -269,7 +269,7 @@ internal class MacroCompiler( confirmNoAnnotations("on variable reference '$name'") val index = signature.indexOfFirst { it.variableName == name } confirm(index >= 0) { "variable '$name' is not recognized" } - expressions[placeholderIndex] = VariableRef(index) + expressions[placeholderIndex] = VariableRef(index, parameter = signature[index]) confirm(!nextValue()) { "Variable expansion should contain only the variable name." } stepOutOfContainer() } diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index 41cbf8e1d..4c7c2091a 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -4,39 +4,14 @@ package com.amazon.ion.impl.macro import com.amazon.ion.* import com.amazon.ion.impl._Private_RecyclingStack -import com.amazon.ion.impl._Private_Utils import com.amazon.ion.impl._Private_Utils.* import com.amazon.ion.impl.macro.Expression.* import com.amazon.ion.impl.macro.MacroEvaluator.ExpanderKind.* +import com.amazon.ion.util.* import java.io.ByteArrayOutputStream import java.lang.StringBuilder import java.math.BigDecimal import java.math.BigInteger -import java.util.IdentityHashMap - -private fun getExpanderKindForSystemMacro(systemMacro: SystemMacro) = when (systemMacro) { - SystemMacro.Annotate -> Annotate - SystemMacro.MakeString -> MakeString - SystemMacro.MakeSymbol -> MakeSymbol - SystemMacro.MakeDecimal -> MakeDecimal - SystemMacro.Repeat -> Repeat - SystemMacro.Sum -> Sum - SystemMacro.Delta -> Delta - SystemMacro.MakeBlob -> MakeBlob - SystemMacro.Flatten -> Flatten - SystemMacro.FlattenStruct -> FlattenStruct - SystemMacro.MakeTimestamp -> MakeTimestamp - SystemMacro.MakeFieldNameAndValue -> MakeFieldNameAndValue - SystemMacro.IfNone -> IfNone - SystemMacro.IfSome -> IfSome - SystemMacro.IfSingle -> IfSingle - SystemMacro.IfMulti -> IfMulti - else -> if (systemMacro.body != null) { - throw IllegalStateException("SystemMacro ${systemMacro.name} should be using its template body.") - } else { - TODO("Not implemented yet: ${systemMacro.name}") - } -} /** * Evaluates an EExpression from a List of [EExpressionBodyExpression] and the [TemplateBodyExpression]s @@ -48,30 +23,74 @@ private fun getExpanderKindForSystemMacro(systemMacro: SystemMacro) = when (syst * if the end of the container or end of expansion has been reached. * - Call [stepIn] when positioned on a container to step into that container. * - Call [stepOut] to step out of the current container. - * - * TODO: Add expansion limit */ class MacroEvaluator { - // TODO: - data class ContainerInfo(var type: Type = Type.Uninitialized, private var _expansion: Expansion? = null) { + private var numExpandedExpressions = 0 + private val expansionLimit: Int = 1_000_000 + private val expanderPool: ArrayList = ArrayList(32) + + // TODO(PERF): Does it improve performance if we make this an `inner` class and remove the evaluator field? + class MacroEvaluationSession(private val evaluator: MacroEvaluator) { + + fun getExpander( + expanderKind: ExpanderKind, + expressions: List, + startInclusive: Int, + endExclusive: Int, + environment: Environment, + ): ExpansionFrame { + val expansion = evaluator.expanderPool.removeLastOrNull() ?: ExpansionFrame(this) + expansion.expanderKind = expanderKind + expansion.expressions = expressions + expansion.i = startInclusive + expansion.endExclusive = endExclusive + expansion.environment = environment + expansion.additionalState = null + expansion.expansionDelegate = null + return expansion + } + + fun returnExpander(ex: ExpansionFrame) { + // TODO: This check is O(n). Remove this when confident. + check(ex !in evaluator.expanderPool) + evaluator.expanderPool.add(ex) + } + + fun incrementStepCounter() { + evaluator.numExpandedExpressions++ + if (evaluator.numExpandedExpressions > evaluator.expansionLimit) { + // Technically, we are not counting "steps" because we don't have a true definition of what a "step" is, + // but this is probably a more user-friendly message than trying to explain what we're actually counting. + throw IonException("Macro expansion exceeded limit of ${evaluator.expansionLimit} steps.") + } + } + } + + private data class ContainerInfo(var type: Type = Type.Uninitialized, private var _expansion: ExpansionFrame? = null) { enum class Type { TopLevel, List, Sexp, Struct, Uninitialized } - fun release() { - _expansion?.release() + fun releaseResources() { + _expansion?.drop() _expansion = null type = Type.Uninitialized } - var expansion: Expansion + var expansion: ExpansionFrame get() = _expansion!! set(value) { _expansion = value } + + fun produceNext(): ExpansionOutputExpression { + return expansion.produceNext() + } } - private val expansionPool = Pool { pool -> Expansion(pool) } + private val session = MacroEvaluationSession(this) private val containerStack = _Private_RecyclingStack(8) { ContainerInfo() } private var currentExpr: DataModelExpression? = null + private fun resetSession() { this.numExpandedExpressions = 0 } + fun getArguments(): List { return containerStack.iterator().next().expansion.expressions } @@ -80,11 +99,10 @@ class MacroEvaluator { * Initialize the macro evaluator with an E-Expression. */ fun initExpansion(encodingExpressions: List) { + resetSession() containerStack.push { ci -> ci.type = ContainerInfo.Type.TopLevel - ci.expansion = expansionPool.acquire { - it.initExpansion(Stream, encodingExpressions, 0, encodingExpressions.size, Environment.EMPTY) - } + ci.expansion = session.getExpander(Stream, encodingExpressions, 0, encodingExpressions.size, Environment.EMPTY) } } @@ -97,21 +115,15 @@ class MacroEvaluator { while (currentExpr == null && !containerStack.isEmpty()) { val currentContainer = containerStack.peek() - val nextExpansionOutput = currentContainer.expansion.produceNext() - when (nextExpansionOutput) { + when (val nextExpansionOutput = currentContainer.produceNext()) { is DataModelExpression -> currentExpr = nextExpansionOutput EndOfExpansion -> { - // TODO: Do we need to release anything? - // TODO: Is there a better way to do this? if (currentContainer.type == ContainerInfo.Type.TopLevel) { - currentContainer.release() + currentContainer.releaseResources() containerStack.pop() } return null } - EndOfContainer -> { - return null - } } } return currentExpr @@ -121,10 +133,10 @@ class MacroEvaluator { * Steps out of the current [DataModelContainer]. */ fun stepOut() { - // step out of anything we find until we have stepped out of a container. + // TODO: We should be able to step out of a "TopLevel" container and/or we need some way to close the evaluation. if (containerStack.size() <= 1) throw IonException("Nothing to step out of.") val popped = containerStack.pop() - popped.release() + popped.releaseResources() } /** @@ -135,27 +147,21 @@ class MacroEvaluator { val expression = requireNotNull(currentExpr) { "Not positioned on a value" } if (expression is DataModelContainer) { val currentContainer = containerStack.peek() - if (expression.isConstructedFromMacro) { - val currentTop = currentContainer.expansion.top() - } else { - containerStack.push { ci -> - ci.type = when (expression.type) { - IonType.LIST -> ContainerInfo.Type.List - IonType.SEXP -> ContainerInfo.Type.Sexp - IonType.STRUCT -> ContainerInfo.Type.Struct - else -> TODO("Unreachable") - } - ci.expansion = expansionPool.acquire { - val topExpansion = currentContainer.expansion.top() - it.initExpansion( - expanderKind = Stream, - expressions = topExpansion.expressions, - startInclusive = expression.startInclusive, - endExclusive = expression.endExclusive, - environment = topExpansion.environment!!, - ) - } + val topExpansion = currentContainer.expansion.top() + containerStack.push { ci -> + ci.type = when (expression.type) { + IonType.LIST -> ContainerInfo.Type.List + IonType.SEXP -> ContainerInfo.Type.Sexp + IonType.STRUCT -> ContainerInfo.Type.Struct + else -> unreachable() } + ci.expansion = session.getExpander( + expanderKind = Stream, + expressions = topExpansion.expressions, + startInclusive = expression.startInclusive, + endExclusive = expression.endExclusive, + environment = topExpansion.environment, + ) } currentExpr = null } else { @@ -163,318 +169,229 @@ class MacroEvaluator { } } + // TODO(PERF): It might be possible to optimize this by changing it to an enum without any methods (or even a set of + // integer constants) and converting all their implementations to static methods. enum class ExpanderKind { Uninitialized { - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { throw IllegalStateException("ExpansionInfo not initialized.") } }, Empty { - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue = EndOfExpansion + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue = EndOfExpansion }, Stream { - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - val self = expansion + private fun invokeMacro(thisExpansion: ExpansionFrame, macro: Macro, next: HasStartAndEnd): ContinueExpansion { + val argIndices = macro.calculateArgumentIndices( + encodingExpressions = thisExpansion.expressions, + argsStartInclusive = next.startInclusive, + argsEndExclusive = next.endExclusive, + ) + val argsIndicesByName = macro.calculateArgumentIndicesByName(thisExpansion.expressions, next.startInclusive, next.endExclusive) + val newEnvironment = thisExpansion.environment.createChild(thisExpansion.expressions, argIndices, argsIndicesByName) + val expanderKind = if (macro.body != null) Stream else getExpanderKindForSystemMacro(macro as SystemMacro) + thisExpansion.expansionDelegate = thisExpansion.session.getExpander( + expanderKind = expanderKind, + expressions = macro.body ?: emptyList(), + startInclusive = 0, + endExclusive = macro.body?.size ?: 0, + environment = newEnvironment, + ) + return ContinueExpansion + } + + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { // If there's a delegate, we'll try that first. - val delegate = self.expansionDelegate + val delegate = thisExpansion.expansionDelegate + check(thisExpansion != delegate) if (delegate != null) { - val result = delegate.produceNext() - return when (result) { + return when (val result = delegate.produceNext()) { is DataModelExpression -> result - // TODO: figure out some way to stick on this... or maybe it's not necessary. - // Test this by attempting to go beyond the end of containers. - EndOfContainer -> EndOfContainer EndOfExpansion -> { - delegate.release() - self.expansionDelegate = null + delegate.drop() + thisExpansion.expansionDelegate = null ContinueExpansion } } } - if (self.i >= self.endExclusive) { - expansion.expanderKind = Empty + if (thisExpansion.i >= thisExpansion.endExclusive) { + thisExpansion.expanderKind = Empty return ContinueExpansion } - val next = self.expressions[self.i] - self.i++ - if (next is HasStartAndEnd) self.i = next.endExclusive + val next = thisExpansion.expressions[thisExpansion.i] + thisExpansion.i++ + if (next is HasStartAndEnd) thisExpansion.i = next.endExclusive return when (next) { is DataModelExpression -> next - is EExpression -> { - val macro = next.macro - val argIndices = macro.calculateArgumentIndices( - encodingExpressions = expansion.expressions, - argsStartInclusive = next.startInclusive, - argsEndExclusive = next.endExclusive, - ) - val newEnvironment = self.environment.createChild(self.expressions, argIndices) - if (macro.body != null) { - self.expansionDelegate = self.expansionPool.acquire { new -> - new.initExpansion( - expanderKind = Stream, - expressions = macro.body!!, - startInclusive = 0, - endExclusive = macro.body!!.size, - environment = newEnvironment, - ) - } - } else { - val expanderKind = getExpanderKindForSystemMacro(macro as SystemMacro) - self.expansionDelegate = self.expansionPool.acquire { new -> - new.initExpansion( - expanderKind = expanderKind, - expressions = emptyList(), - startInclusive = 0, - endExclusive = 0, - environment = newEnvironment, - ) - } - } - ContinueExpansion - } - is MacroInvocation -> { - // TODO: Verify if this is correct - val macro = next.macro - val argIndices = macro.calculateArgumentIndices( - encodingExpressions = expansion.expressions, - argsStartInclusive = next.startInclusive, - argsEndExclusive = next.endExclusive, - ) - val newEnvironment = self.environment.createChild(self.expressions, argIndices) - if (macro.body != null) { - self.expansionDelegate = self.expansionPool.acquire { new -> - new.initExpansion( - expanderKind = Stream, - expressions = macro.body!!, - startInclusive = 0, - endExclusive = macro.body!!.size, - environment = newEnvironment, - ) - } - } else { - val expanderKind = getExpanderKindForSystemMacro(macro as SystemMacro) - self.expansionDelegate = self.expansionPool.acquire { new -> - new.initExpansion( - expanderKind = expanderKind, - expressions = emptyList(), - startInclusive = 0, - endExclusive = 0, - environment = newEnvironment, - ) - } - } - ContinueExpansion - } + is EExpression -> invokeMacro(thisExpansion, next.macro, next) + is MacroInvocation -> invokeMacro(thisExpansion, next.macro, next) is ExpressionGroup -> { - self.expansionDelegate = self.expansionPool.acquire { new -> - new.initExpansion( - expanderKind = Stream, - expressions = self.expressions, - startInclusive = next.startInclusive, - endExclusive = next.endExclusive, - environment = self.environment, - ) - } + thisExpansion.expansionDelegate = thisExpansion.session.getExpander( + expanderKind = Stream, + expressions = thisExpansion.expressions, + startInclusive = next.startInclusive, + endExclusive = next.endExclusive, + environment = thisExpansion.environment, + ) + ContinueExpansion } is VariableRef -> { - self.expansionDelegate = self.readArgument(next) + thisExpansion.expansionDelegate = thisExpansion.readArgument(next) ContinueExpansion } - - Placeholder -> TODO("Unreachable") + Placeholder -> unreachable() } } }, - OneValuedStream { - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - if (expansion.additionalState != 1) { - return when (val firstValue = Stream.produceNext(expansion)) { + // TODO: Move this into the variable expansion? + ExactlyOneValueStream { + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + if (thisExpansion.additionalState != 1) { + return when (val firstValue = Stream.produceNext(thisExpansion)) { is DataModelExpression -> { - expansion.additionalState = 1 + thisExpansion.additionalState = 1 firstValue } ContinueExpansion -> ContinueExpansion EndOfExpansion -> throw IonException("Expected one value, found 0") - EndOfContainer -> TODO("Unused?") } } else { - return when (val secondValue = Stream.produceNext(expansion)) { + return when (val secondValue = Stream.produceNext(thisExpansion)) { is DataModelExpression -> throw IonException("Expected one value, found multiple") ContinueExpansion -> ContinueExpansion EndOfExpansion -> secondValue - EndOfContainer -> TODO("Unused?") } } } }, - IfNone { - private val ARG_TO_TEST = VariableRef(0) - private val TRUE_BRANCH = VariableRef(1) - private val FALSE_BRANCH = VariableRef(2) - - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - val testArg = expansion.readArgument(ARG_TO_TEST) - var n = 0 - while (n < 2) { - if (testArg.produceNext() is EndOfExpansion) break - n++ + NonEmptyStream { + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + return when (val firstValue = Stream.produceNext(thisExpansion)) { + EndOfExpansion -> throw IonException("Expected at least one value, found 0") + ContinueExpansion -> ContinueExpansion + is DataModelExpression -> { + thisExpansion.expanderKind = Stream + firstValue + } } - - val branch = if (n > 0) FALSE_BRANCH else TRUE_BRANCH - val branchExpansion = expansion.readArgument(branch) - expansion.reInitializeFrom(branchExpansion) - branchExpansion.release() - testArg.release() - return ContinueExpansion } }, - IfSome { - private val ARG_TO_TEST = VariableRef(0) - private val TRUE_BRANCH = VariableRef(1) - private val FALSE_BRANCH = VariableRef(2) - - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - val testArg = expansion.readArgument(ARG_TO_TEST) - var n = 0 - while (n < 2) { - if (testArg.produceNext() is EndOfExpansion) break - n++ + AtMostOneValueStream { + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + if (thisExpansion.additionalState != 1) { + return when (val firstValue = Stream.produceNext(thisExpansion)) { + is DataModelExpression -> { + thisExpansion.additionalState = 1 + firstValue + } + ContinueExpansion -> ContinueExpansion + EndOfExpansion -> EndOfExpansion + } + } else { + return when (val secondValue = Stream.produceNext(thisExpansion)) { + is DataModelExpression -> throw IonException("Expected one value, found multiple") + ContinueExpansion -> ContinueExpansion + EndOfExpansion -> secondValue + } } - testArg.release() - - val branch = if (n > 0) TRUE_BRANCH else FALSE_BRANCH - val branchExpansion = expansion.readArgument(branch) - expansion.reInitializeFrom(branchExpansion) - branchExpansion.release() - return ContinueExpansion } }, - IfSingle { - private val ARG_TO_TEST = VariableRef(0) - private val TRUE_BRANCH = VariableRef(1) - private val FALSE_BRANCH = VariableRef(2) - - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - val testArg = expansion.readArgument(ARG_TO_TEST) - var n = 0 - while (n < 2) { - if (testArg.produceNext() is EndOfExpansion) break - n++ - } - testArg.release() - val branch = if (n == 1) TRUE_BRANCH else FALSE_BRANCH - val branchExpansion = expansion.readArgument(branch) - expansion.reInitializeFrom(branchExpansion) - branchExpansion.release() - return ContinueExpansion - } + IfNone { + override fun produceNext(thisExpansion: ExpansionFrame) = checkExpansionSize(thisExpansion) { it == 0 } + }, + IfSome { + override fun produceNext(thisExpansion: ExpansionFrame) = checkExpansionSize(thisExpansion) { it > 0 } + }, + IfSingle { + override fun produceNext(thisExpansion: ExpansionFrame) = checkExpansionSize(thisExpansion) { it == 1 } }, IfMulti { - private val ARG_TO_TEST = VariableRef(0) - private val TRUE_BRANCH = VariableRef(1) - private val FALSE_BRANCH = VariableRef(2) - - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - val testArg = expansion.readArgument(ARG_TO_TEST) - var n = 0 - while (n < 2) { - if (testArg.produceNext() is EndOfExpansion) break - n++ - } - testArg.release() - - val branch = if (n > 1) TRUE_BRANCH else FALSE_BRANCH - val branchExpansion = expansion.readArgument(branch) - expansion.reInitializeFrom(branchExpansion) - branchExpansion.release() - return ContinueExpansion - } + override fun produceNext(thisExpansion: ExpansionFrame) = checkExpansionSize(thisExpansion) { it > 1 } }, - Annotate { private val ANNOTATIONS_ARG = VariableRef(0) private val VALUE_TO_ANNOTATE_ARG = VariableRef(1) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - val annotations = expansion.readArgument(ANNOTATIONS_ARG).map { + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + val annotations = thisExpansion.map(ANNOTATIONS_ARG) { when (it) { - is StringValue -> _Private_Utils.newSymbolToken(it.value) + is StringValue -> newSymbolToken(it.value) is SymbolValue -> it.value is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}") - else -> TODO("Unreachable without stepping in to a container") + else -> unreachable("Unreachable without stepping in to a container") } } - val valueToAnnotateExpansion = expansion.readArgument(VALUE_TO_ANNOTATE_ARG) + val valueToAnnotateExpansion = thisExpansion.readArgument(VALUE_TO_ANNOTATE_ARG) val annotatedExpression = valueToAnnotateExpansion.produceNext().let { it as? DataModelValue ?: throw IonException("Required at least one value.") it.withAnnotations(annotations + it.annotations) } - // Tail-recursion-like optimization - expansion.reInitializeFrom(valueToAnnotateExpansion) - expansion.expanderKind = OneValuedStream - return annotatedExpression + + return annotatedExpression.also { + thisExpansion.tailCall(valueToAnnotateExpansion) + thisExpansion.expanderKind = ExactlyOneValueStream + } } }, MakeString { private val STRINGS_ARG = VariableRef(0) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { val sb = StringBuilder() - expansion.readArgument(STRINGS_ARG).forEach { + thisExpansion.forEach(STRINGS_ARG) { when (it) { is StringValue -> sb.append(it.value) is SymbolValue -> sb.append(it.value.assumeText()) is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}") - is FieldName -> TODO("Unreachable.") + is FieldName -> unreachable() } } - expansion.expanderKind = Empty + thisExpansion.expanderKind = Empty return StringValue(value = sb.toString()) } }, MakeSymbol { private val STRINGS_ARG = VariableRef(0) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - if (expansion.additionalState != null) return EndOfExpansion - expansion.additionalState = Unit + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + if (thisExpansion.additionalState != null) return EndOfExpansion + thisExpansion.additionalState = Unit val sb = StringBuilder() - expansion.readArgument(STRINGS_ARG).forEach { + thisExpansion.forEach(STRINGS_ARG) { when (it) { is StringValue -> sb.append(it.value) is SymbolValue -> sb.append(it.value.assumeText()) is DataModelValue -> throw IonException("Invalid argument type for 'make_symbol': ${it.type}") - is FieldName -> TODO("Unreachable.") + is FieldName -> unreachable() } } - return SymbolValue(value = _Private_Utils.newSymbolToken(sb.toString())) + return SymbolValue(value = newSymbolToken(sb.toString())) } }, MakeBlob { private val LOB_ARG = VariableRef(0) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - // TODO: Optimize to see if we can create a Byte "view" over the existing byte arrays. - if (expansion.additionalState != null) return EndOfExpansion - expansion.additionalState = Unit + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + if (thisExpansion.additionalState != null) return EndOfExpansion + thisExpansion.additionalState = Unit val baos = ByteArrayOutputStream() - expansion.readArgument(LOB_ARG).forEach { + thisExpansion.forEach(LOB_ARG) { when (it) { is LobValue -> baos.write(it.value) is DataModelValue -> throw IonException("Invalid argument type for 'make_blob': ${it.type}") - is FieldName -> TODO("Unreachable.") + is FieldName -> unreachable() } } return BlobValue(value = baos.toByteArray()) @@ -484,12 +401,12 @@ class MacroEvaluator { private val COEFFICIENT_ARG = VariableRef(0) private val EXPONENT_ARG = VariableRef(1) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - if (expansion.additionalState != null) return EndOfExpansion - expansion.additionalState = Unit + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + if (thisExpansion.additionalState != null) return EndOfExpansion + thisExpansion.additionalState = Unit - val coefficient = expansion.readExactlyOneArgument(COEFFICIENT_ARG).bigIntegerValue - val exponent = expansion.readExactlyOneArgument(EXPONENT_ARG).bigIntegerValue + val coefficient = thisExpansion.readExactlyOneArgument(COEFFICIENT_ARG).bigIntegerValue + val exponent = thisExpansion.readExactlyOneArgument(EXPONENT_ARG).bigIntegerValue return DecimalValue(value = BigDecimal(coefficient, -1 * exponent.intValueExact())) } }, @@ -502,13 +419,13 @@ class MacroEvaluator { private val SECOND_ARG = VariableRef(5) private val OFFSET_ARG = VariableRef(6) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - val year = expansion.readExactlyOneArgument(YEAR_ARG).longValue.toInt() - val month = expansion.readZeroOrOneArgument(MONTH_ARG)?.longValue?.toInt() - val day = expansion.readZeroOrOneArgument(DAY_ARG)?.longValue?.toInt() - val hour = expansion.readZeroOrOneArgument(HOUR_ARG)?.longValue?.toInt() - val minute = expansion.readZeroOrOneArgument(MINUTE_ARG)?.longValue?.toInt() - val second = expansion.readZeroOrOneArgument(SECOND_ARG)?.let { + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + val year = thisExpansion.readExactlyOneArgument(YEAR_ARG).longValue.toInt() + val month = thisExpansion.readZeroOrOneArgument(MONTH_ARG)?.longValue?.toInt() + val day = thisExpansion.readZeroOrOneArgument(DAY_ARG)?.longValue?.toInt() + val hour = thisExpansion.readZeroOrOneArgument(HOUR_ARG)?.longValue?.toInt() + val minute = thisExpansion.readZeroOrOneArgument(MINUTE_ARG)?.longValue?.toInt() + val second = thisExpansion.readZeroOrOneArgument(SECOND_ARG)?.let { when (it) { is DecimalValue -> it.value is IntValue -> it.longValue.toBigDecimal() @@ -516,7 +433,7 @@ class MacroEvaluator { } } - val offsetMinutes = expansion.readZeroOrOneArgument(OFFSET_ARG)?.longValue?.toInt() + val offsetMinutes = thisExpansion.readZeroOrOneArgument(OFFSET_ARG)?.longValue?.toInt() try { val ts = if (second != null) { @@ -543,71 +460,64 @@ class MacroEvaluator { Timestamp.forYear(year) } } - expansion.expanderKind = Empty + thisExpansion.expanderKind = Empty return TimestampValue(value = ts) } catch (e: IllegalArgumentException) { throw IonException(e.message) } } }, - MakeFieldNameAndValue { + _Private_MakeFieldNameAndValue { private val FIELD_NAME = VariableRef(0) private val FIELD_VALUE = VariableRef(1) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - val fieldName = expansion.readExactlyOneArgument(FIELD_NAME) + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + val fieldName = thisExpansion.readExactlyOneArgument(FIELD_NAME) val fieldNameExpression = when (fieldName) { is SymbolValue -> FieldName(fieldName.value) is StringValue -> FieldName(newSymbolToken(fieldName.value)) } - expansion.readExactlyOneArgument(FIELD_VALUE) + thisExpansion.readExactlyOneArgument(FIELD_VALUE) - val valueExpansion = expansion.readArgument(FIELD_VALUE) + val valueExpansion = thisExpansion.readArgument(FIELD_VALUE) - expansion.reInitializeFrom(valueExpansion) - expansion.expanderKind = OneValuedStream - return fieldNameExpression + return fieldNameExpression.also { + thisExpansion.tailCall(valueExpansion) + thisExpansion.expanderKind = ExactlyOneValueStream + } } }, - FlattenStruct { + _Private_FlattenStruct { private val STRUCTS = VariableRef(0) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - var argumentExpansion: Expansion? = expansion.additionalState as Expansion? + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + var argumentExpansion: ExpansionFrame? = thisExpansion.additionalState as ExpansionFrame? if (argumentExpansion == null) { - argumentExpansion = expansion.readArgument(STRUCTS) - expansion.additionalState = argumentExpansion + argumentExpansion = thisExpansion.readArgument(STRUCTS) + thisExpansion.additionalState = argumentExpansion } - val currentChildExpansion = expansion.expansionDelegate + val currentChildExpansion = thisExpansion.expansionDelegate return when (val next = currentChildExpansion?.produceNext()) { is DataModelExpression -> next - EndOfContainer -> TODO("I think this is unused!") - EndOfExpansion -> { - expansion.expansionDelegate!!.release() - expansion.expansionDelegate = null - ContinueExpansion - } + EndOfExpansion -> thisExpansion.dropDelegateAndContinue() // Only possible if expansionDelegate is null null -> when (val nextSequence = argumentExpansion.produceNext()) { is StructValue -> { - expansion.expansionDelegate = expansion.expansionPool.acquire { child -> - child.initExpansion( - expanderKind = Stream, - expressions = argumentExpansion.top().expressions, - startInclusive = nextSequence.startInclusive, - endExclusive = nextSequence.endExclusive, - environment = argumentExpansion.top().environment, - ) - } + thisExpansion.expansionDelegate = thisExpansion.session.getExpander( + expanderKind = Stream, + expressions = argumentExpansion.top().expressions, + startInclusive = nextSequence.startInclusive, + endExclusive = nextSequence.endExclusive, + environment = argumentExpansion.top().environment, + ) ContinueExpansion } EndOfExpansion -> EndOfExpansion is DataModelExpression -> throw IonException("invalid argument; make_struct expects structs") - EndOfContainer -> TODO("Unreachable") } } } @@ -616,41 +526,34 @@ class MacroEvaluator { Flatten { private val SEQUENCES = VariableRef(0) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - var argumentExpansion: Expansion? = expansion.additionalState as Expansion? + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + var argumentExpansion: ExpansionFrame? = thisExpansion.additionalState as ExpansionFrame? if (argumentExpansion == null) { - argumentExpansion = expansion.readArgument(SEQUENCES) - expansion.additionalState = argumentExpansion + argumentExpansion = thisExpansion.readArgument(SEQUENCES) + thisExpansion.additionalState = argumentExpansion } - val currentChildExpansion = expansion.expansionDelegate + val currentChildExpansion = thisExpansion.expansionDelegate return when (val next = currentChildExpansion?.produceNext()) { is DataModelExpression -> next - EndOfContainer -> TODO("I think this is unused!") - EndOfExpansion -> { - expansion.expansionDelegate!!.release() - expansion.expansionDelegate = null - ContinueExpansion - } + EndOfExpansion -> thisExpansion.dropDelegateAndContinue() // Only possible if expansionDelegate is null null -> when (val nextSequence = argumentExpansion.produceNext()) { is StructValue -> throw IonException("invalid argument; flatten expects sequences") is DataModelContainer -> { - expansion.expansionDelegate = expansion.expansionPool.acquire { child -> - child.initExpansion( - expanderKind = Stream, - expressions = argumentExpansion.top().expressions, - startInclusive = nextSequence.startInclusive, - endExclusive = nextSequence.endExclusive, - environment = argumentExpansion.top().environment, - ) - } + thisExpansion.expansionDelegate = thisExpansion.session.getExpander( + expanderKind = Stream, + expressions = argumentExpansion.top().expressions, + startInclusive = nextSequence.startInclusive, + endExclusive = nextSequence.endExclusive, + environment = argumentExpansion.top().environment, + ) + ContinueExpansion } EndOfExpansion -> EndOfExpansion is DataModelExpression -> throw IonException("invalid argument; flatten expects sequences") - EndOfContainer -> TODO("Unreachable") } } } @@ -659,12 +562,12 @@ class MacroEvaluator { private val ARG_A = VariableRef(0) private val ARG_B = VariableRef(1) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - if (expansion.additionalState != null) return EndOfExpansion - expansion.additionalState = Unit + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + if (thisExpansion.additionalState != null) return EndOfExpansion + thisExpansion.additionalState = Unit - val a = expansion.readExactlyOneArgument(ARG_A).bigIntegerValue - val b = expansion.readExactlyOneArgument(ARG_B).bigIntegerValue + val a = thisExpansion.readExactlyOneArgument(ARG_A).bigIntegerValue + val b = thisExpansion.readExactlyOneArgument(ARG_B).bigIntegerValue return BigIntValue(value = a + b) } }, @@ -672,25 +575,25 @@ class MacroEvaluator { private val ARGS = VariableRef(0) // Initial value = 0 - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { // TODO: Optimize to use LongIntValue when possible - var delegate = expansion.expansionDelegate - val runningTotal = expansion.additionalState as? BigInteger ?: BigInteger.ZERO + var delegate = thisExpansion.expansionDelegate + val runningTotal = thisExpansion.additionalState as? BigInteger ?: BigInteger.ZERO if (delegate == null) { - delegate = expansion.readArgument(ARGS) - expansion.expansionDelegate = delegate + delegate = thisExpansion.readArgument(ARGS) + thisExpansion.expansionDelegate = delegate } when (val nextExpandedArg = delegate.produceNext()) { is IntValue -> { val nextDelta = nextExpandedArg.bigIntegerValue val nextOutput = runningTotal + nextDelta - expansion.additionalState = nextOutput + thisExpansion.additionalState = nextOutput return BigIntValue(value = nextOutput) } EndOfExpansion -> return nextExpandedArg is DataModelValue -> throw IonException("delta arguments must be integers") - is FieldName, EndOfContainer -> TODO("Unreachable") + is FieldName -> unreachable() } } }, @@ -698,77 +601,111 @@ class MacroEvaluator { private val COUNT_ARG = VariableRef(0) private val THING_TO_REPEAT = VariableRef(1) - override fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue { - var n = expansion.additionalState as Long? + override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + var n = thisExpansion.additionalState as Long? if (n == null) { - n = expansion.readExactlyOneArgument(COUNT_ARG).longValue + n = thisExpansion.readExactlyOneArgument(COUNT_ARG).longValue if (n < 0) throw IonException("invalid argument; 'n' must be non-negative") - expansion.additionalState = n + thisExpansion.additionalState = n } - if (expansion.expansionDelegate == null) { + if (thisExpansion.expansionDelegate == null) { if (n > 0) { - expansion.expansionDelegate = expansion.readArgument(THING_TO_REPEAT) - expansion.additionalState = n - 1 + thisExpansion.expansionDelegate = thisExpansion.readArgument(THING_TO_REPEAT) + thisExpansion.additionalState = n - 1 } else { return EndOfExpansion } } - val repeated = expansion.expansionDelegate!! + val repeated = thisExpansion.expansionDelegate!! return when (val maybeNext = repeated.produceNext()) { - is DataModelExpression, EndOfContainer -> maybeNext - EndOfExpansion -> { - expansion.expansionDelegate!!.release() - expansion.expansionDelegate = null - ContinueExpansion - } + is DataModelExpression -> maybeNext + EndOfExpansion -> thisExpansion.dropDelegateAndContinue() } } }, ; - abstract fun produceNext(expansion: Expansion): ExpansionOutputExpressionOrContinue + abstract fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue - protected fun Expansion.readArgument(variableRef: VariableRef): Expansion { - val argIndex = environment.argumentIndices[variableRef.signatureIndex] + internal inline fun checkExpansionSize(thisExpansion: ExpansionFrame, condition: (Int) -> Boolean): ContinueExpansion { + val argToTest = VariableRef(0) + val trueBranch = VariableRef(1) + val falseBranch = VariableRef(2) + + val testArg = thisExpansion.readArgument(argToTest) + var n = 0 + while (n < 2) { + if (testArg.produceNext() is EndOfExpansion) break + n++ + } + testArg.drop() + + val branch = if (condition(n)) trueBranch else falseBranch + val branchExpansion = thisExpansion.readArgument(branch) + + thisExpansion.tailCall(branchExpansion) + return ContinueExpansion + } + + internal fun VariableRef.readFrom(environment: Environment, session: MacroEvaluationSession): ExpansionFrame { + val argIndex = environment.argumentIndices[signatureIndex] if (argIndex < 0) { // Argument was elided. - return expansionPool.acquire { it.expanderKind = Empty } + return session.getExpander(Empty, emptyList(), 0, 0, Environment.EMPTY) } val firstArgExpression = environment.arguments[argIndex] - return expansionPool.acquire { new -> - new.initExpansion( - expanderKind = Stream, - expressions = environment.arguments, - startInclusive = if (firstArgExpression is ExpressionGroup) firstArgExpression.startInclusive else argIndex, - endExclusive = if (firstArgExpression is HasStartAndEnd) firstArgExpression.endExclusive else argIndex + 1, - environment = environment.parentEnvironment!! - ) + return session.getExpander( + expanderKind = Stream, + expressions = environment.arguments, + startInclusive = if (firstArgExpression is ExpressionGroup) firstArgExpression.startInclusive else argIndex, + endExclusive = if (firstArgExpression is HasStartAndEnd) firstArgExpression.endExclusive else argIndex + 1, + environment = environment.parentEnvironment!! + ) + } + + internal fun ExpansionFrame.readArgument(variableRef: VariableRef): ExpansionFrame { + // println("Reading argument for $variableRef") + // println("From $environment") + val argIndex = environment.argumentIndices[variableRef.signatureIndex] + if (argIndex < 0) { + // Argument was elided. + return session.getExpander(Empty, emptyList(), 0, 0, Environment.EMPTY) } + val firstArgExpression = environment.arguments[argIndex] + return session.getExpander( + expanderKind = Stream, + expressions = environment.arguments, + startInclusive = if (firstArgExpression is ExpressionGroup) firstArgExpression.startInclusive else argIndex, + endExclusive = if (firstArgExpression is HasStartAndEnd) firstArgExpression.endExclusive else argIndex + 1, + environment = environment.parentEnvironment!! + )//.also { println("Variable $variableRef $it") } } - protected inline fun Expansion.forEach(action: (DataModelExpression) -> Unit) { + internal inline fun ExpansionFrame.forEach(variableRef: VariableRef, action: (DataModelExpression) -> Unit) { + val variableExpansion = readArgument(variableRef) while (true) { - when (val next = produceNext()) { - EndOfContainer, EndOfExpansion -> return + when (val next = variableExpansion.produceNext()) { + EndOfExpansion -> return is DataModelExpression -> action(next) } } } - protected inline fun Expansion.map(action: (DataModelExpression) -> T): List { + internal inline fun ExpansionFrame.map(variableRef: VariableRef, action: (DataModelExpression) -> T): List { + val variableExpansion = readArgument(variableRef) val result = mutableListOf() while (true) { - when (val next = produceNext()) { - EndOfContainer, EndOfExpansion -> return result + when (val next = variableExpansion.produceNext()) { + EndOfExpansion -> return result is DataModelExpression -> result.add(action(next)) } } } - protected inline fun Expansion.readZeroOrOneArgument(variableRef: VariableRef): T? { + internal inline fun ExpansionFrame.readZeroOrOneArgument(variableRef: VariableRef): T? { val argExpansion = readArgument(variableRef) var argValue: T? = null while (true) { @@ -780,21 +717,46 @@ class MacroEvaluator { } is DataModelValue -> throw IonException("invalid argument; found ${it.type}") EndOfExpansion -> break - EndOfContainer, - is FieldName -> TODO("Unreachable without stepping into a container") + is FieldName -> unreachable("Unreachable without stepping into a container") } } return argValue } - protected inline fun Expansion.readExactlyOneArgument(variableRef: VariableRef): T { + internal inline fun ExpansionFrame.readExactlyOneArgument(variableRef: VariableRef): T { return readZeroOrOneArgument(variableRef) ?: throw IonException("invalid argument; no value when one is expected") } - } - class Expansion( - @JvmField val expansionPool: Pool, + companion object { + @JvmStatic + fun getExpanderKindForSystemMacro(systemMacro: SystemMacro) = when (systemMacro) { + SystemMacro.Annotate -> Annotate + SystemMacro.MakeString -> MakeString + SystemMacro.MakeSymbol -> MakeSymbol + SystemMacro.MakeDecimal -> MakeDecimal + SystemMacro.Repeat -> Repeat + SystemMacro.Sum -> Sum + SystemMacro.Delta -> Delta + SystemMacro.MakeBlob -> MakeBlob + SystemMacro.Flatten -> Flatten + SystemMacro._Private_FlattenStruct -> _Private_FlattenStruct + SystemMacro.MakeTimestamp -> MakeTimestamp + SystemMacro._Private_MakeFieldNameAndValue -> _Private_MakeFieldNameAndValue + SystemMacro.IfNone -> IfNone + SystemMacro.IfSome -> IfSome + SystemMacro.IfSingle -> IfSingle + SystemMacro.IfMulti -> IfMulti + else -> if (systemMacro.body != null) { + throw IllegalStateException("SystemMacro ${systemMacro.name} should be using its template body.") + } else { + TODO("Not implemented yet: ${systemMacro.name}") + } + } + } + } + class ExpansionFrame( + @JvmField val session: MacroEvaluationSession, @JvmField var expanderKind: ExpanderKind = Uninitialized, /** * The [Expression]s being expanded. This MUST be the original list, not a sublist because @@ -804,23 +766,37 @@ class MacroEvaluator { @JvmField var expressions: List = emptyList(), /** Current position within [expressions] of this expansion */ @JvmField var i: Int = 0, - /** End of [expressions] that are applicable for this [ExpansionInfo] */ + /** End of [expressions] that are applicable for this [ExpansionFrame] */ @JvmField var endExclusive: Int = 0, - /** - * The evaluation [Environment]—i.e. variable bindings. - */ + /** The evaluation [Environment]—i.e. variable bindings. */ @JvmField var environment: Environment = Environment.EMPTY, - // TODO: Should this be "additional state"? - @JvmField var expansionDelegate: Expansion? = null, + @JvmField var _expansionDelegate: ExpansionFrame? = null, @JvmField var additionalState: Any? = null, ) { - fun top(): Expansion = expansionDelegate?.top() ?: this - fun release() { + var expansionDelegate: ExpansionFrame? + get() = _expansionDelegate + set(value) { + check(value != this) + _expansionDelegate = value + } + + fun dropDelegateAndContinue(): ContinueExpansion { + expansionDelegate?.drop() + expansionDelegate = null + return ContinueExpansion + } + + fun top(): ExpansionFrame = expansionDelegate?.top() ?: this + + fun drop() { expanderKind = Uninitialized additionalState = null - expansionDelegate?.release() - expansionPool.take(this) + environment = Environment.EMPTY + expressions = emptyList() + expansionDelegate?.drop() + expansionDelegate = null + session.returnExpander(this) } fun initExpansion( @@ -839,13 +815,17 @@ class MacroEvaluator { expansionDelegate = null } - fun reInitializeFrom(other: Expansion) { + fun tailCall(other: ExpansionFrame) { this.expanderKind = other.expanderKind this.expressions = other.expressions this.i = other.i this.endExclusive = other.endExclusive this.expansionDelegate = other.expansionDelegate this.additionalState = other.additionalState + this.environment = other.environment + // Drop `other` + other.expansionDelegate = null + other.drop() } fun produceNext(): ExpansionOutputExpression { @@ -858,11 +838,11 @@ class MacroEvaluator { } override fun toString() = """ - |ExpansionInfo( + |ExpansionFrame( | expansionKind: $expanderKind, - | environment: $environment, + | environment: ${environment.toString().lines().joinToString("\n| ")}, | expressions: [ - | ${expressions.joinToString(",\n| ") { it.toString() } } + | ${expressions.mapIndexed { index, expression -> "$index. $expression" }.joinToString(",\n| ") { it.toString() } } | ], | endExclusive: $endExclusive, | i: $i, @@ -871,34 +851,6 @@ class MacroEvaluator { |) """.trimMargin() } - - /** - * Suitable for single-threaded use only. - * - * TODO: Clean up the debugging parts. - */ - class Pool(private val objectFactory: (Pool) -> T) { - private val availableElements = ArrayList(32) - private val allElements = IdentityHashMap(32) - private var acquireCount = 0 - private var releaseCount = 0 - fun acquire(init: (T) -> Unit): T { - val element = availableElements.removeLastOrNull() ?: objectFactory(this) - element.apply(init) - allElements[element] = 1 - // println("Pool(a=${++acquireCount},r=$releaseCount)") - if (acquireCount - releaseCount > 1000) throw IllegalStateException("Probable runtime stack overflow or memory leak") - return element - } - fun take(t: T) { - check(allElements[t] != 0) { "Double return!" } - if (allElements[t] == 1) { - availableElements.add(t) - allElements[t] = 0 - } - // println("Pool(a=$acquireCount,r=${++releaseCount})") - } - } } /** @@ -946,3 +898,40 @@ private fun Macro.calculateArgumentIndices( } return argsIndices.toList() } + +private fun Macro.calculateArgumentIndicesByName( + encodingExpressions: List, + argsStartInclusive: Int, + argsEndExclusive: Int +): Map { + // TODO: For TDL macro invocations, see if we can calculate this during the "compile" step. + var numArgs = 0 + val argsIndices = IntArray(signature.size) + var currentArgIndex = argsStartInclusive + + for (p in signature) { + if (currentArgIndex >= argsEndExclusive) { + if (!p.cardinality.canBeVoid) throw IonException("No value provided for parameter ${p.variableName}") + // Elided rest parameter. + argsIndices[numArgs] = -1 + } else { + argsIndices[numArgs] = currentArgIndex + currentArgIndex = when (val expr = encodingExpressions[currentArgIndex]) { + is HasStartAndEnd -> expr.endExclusive + else -> currentArgIndex + 1 + } + } + numArgs++ + } + while (currentArgIndex < argsEndExclusive) { + currentArgIndex = when (val expr = encodingExpressions[currentArgIndex]) { + is HasStartAndEnd -> expr.endExclusive + else -> currentArgIndex + 1 + } + numArgs++ + } + if (numArgs > signature.size) { + throw IonException("Too many arguments. Expected ${signature.size}, but found $numArgs") + } + return argsIndices.mapIndexed { i, it -> signature[i] to it }.toMap() +} diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt index 30d355c1d..254b37bf3 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt @@ -43,10 +43,6 @@ class MacroEvaluatorAsIonReader( } is Expression.FieldName -> queuedFieldName = nextCandidate is Expression.DataModelValue -> queuedValueExpression = nextCandidate - Expression.EndOfContainer -> { - queuedFieldName = null - return - } Expression.EndOfExpansion -> { queuedFieldName = null return diff --git a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt index 4b0d3e37c..329ed6910 100644 --- a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt @@ -27,8 +27,8 @@ enum class SystemMacro( IfMulti(-1, IF_MULTI, listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), // Unnameable, unaddressable macros used for the internals of certain other system macros - FlattenStruct(-1, systemSymbol = null, listOf(zeroToManyTagged("structs"))), - MakeFieldNameAndValue(-1, systemSymbol = null, listOf(exactlyOneTagged("fieldName"), exactlyOneTagged("value"))), + _Private_FlattenStruct(-1, systemSymbol = null, listOf(zeroToManyTagged("structs"))), + _Private_MakeFieldNameAndValue(-1, systemSymbol = null, listOf(exactlyOneTagged("fieldName"), exactlyOneTagged("value"))), // The real macros Values(1, VALUES, listOf(zeroToManyTagged("values")), templateBody { variable(0) }), @@ -63,7 +63,8 @@ enum class SystemMacro( ), MakeBlob(13, MAKE_BLOB, listOf(zeroToManyTagged("bytes"))), MakeList( - 14, MAKE_LIST, listOf(zeroToManyTagged("sequences")), templateBody { + 14, MAKE_LIST, listOf(zeroToManyTagged("sequences")), + templateBody { list { macro(Flatten) { variable(0) @@ -72,7 +73,8 @@ enum class SystemMacro( } ), MakeSExp( - 15, MAKE_SEXP, listOf(zeroToManyTagged("sequences")), templateBody { + 15, MAKE_SEXP, listOf(zeroToManyTagged("sequences")), + templateBody { sexp { macro(Flatten) { variable(0) @@ -85,7 +87,7 @@ enum class SystemMacro( 16, MAKE_FIELD, listOf(exactlyOneTagged("fieldName"), exactlyOneTagged("value")), templateBody { struct { - macro(MakeFieldNameAndValue) { + macro(_Private_MakeFieldNameAndValue) { variable(0) variable(1) } @@ -97,7 +99,7 @@ enum class SystemMacro( 17, MAKE_STRUCT, listOf(zeroToManyTagged("structs")), templateBody { struct { - macro(FlattenStruct) { + macro(_Private_FlattenStruct) { variable(0) } } @@ -105,7 +107,6 @@ enum class SystemMacro( ), ParseIon(18, PARSE_ION, listOf(zeroToManyTagged("data"))), // TODO: parse_ion - /** * ```ion * (macro set_symbols (symbols*) @@ -236,12 +237,12 @@ enum class SystemMacro( sexp { symbol(IMPORT) symbol(theModule) - variable(0) + variable(0, exactlyOneTagged("catalog_key")) // This is equivalent to `(.default (%version) 1)`, but eliminates a layer of indirection. macro(IfNone) { - variable(1) + variable(1, zeroOrOneTagged("version")) int(1) - variable(1) + variable(1, zeroOrOneTagged("version")) } } sexp { diff --git a/src/main/java/com/amazon/ion/util/Assumptions.kt b/src/main/java/com/amazon/ion/util/Assumptions.kt index b1d86b0d4..a4a9c097f 100644 --- a/src/main/java/com/amazon/ion/util/Assumptions.kt +++ b/src/main/java/com/amazon/ion/util/Assumptions.kt @@ -47,3 +47,5 @@ internal inline fun confirm(assumption: Boolean, lazyMessage: () -> String) { throw IonException(lazyMessage()) } } + +internal fun unreachable(reason: String? = null): Nothing = throw IllegalStateException(reason) diff --git a/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt b/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt index 108d2234c..f8984caba 100644 --- a/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt +++ b/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt @@ -87,6 +87,12 @@ abstract class ConformanceTestRunner( "set_symbols does not accept null.string" in completeTestName -> false "set_symbols does not accept annotated arguments" in completeTestName -> false + // FIXME: Add syntax checks in MacroCompiler + "tdl/expression_groups.ion" in file.absolutePath -> false + + // FIXME: Implicit rest args don't always work + "implicit rest args" in completeTestName -> false + // FIXME: Ensure that the text reader throws if unexpected extra args are encountered "sum arguments may not be more than two integers" in completeTestName -> false "none signals an error when argument is" in completeTestName -> false diff --git a/src/test/java/com/amazon/ion/conformance/structure.kt b/src/test/java/com/amazon/ion/conformance/structure.kt index 07f595bf1..dc9701686 100644 --- a/src/test/java/com/amazon/ion/conformance/structure.kt +++ b/src/test/java/com/amazon/ion/conformance/structure.kt @@ -162,8 +162,8 @@ private fun ParserState.readExtension(): List { private fun ParserState.readContinuation(): DynamicNode { val continuation = sexp.tailFrom(pos) - val firstExpression = continuation.first() - firstExpression as? SeqElement ?: builder.reportSyntaxError(firstExpression, "continuation") + val firstExpression = continuation.firstOrNull() + firstExpression as? SeqElement ?: builder.reportSyntaxError(sexp, "continuation") return continuation.flatMap { it as? SeqElement ?: builder.reportSyntaxError(it, "extension") diff --git a/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt b/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt index 19124b691..4a9d68ecb 100644 --- a/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt +++ b/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt @@ -5,7 +5,6 @@ package com.amazon.ion.impl import com.amazon.ion.* import com.amazon.ion.impl.macro.* import com.amazon.ion.system.* -import java.lang.AssertionError import java.math.BigDecimal import java.math.BigInteger import org.junit.jupiter.api.Assertions.assertEquals diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt index f993bf5a3..291228a16 100644 --- a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt +++ b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt @@ -937,7 +937,7 @@ class MacroEvaluatorTest { @MethodSource("com.amazon.ion.impl.macro.MacroEvaluatorTest\$IfExpanderTestParameters#parameters") fun `check 'if' expansion logic`(ifSpecialForm: SystemMacro, expressionToTest: Macro, expectMatches: Boolean) { // Given: - // (macro test_if (x*) ( (%x) "a" "b")) + // (macro test_if (x*) (. (%x) "a" "b")) // When: // (:test_if ) // Then: From d4a7f6cf54802cc6a2462283256e1bf6eb84d0b2 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Mon, 16 Dec 2024 16:11:10 -0800 Subject: [PATCH 03/10] Cleanup --- .../com/amazon/ion/impl/macro/Environment.kt | 9 ++-- .../com/amazon/ion/impl/macro/Expression.kt | 12 ++--- .../amazon/ion/impl/macro/MacroEvaluator.kt | 50 ++----------------- .../java/com/amazon/ion/util/Assumptions.kt | 3 ++ .../amazon/ion/conformance/expectations.kt | 2 + .../ion/impl/macro/MacroEvaluatorTest.kt | 13 ++--- 6 files changed, 23 insertions(+), 66 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/macro/Environment.kt b/src/main/java/com/amazon/ion/impl/macro/Environment.kt index 8b0478e4c..18c854c89 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Environment.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Environment.kt @@ -18,16 +18,13 @@ data class Environment private constructor( val arguments: List, // TODO: Replace with IntArray val argumentIndices: List, - val argumentsByName: Map, val parentEnvironment: Environment?, ) { - fun createChild(arguments: List, argumentIndices: List, byName: Map) = Environment(arguments, argumentIndices, byName, this) + fun createChild(arguments: List, argumentIndices: List) = Environment(arguments, argumentIndices, this) override fun toString() = """ |Environment( | argumentIndices: $argumentIndices, - | argumentsByName: [${argumentsByName.map { (name, index) -> "\n| $name -> $index" }.joinToString() } - | ], | argumentExpressions: [${arguments.mapIndexed { index, expression -> "\n| $index. $expression" }.joinToString() } | ], | parent: ${parentEnvironment.toString().lines().joinToString("\n| ")}, @@ -36,8 +33,8 @@ data class Environment private constructor( companion object { @JvmStatic - val EMPTY = Environment(emptyList(), emptyList(), emptyMap(), null) + val EMPTY = Environment(emptyList(), emptyList(), null) @JvmStatic - fun create(arguments: List, argumentIndices: List, byName: Map) = Environment(arguments, argumentIndices, byName, null) + fun create(arguments: List, argumentIndices: List) = Environment(arguments, argumentIndices, null) } } diff --git a/src/main/java/com/amazon/ion/impl/macro/Expression.kt b/src/main/java/com/amazon/ion/impl/macro/Expression.kt index 12d5d27c4..f703b132b 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Expression.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Expression.kt @@ -196,10 +196,10 @@ sealed interface Expression { * @property selfIndex the index of the first expression of the list (i.e. this instance) * @property endExclusive the index of the last expression contained in the list */ - data class ListValue @JvmOverloads constructor( + data class ListValue( override val annotations: List = emptyList(), override val selfIndex: Int, - override val endExclusive: Int, + override val endExclusive: Int ) : DataModelContainer { override val type: IonType get() = IonType.LIST override fun withAnnotations(annotations: List) = copy(annotations = annotations) @@ -208,10 +208,10 @@ sealed interface Expression { /** * An Ion SExp that could contain variables or macro invocations. */ - data class SExpValue @JvmOverloads constructor( + data class SExpValue( override val annotations: List = emptyList(), override val selfIndex: Int, - override val endExclusive: Int, + override val endExclusive: Int ) : DataModelContainer { override val type: IonType get() = IonType.SEXP override fun withAnnotations(annotations: List) = copy(annotations = annotations) @@ -220,11 +220,11 @@ sealed interface Expression { /** * An Ion Struct that could contain variables or macro invocations. */ - data class StructValue @JvmOverloads constructor( + data class StructValue( override val annotations: List = emptyList(), override val selfIndex: Int, override val endExclusive: Int, - val templateStructIndex: Map> = emptyMap(), + val templateStructIndex: Map> ) : DataModelContainer { override val type: IonType get() = IonType.STRUCT override fun withAnnotations(annotations: List) = copy(annotations = annotations) diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index 4c7c2091a..3d2da0ce0 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -187,8 +187,7 @@ class MacroEvaluator { argsStartInclusive = next.startInclusive, argsEndExclusive = next.endExclusive, ) - val argsIndicesByName = macro.calculateArgumentIndicesByName(thisExpansion.expressions, next.startInclusive, next.endExclusive) - val newEnvironment = thisExpansion.environment.createChild(thisExpansion.expressions, argIndices, argsIndicesByName) + val newEnvironment = thisExpansion.environment.createChild(thisExpansion.expressions, argIndices) val expanderKind = if (macro.body != null) Stream else getExpanderKindForSystemMacro(macro as SystemMacro) thisExpansion.expansionDelegate = thisExpansion.session.getExpander( expanderKind = expanderKind, @@ -667,8 +666,6 @@ class MacroEvaluator { } internal fun ExpansionFrame.readArgument(variableRef: VariableRef): ExpansionFrame { - // println("Reading argument for $variableRef") - // println("From $environment") val argIndex = environment.argumentIndices[variableRef.signatureIndex] if (argIndex < 0) { // Argument was elided. @@ -681,7 +678,7 @@ class MacroEvaluator { startInclusive = if (firstArgExpression is ExpressionGroup) firstArgExpression.startInclusive else argIndex, endExclusive = if (firstArgExpression is HasStartAndEnd) firstArgExpression.endExclusive else argIndex + 1, environment = environment.parentEnvironment!! - )//.also { println("Variable $variableRef $it") } + ) } internal inline fun ExpansionFrame.forEach(variableRef: VariableRef, action: (DataModelExpression) -> Unit) { @@ -831,9 +828,9 @@ class MacroEvaluator { fun produceNext(): ExpansionOutputExpression { while (true) { val next = expanderKind.produceNext(this) - if (next is ExpansionOutputExpression) return next - // Implied: - // if (next is ContinueExpansion) continue + if (next is ContinueExpansion) continue + session.incrementStepCounter() + return next as ExpansionOutputExpression } } @@ -898,40 +895,3 @@ private fun Macro.calculateArgumentIndices( } return argsIndices.toList() } - -private fun Macro.calculateArgumentIndicesByName( - encodingExpressions: List, - argsStartInclusive: Int, - argsEndExclusive: Int -): Map { - // TODO: For TDL macro invocations, see if we can calculate this during the "compile" step. - var numArgs = 0 - val argsIndices = IntArray(signature.size) - var currentArgIndex = argsStartInclusive - - for (p in signature) { - if (currentArgIndex >= argsEndExclusive) { - if (!p.cardinality.canBeVoid) throw IonException("No value provided for parameter ${p.variableName}") - // Elided rest parameter. - argsIndices[numArgs] = -1 - } else { - argsIndices[numArgs] = currentArgIndex - currentArgIndex = when (val expr = encodingExpressions[currentArgIndex]) { - is HasStartAndEnd -> expr.endExclusive - else -> currentArgIndex + 1 - } - } - numArgs++ - } - while (currentArgIndex < argsEndExclusive) { - currentArgIndex = when (val expr = encodingExpressions[currentArgIndex]) { - is HasStartAndEnd -> expr.endExclusive - else -> currentArgIndex + 1 - } - numArgs++ - } - if (numArgs > signature.size) { - throw IonException("Too many arguments. Expected ${signature.size}, but found $numArgs") - } - return argsIndices.mapIndexed { i, it -> signature[i] to it }.toMap() -} diff --git a/src/main/java/com/amazon/ion/util/Assumptions.kt b/src/main/java/com/amazon/ion/util/Assumptions.kt index a4a9c097f..92eab026a 100644 --- a/src/main/java/com/amazon/ion/util/Assumptions.kt +++ b/src/main/java/com/amazon/ion/util/Assumptions.kt @@ -48,4 +48,7 @@ internal inline fun confirm(assumption: Boolean, lazyMessage: () -> String) { } } +/** + * Marks a branch as unreachable (for human readability). + */ internal fun unreachable(reason: String? = null): Nothing = throw IllegalStateException(reason) diff --git a/src/test/java/com/amazon/ion/conformance/expectations.kt b/src/test/java/com/amazon/ion/conformance/expectations.kt index a9ecae7a7..05974b572 100644 --- a/src/test/java/com/amazon/ion/conformance/expectations.kt +++ b/src/test/java/com/amazon/ion/conformance/expectations.kt @@ -19,6 +19,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.fail /** * Asserts that fully traversing the reader will result in an [IonException]. @@ -53,6 +54,7 @@ fun TestCaseSupport.assertSignals(sexp: SeqElement, r: IonReader) { private fun IonReader.walk(): List { val events = mutableListOf() fun recordEvent(eventType: String = type.toString(), value: Any? = "") { + if (events.size > 10_000_000) fail("Ion stream does not appear to terminate.") events.add("[$eventType] $value") } recordEvent("START") diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt index 291228a16..9bb23e26e 100644 --- a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt +++ b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt @@ -285,8 +285,7 @@ class MacroEvaluatorTest { } } - val actual = evaluator.expandNext() - assertEquals(BoolValue(emptyList(), true), actual) + assertEquals(BoolValue(emptyList(), true), evaluator.expandNext()) assertEquals(null, evaluator.expandNext()) } @@ -370,24 +369,20 @@ class MacroEvaluatorTest { fun `invoke values with scalars`() { // Given: // When: - // (:values 1 2 3 "a") + // (:values 1 "a") // Then: - // 1 2 3 "a" + // 1 "a" evaluator.initExpansion { eexp(Values) { expressionGroup { int(1) - int(2) - int(3) string("a") } } } assertEquals(LongIntValue(emptyList(), 1), evaluator.expandNext()) - assertEquals(LongIntValue(emptyList(), 2), evaluator.expandNext()) - assertEquals(LongIntValue(emptyList(), 3), evaluator.expandNext()) assertEquals(StringValue(emptyList(), "a"), evaluator.expandNext()) assertEquals(null, evaluator.expandNext()) } @@ -937,7 +932,7 @@ class MacroEvaluatorTest { @MethodSource("com.amazon.ion.impl.macro.MacroEvaluatorTest\$IfExpanderTestParameters#parameters") fun `check 'if' expansion logic`(ifSpecialForm: SystemMacro, expressionToTest: Macro, expectMatches: Boolean) { // Given: - // (macro test_if (x*) (. (%x) "a" "b")) + // (macro test_if (x*) ( (%x) "a" "b")) // When: // (:test_if ) // Then: From 474be3f18973bf933d7fab33d37c6481b64fe5eb Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Mon, 16 Dec 2024 16:21:03 -0800 Subject: [PATCH 04/10] More cleanup --- src/main/java/com/amazon/ion/impl/macro/Expression.kt | 4 ++-- .../java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt | 4 ++-- src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt | 2 +- src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt | 2 +- .../com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt | 7 +------ src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt | 6 +++--- 6 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/macro/Expression.kt b/src/main/java/com/amazon/ion/impl/macro/Expression.kt index f703b132b..fc5f983a5 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Expression.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Expression.kt @@ -86,7 +86,7 @@ sealed interface Expression { * * TODO: See if we can get rid of this by e.g. using nulls during macro compilation. */ - data object Placeholder : TemplateBodyExpression, EExpressionBodyExpression + object Placeholder : TemplateBodyExpression, EExpressionBodyExpression /** * A group of expressions that form the argument for one macro parameter. @@ -235,7 +235,7 @@ sealed interface Expression { /** * A reference to a variable that needs to be expanded. */ - data class VariableRef @JvmOverloads constructor(val signatureIndex: Int, val parameter: Macro.Parameter? = null) : TemplateBodyExpression + data class VariableRef(val signatureIndex: Int) : TemplateBodyExpression sealed interface InvokableExpression : HasStartAndEnd, Expression { val macro: Macro diff --git a/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt b/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt index b67565a66..7a1756361 100644 --- a/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt +++ b/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt @@ -57,7 +57,7 @@ internal interface DataModelDsl : ValuesDsl { @ExpressionBuilderDslMarker internal interface TemplateDsl : ValuesDsl { fun macro(macro: Macro, arguments: InvocationBody.() -> Unit) - fun variable(signatureIndex: Int, parameter: Macro.Parameter? = null) + fun variable(signatureIndex: Int) fun list(content: TemplateDsl.() -> Unit) fun sexp(content: TemplateDsl.() -> Unit) fun struct(content: Fields.() -> Unit) @@ -186,7 +186,7 @@ internal sealed class ExpressionBuilderDsl : ValuesDsl, ValuesDsl.Fields { override fun list(content: TemplateDsl.() -> Unit) = containerWithAnnotations(content, ::ListValue) override fun sexp(content: TemplateDsl.() -> Unit) = containerWithAnnotations(content, ::SExpValue) override fun struct(content: TemplateDsl.Fields.() -> Unit) = containerWithAnnotations(content, ::newStruct) - override fun variable(signatureIndex: Int, parameter: Macro.Parameter?) { expressions.add(VariableRef(signatureIndex, parameter)) } + override fun variable(signatureIndex: Int) { expressions.add(VariableRef(signatureIndex)) } override fun macro(macro: Macro, arguments: TemplateDsl.InvocationBody.() -> Unit) = container(arguments) { start, end -> MacroInvocation(macro, start, end) } override fun expressionGroup(content: TemplateDsl.() -> Unit) = container(content, ::ExpressionGroup) } diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt b/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt index 3f67e9146..f64da15a6 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt @@ -269,7 +269,7 @@ internal class MacroCompiler( confirmNoAnnotations("on variable reference '$name'") val index = signature.indexOfFirst { it.variableName == name } confirm(index >= 0) { "variable '$name' is not recognized" } - expressions[placeholderIndex] = VariableRef(index, parameter = signature[index]) + expressions[placeholderIndex] = VariableRef(index) confirm(!nextValue()) { "Variable expansion should contain only the variable name." } stepOutOfContainer() } diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index 3d2da0ce0..b3ccfc5fc 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -110,7 +110,7 @@ class MacroEvaluator { * Evaluate the macro expansion until the next [DataModelExpression] can be returned. * Returns null if at the end of a container or at the end of the expansion. */ - fun expandNext(): ExpansionOutputExpression? { + fun expandNext(): DataModelExpression? { currentExpr = null while (currentExpr == null && !containerStack.isEmpty()) { val currentContainer = containerStack.peek() diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt index 254b37bf3..98ea85437 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt @@ -35,18 +35,13 @@ class MacroEvaluatorAsIonReader( private fun queueNext() { queuedValueExpression = null while (queuedValueExpression == null) { - val nextCandidate = evaluator.expandNext() - when (nextCandidate) { + when (val nextCandidate = evaluator.expandNext()) { null -> { queuedFieldName = null return } is Expression.FieldName -> queuedFieldName = nextCandidate is Expression.DataModelValue -> queuedValueExpression = nextCandidate - Expression.EndOfExpansion -> { - queuedFieldName = null - return - } } } } diff --git a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt index 329ed6910..67cead5b3 100644 --- a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt @@ -237,12 +237,12 @@ enum class SystemMacro( sexp { symbol(IMPORT) symbol(theModule) - variable(0, exactlyOneTagged("catalog_key")) + variable(0) // This is equivalent to `(.default (%version) 1)`, but eliminates a layer of indirection. macro(IfNone) { - variable(1, zeroOrOneTagged("version")) + variable(1) int(1) - variable(1, zeroOrOneTagged("version")) + variable(1) } } sexp { From 1bd97019616a5b2add49cc8ddb0b0f1c50ae61a0 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Mon, 16 Dec 2024 16:23:11 -0800 Subject: [PATCH 05/10] Even more cleanup --- .../com/amazon/ion/impl/macro/MacroEvaluatorTest.kt | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt index 9bb23e26e..f71954120 100644 --- a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt +++ b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt @@ -427,9 +427,7 @@ class MacroEvaluatorTest { } evaluator.initExpansion { - eexp(voidableIdentityMacro) { - expressionGroup { } - } + eexp(voidableIdentityMacro) {} } assertEquals(null, evaluator.expandNext()) @@ -1050,15 +1048,11 @@ class MacroEvaluatorTest { evaluator.initExpansion { eexp(Repeat) { - int(4) - expressionGroup { - int(0) - } + int(2) + int(0) } } - assertEquals(LongIntValue(value = 0), evaluator.expandNext()) - assertEquals(LongIntValue(value = 0), evaluator.expandNext()) assertEquals(LongIntValue(value = 0), evaluator.expandNext()) assertEquals(LongIntValue(value = 0), evaluator.expandNext()) assertEquals(null, evaluator.expandNext()) From 1e8bc036a2de889fb6cde67e5f11b29f32b97d01 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Mon, 16 Dec 2024 16:23:48 -0800 Subject: [PATCH 06/10] Update ion-tests submodule --- ion-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ion-tests b/ion-tests index 89a6fd4c9..a22bf4371 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit 89a6fd4c9a1eae3b90457ea77758dc8cb8219a26 +Subproject commit a22bf437148d6c88c48461c592096b27a1b06fa6 From 92560f5dfd6e37983b2324e8f5a6258d1338ddcd Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Mon, 16 Dec 2024 16:29:48 -0800 Subject: [PATCH 07/10] Even moar cleanup --- .../ion/impl/bin/IonManagedWriter_1_1.kt | 2 +- .../com/amazon/ion/impl/macro/Environment.kt | 2 +- .../amazon/ion/impl/macro/MacroEvaluator.kt | 629 +++++++++--------- .../com/amazon/ion/impl/macro/SystemMacro.kt | 14 +- .../ion/impl/IonRawTextWriterTest_1_1.kt | 6 +- 5 files changed, 349 insertions(+), 304 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt index bd6bfcbad..7848944fc 100644 --- a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt @@ -628,7 +628,7 @@ internal class IonManagedWriter_1_1( is Expression.MacroInvocation -> { val invokedMacro = expression.macro if (invokedMacro is SystemMacro) { - stepInTdlSystemMacroInvocation(invokedMacro.systemSymbol!!) + stepInTdlSystemMacroInvocation(invokedMacro.systemSymbol) } else { val invokedAddress = macroTable[invokedMacro] ?: newMacros[invokedMacro] diff --git a/src/main/java/com/amazon/ion/impl/macro/Environment.kt b/src/main/java/com/amazon/ion/impl/macro/Environment.kt index 18c854c89..d348612a1 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Environment.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Environment.kt @@ -29,7 +29,7 @@ data class Environment private constructor( | ], | parent: ${parentEnvironment.toString().lines().joinToString("\n| ")}, |) - """.trimMargin() + """.trimMargin() companion object { @JvmStatic diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index b3ccfc5fc..af3ecf589 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -6,7 +6,8 @@ import com.amazon.ion.* import com.amazon.ion.impl._Private_RecyclingStack import com.amazon.ion.impl._Private_Utils.* import com.amazon.ion.impl.macro.Expression.* -import com.amazon.ion.impl.macro.MacroEvaluator.ExpanderKind.* +import com.amazon.ion.impl.macro.MacroEvaluator.ExpanderKind.Stream +import com.amazon.ion.impl.macro.MacroEvaluator.ExpanderKind.Uninitialized import com.amazon.ion.util.* import java.io.ByteArrayOutputStream import java.lang.StringBuilder @@ -23,24 +24,49 @@ import java.math.BigInteger * if the end of the container or end of expansion has been reached. * - Call [stepIn] when positioned on a container to step into that container. * - Call [stepOut] to step out of the current container. + * + * TODO: Make expansion limit configurable. + * + * ### Implementation Overview: + * + * The macro evaluator can be thought of as a stack of containers where each container has a stack of expansion frames + * (i.e. [ExpansionInfo]). To get the next value at the current depth, the macro evaluator starts with the bottom frame + * of the top container. Expansion frames may delegate to and/or intercept other expansions that are further up the stack. + * + * One might visualize it like this: + * ``` + * 3. List : Stream --> Delta --> Variable + * 2. List : Stream --> Flatten --> Stream + * 1. Struct : Stream --> Variable --> TemplateBody --> Stream --> TemplateBody + * 0. TopLevel : Stream --> TemplateBody --> TemplateBody + * ``` + * + * When calling [expandNext], the evaluator looks at the first expansion frame of the top container in the stack. + * Then it calls `produceNext` for the first expansion in that container. That expansion may produce a result all on its + * own, or it may call the next expansion in the chain and return that value (with or without further modification). + * + * In practice, it is a little more complex. A single expansion frame may hold more than one child frame sequentially + * (e.g. `repeat`, `annotate`) or concurrently (e.g. `for`, `flatten`). */ class MacroEvaluator { - private var numExpandedExpressions = 0 - private val expansionLimit: Int = 1_000_000 - private val expanderPool: ArrayList = ArrayList(32) - - // TODO(PERF): Does it improve performance if we make this an `inner` class and remove the evaluator field? - class MacroEvaluationSession(private val evaluator: MacroEvaluator) { - - fun getExpander( - expanderKind: ExpanderKind, - expressions: List, - startInclusive: Int, - endExclusive: Int, - environment: Environment, - ): ExpansionFrame { - val expansion = evaluator.expanderPool.removeLastOrNull() ?: ExpansionFrame(this) + /** + * Holds state that is shared across all macro evaluations that are part of this evaluator. + * This state pertains to a single "session" of the macro evaluator, and is reset every time [initExpansion] is called. + * For now, this includes managing the pool of [ExpansionInfo] and tracking the expansion step limit. + */ + private class Session( + /** Number of expansion steps at which the evaluation session should be aborted. */ + private val expansionLimit: Int = 1_000_000 + ) { + /** Internal state for tracking the number of expansion steps. */ + private var numExpandedExpressions = 0 + /** Pool of [ExpansionInfo] to minimize allocation and garbage collection. */ + private val expanderPool: ArrayList = ArrayList(32) + + /** Gets an [ExpansionInfo] from the pool (or allocates a new one if necessary), initializing it with the provided values. */ + fun getExpander(expanderKind: ExpanderKind, expressions: List, startInclusive: Int, endExclusive: Int, environment: Environment): ExpansionInfo { + val expansion = expanderPool.removeLastOrNull() ?: ExpansionInfo(this) expansion.expanderKind = expanderKind expansion.expressions = expressions expansion.i = startInclusive @@ -51,32 +77,40 @@ class MacroEvaluator { return expansion } - fun returnExpander(ex: ExpansionFrame) { - // TODO: This check is O(n). Remove this when confident. - check(ex !in evaluator.expanderPool) - evaluator.expanderPool.add(ex) + /** Reclaims an [ExpansionInfo] to the available pool. */ + fun reclaimExpander(ex: ExpansionInfo) { + // TODO: This check is O(n). Consider removing this when confident there are no double frees. + check(ex !in expanderPool) + expanderPool.add(ex) } fun incrementStepCounter() { - evaluator.numExpandedExpressions++ - if (evaluator.numExpandedExpressions > evaluator.expansionLimit) { + numExpandedExpressions++ + if (numExpandedExpressions > expansionLimit) { // Technically, we are not counting "steps" because we don't have a true definition of what a "step" is, // but this is probably a more user-friendly message than trying to explain what we're actually counting. - throw IonException("Macro expansion exceeded limit of ${evaluator.expansionLimit} steps.") + throw IonException("Macro expansion exceeded limit of $expansionLimit steps.") } } + + fun reset() { + numExpandedExpressions = 0 + } } - private data class ContainerInfo(var type: Type = Type.Uninitialized, private var _expansion: ExpansionFrame? = null) { + /** + * A container in the macro evaluator's [containerStack]. + */ + private data class ContainerInfo(var type: Type = Type.Uninitialized, private var _expansion: ExpansionInfo? = null) { enum class Type { TopLevel, List, Sexp, Struct, Uninitialized } - fun releaseResources() { - _expansion?.drop() + fun close() { + _expansion?.close() _expansion = null type = Type.Uninitialized } - var expansion: ExpansionFrame + var expansion: ExpansionInfo get() = _expansion!! set(value) { _expansion = value } @@ -85,122 +119,20 @@ class MacroEvaluator { } } - private val session = MacroEvaluationSession(this) - private val containerStack = _Private_RecyclingStack(8) { ContainerInfo() } - private var currentExpr: DataModelExpression? = null - - private fun resetSession() { this.numExpandedExpressions = 0 } - - fun getArguments(): List { - return containerStack.iterator().next().expansion.expressions - } - /** - * Initialize the macro evaluator with an E-Expression. + * Stateless functions that operate on the expansion frames (i.e. [ExpansionInfo]). */ - fun initExpansion(encodingExpressions: List) { - resetSession() - containerStack.push { ci -> - ci.type = ContainerInfo.Type.TopLevel - ci.expansion = session.getExpander(Stream, encodingExpressions, 0, encodingExpressions.size, Environment.EMPTY) - } - } - - /** - * Evaluate the macro expansion until the next [DataModelExpression] can be returned. - * Returns null if at the end of a container or at the end of the expansion. - */ - fun expandNext(): DataModelExpression? { - currentExpr = null - while (currentExpr == null && !containerStack.isEmpty()) { - val currentContainer = containerStack.peek() - - when (val nextExpansionOutput = currentContainer.produceNext()) { - is DataModelExpression -> currentExpr = nextExpansionOutput - EndOfExpansion -> { - if (currentContainer.type == ContainerInfo.Type.TopLevel) { - currentContainer.releaseResources() - containerStack.pop() - } - return null - } - } - } - return currentExpr - } - - /** - * Steps out of the current [DataModelContainer]. - */ - fun stepOut() { - // TODO: We should be able to step out of a "TopLevel" container and/or we need some way to close the evaluation. - if (containerStack.size() <= 1) throw IonException("Nothing to step out of.") - val popped = containerStack.pop() - popped.releaseResources() - } - - /** - * Steps in to the current [DataModelContainer]. - * Throws [IonException] if not positioned on a container. - */ - fun stepIn() { - val expression = requireNotNull(currentExpr) { "Not positioned on a value" } - if (expression is DataModelContainer) { - val currentContainer = containerStack.peek() - val topExpansion = currentContainer.expansion.top() - containerStack.push { ci -> - ci.type = when (expression.type) { - IonType.LIST -> ContainerInfo.Type.List - IonType.SEXP -> ContainerInfo.Type.Sexp - IonType.STRUCT -> ContainerInfo.Type.Struct - else -> unreachable() - } - ci.expansion = session.getExpander( - expanderKind = Stream, - expressions = topExpansion.expressions, - startInclusive = expression.startInclusive, - endExclusive = expression.endExclusive, - environment = topExpansion.environment, - ) - } - currentExpr = null - } else { - throw IonException("Not positioned on a container.") - } - } - // TODO(PERF): It might be possible to optimize this by changing it to an enum without any methods (or even a set of // integer constants) and converting all their implementations to static methods. - enum class ExpanderKind { + private enum class ExpanderKind { Uninitialized { - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { - throw IllegalStateException("ExpansionInfo not initialized.") - } + override fun produceNext(thisExpansion: ExpansionInfo): Nothing = throw IllegalStateException("ExpansionInfo not initialized.") }, Empty { - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue = EndOfExpansion + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue = EndOfExpansion }, Stream { - private fun invokeMacro(thisExpansion: ExpansionFrame, macro: Macro, next: HasStartAndEnd): ContinueExpansion { - val argIndices = macro.calculateArgumentIndices( - encodingExpressions = thisExpansion.expressions, - argsStartInclusive = next.startInclusive, - argsEndExclusive = next.endExclusive, - ) - val newEnvironment = thisExpansion.environment.createChild(thisExpansion.expressions, argIndices) - val expanderKind = if (macro.body != null) Stream else getExpanderKindForSystemMacro(macro as SystemMacro) - thisExpansion.expansionDelegate = thisExpansion.session.getExpander( - expanderKind = expanderKind, - expressions = macro.body ?: emptyList(), - startInclusive = 0, - endExclusive = macro.body?.size ?: 0, - environment = newEnvironment, - ) - - return ContinueExpansion - } - - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { // If there's a delegate, we'll try that first. val delegate = thisExpansion.expansionDelegate check(thisExpansion != delegate) @@ -208,7 +140,7 @@ class MacroEvaluator { return when (val result = delegate.produceNext()) { is DataModelExpression -> result EndOfExpansion -> { - delegate.drop() + delegate.close() thisExpansion.expansionDelegate = null ContinueExpansion } @@ -226,11 +158,23 @@ class MacroEvaluator { return when (next) { is DataModelExpression -> next - is EExpression -> invokeMacro(thisExpansion, next.macro, next) - is MacroInvocation -> invokeMacro(thisExpansion, next.macro, next) + is InvokableExpression -> { + val macro = next.macro + val argIndices = calculateArgumentIndices(macro, thisExpansion.expressions, next.startInclusive, next.endExclusive) + val newEnvironment = thisExpansion.environment.createChild(thisExpansion.expressions, argIndices) + val expanderKind = ExpanderKind.forMacro(macro) + thisExpansion.expansionDelegate = thisExpansion.session.getExpander( + expanderKind = expanderKind, + expressions = macro.body ?: emptyList(), + startInclusive = 0, + endExclusive = macro.body?.size ?: 0, + environment = newEnvironment, + ) + ContinueExpansion + } is ExpressionGroup -> { thisExpansion.expansionDelegate = thisExpansion.session.getExpander( - expanderKind = Stream, + expanderKind = ExprGroup, expressions = thisExpansion.expressions, startInclusive = next.startInclusive, endExclusive = next.endExclusive, @@ -248,41 +192,26 @@ class MacroEvaluator { } } }, - // TODO: Move this into the variable expansion? - ExactlyOneValueStream { - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { - if (thisExpansion.additionalState != 1) { - return when (val firstValue = Stream.produceNext(thisExpansion)) { - is DataModelExpression -> { - thisExpansion.additionalState = 1 - firstValue - } - ContinueExpansion -> ContinueExpansion - EndOfExpansion -> throw IonException("Expected one value, found 0") - } - } else { - return when (val secondValue = Stream.produceNext(thisExpansion)) { - is DataModelExpression -> throw IonException("Expected one value, found multiple") - ContinueExpansion -> ContinueExpansion - EndOfExpansion -> secondValue - } - } + /** Alias of [Stream] to aid in debugging */ + Variable { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { + return Stream.produceNext(thisExpansion) } }, - NonEmptyStream { - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { - return when (val firstValue = Stream.produceNext(thisExpansion)) { - EndOfExpansion -> throw IonException("Expected at least one value, found 0") - ContinueExpansion -> ContinueExpansion - is DataModelExpression -> { - thisExpansion.expanderKind = Stream - firstValue - } - } + /** Alias of [Stream] to aid in debugging */ + TemplateBody { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { + return Stream.produceNext(thisExpansion) } }, - AtMostOneValueStream { - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + /** Alias of [Stream] to aid in debugging */ + ExprGroup { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { + return Stream.produceNext(thisExpansion) + } + }, + ExactlyOneValueStream { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { if (thisExpansion.additionalState != 1) { return when (val firstValue = Stream.produceNext(thisExpansion)) { is DataModelExpression -> { @@ -290,7 +219,7 @@ class MacroEvaluator { firstValue } ContinueExpansion -> ContinueExpansion - EndOfExpansion -> EndOfExpansion + EndOfExpansion -> throw IonException("Expected one value, found 0") } } else { return when (val secondValue = Stream.produceNext(thisExpansion)) { @@ -303,23 +232,23 @@ class MacroEvaluator { }, IfNone { - override fun produceNext(thisExpansion: ExpansionFrame) = checkExpansionSize(thisExpansion) { it == 0 } + override fun produceNext(thisExpansion: ExpansionInfo) = thisExpansion.branchIf { it == 0 } }, IfSome { - override fun produceNext(thisExpansion: ExpansionFrame) = checkExpansionSize(thisExpansion) { it > 0 } + override fun produceNext(thisExpansion: ExpansionInfo) = thisExpansion.branchIf { it > 0 } }, IfSingle { - override fun produceNext(thisExpansion: ExpansionFrame) = checkExpansionSize(thisExpansion) { it == 1 } + override fun produceNext(thisExpansion: ExpansionInfo) = thisExpansion.branchIf { it == 1 } }, IfMulti { - override fun produceNext(thisExpansion: ExpansionFrame) = checkExpansionSize(thisExpansion) { it > 1 } + override fun produceNext(thisExpansion: ExpansionInfo) = thisExpansion.branchIf { it > 1 } }, Annotate { private val ANNOTATIONS_ARG = VariableRef(0) private val VALUE_TO_ANNOTATE_ARG = VariableRef(1) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { val annotations = thisExpansion.map(ANNOTATIONS_ARG) { when (it) { is StringValue -> newSymbolToken(it.value) @@ -335,17 +264,19 @@ class MacroEvaluator { it as? DataModelValue ?: throw IonException("Required at least one value.") it.withAnnotations(annotations + it.annotations) } + if (valueToAnnotateExpansion.produceNext() != EndOfExpansion) { + throw IonException("Can only annotate exactly one value") + } return annotatedExpression.also { thisExpansion.tailCall(valueToAnnotateExpansion) - thisExpansion.expanderKind = ExactlyOneValueStream } } }, MakeString { private val STRINGS_ARG = VariableRef(0) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { val sb = StringBuilder() thisExpansion.forEach(STRINGS_ARG) { when (it) { @@ -362,7 +293,7 @@ class MacroEvaluator { MakeSymbol { private val STRINGS_ARG = VariableRef(0) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { if (thisExpansion.additionalState != null) return EndOfExpansion thisExpansion.additionalState = Unit @@ -381,10 +312,7 @@ class MacroEvaluator { MakeBlob { private val LOB_ARG = VariableRef(0) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { - if (thisExpansion.additionalState != null) return EndOfExpansion - thisExpansion.additionalState = Unit - + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { val baos = ByteArrayOutputStream() thisExpansion.forEach(LOB_ARG) { when (it) { @@ -393,6 +321,7 @@ class MacroEvaluator { is FieldName -> unreachable() } } + thisExpansion.expanderKind = Empty return BlobValue(value = baos.toByteArray()) } }, @@ -400,12 +329,10 @@ class MacroEvaluator { private val COEFFICIENT_ARG = VariableRef(0) private val EXPONENT_ARG = VariableRef(1) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { - if (thisExpansion.additionalState != null) return EndOfExpansion - thisExpansion.additionalState = Unit - + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { val coefficient = thisExpansion.readExactlyOneArgument(COEFFICIENT_ARG).bigIntegerValue val exponent = thisExpansion.readExactlyOneArgument(EXPONENT_ARG).bigIntegerValue + thisExpansion.expanderKind = Empty return DecimalValue(value = BigDecimal(coefficient, -1 * exponent.intValueExact())) } }, @@ -418,7 +345,7 @@ class MacroEvaluator { private val SECOND_ARG = VariableRef(5) private val OFFSET_ARG = VariableRef(6) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { val year = thisExpansion.readExactlyOneArgument(YEAR_ARG).longValue.toInt() val month = thisExpansion.readZeroOrOneArgument(MONTH_ARG)?.longValue?.toInt() val day = thisExpansion.readZeroOrOneArgument(DAY_ARG)?.longValue?.toInt() @@ -470,7 +397,7 @@ class MacroEvaluator { private val FIELD_NAME = VariableRef(0) private val FIELD_VALUE = VariableRef(1) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { val fieldName = thisExpansion.readExactlyOneArgument(FIELD_NAME) val fieldNameExpression = when (fieldName) { is SymbolValue -> FieldName(fieldName.value) @@ -491,8 +418,8 @@ class MacroEvaluator { _Private_FlattenStruct { private val STRUCTS = VariableRef(0) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { - var argumentExpansion: ExpansionFrame? = thisExpansion.additionalState as ExpansionFrame? + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { + var argumentExpansion: ExpansionInfo? = thisExpansion.additionalState as ExpansionInfo? if (argumentExpansion == null) { argumentExpansion = thisExpansion.readArgument(STRUCTS) thisExpansion.additionalState = argumentExpansion @@ -502,7 +429,7 @@ class MacroEvaluator { return when (val next = currentChildExpansion?.produceNext()) { is DataModelExpression -> next - EndOfExpansion -> thisExpansion.dropDelegateAndContinue() + EndOfExpansion -> thisExpansion.closeDelegateAndContinue() // Only possible if expansionDelegate is null null -> when (val nextSequence = argumentExpansion.produceNext()) { is StructValue -> { @@ -522,11 +449,16 @@ class MacroEvaluator { } }, + /** + * Iterates over the sequences, returning the values contained in the sequences. + * The expansion for the sequences argument is stored in [ExpansionInfo.additionalState]. + * When + */ Flatten { private val SEQUENCES = VariableRef(0) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { - var argumentExpansion: ExpansionFrame? = thisExpansion.additionalState as ExpansionFrame? + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { + var argumentExpansion: ExpansionInfo? = thisExpansion.additionalState as ExpansionInfo? if (argumentExpansion == null) { argumentExpansion = thisExpansion.readArgument(SEQUENCES) thisExpansion.additionalState = argumentExpansion @@ -536,7 +468,7 @@ class MacroEvaluator { return when (val next = currentChildExpansion?.produceNext()) { is DataModelExpression -> next - EndOfExpansion -> thisExpansion.dropDelegateAndContinue() + EndOfExpansion -> thisExpansion.closeDelegateAndContinue() // Only possible if expansionDelegate is null null -> when (val nextSequence = argumentExpansion.produceNext()) { is StructValue -> throw IonException("invalid argument; flatten expects sequences") @@ -561,21 +493,19 @@ class MacroEvaluator { private val ARG_A = VariableRef(0) private val ARG_B = VariableRef(1) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { - if (thisExpansion.additionalState != null) return EndOfExpansion - thisExpansion.additionalState = Unit - + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { + // TODO(PERF): consider checking whether the value would fit in a long and returning a `LongIntValue`. val a = thisExpansion.readExactlyOneArgument(ARG_A).bigIntegerValue val b = thisExpansion.readExactlyOneArgument(ARG_B).bigIntegerValue + thisExpansion.expanderKind = Empty return BigIntValue(value = a + b) } }, Delta { private val ARGS = VariableRef(0) - // Initial value = 0 - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { - // TODO: Optimize to use LongIntValue when possible + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { + // TODO(PERF): Optimize to use LongIntValue when possible var delegate = thisExpansion.expansionDelegate val runningTotal = thisExpansion.additionalState as? BigInteger ?: BigInteger.ZERO if (delegate == null) { @@ -590,9 +520,8 @@ class MacroEvaluator { thisExpansion.additionalState = nextOutput return BigIntValue(value = nextOutput) } - EndOfExpansion -> return nextExpandedArg - is DataModelValue -> throw IonException("delta arguments must be integers") - is FieldName -> unreachable() + EndOfExpansion -> return EndOfExpansion + else -> throw IonException("delta arguments must be integers") } } }, @@ -600,7 +529,7 @@ class MacroEvaluator { private val COUNT_ARG = VariableRef(0) private val THING_TO_REPEAT = VariableRef(1) - override fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue { + override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { var n = thisExpansion.additionalState as Long? if (n == null) { n = thisExpansion.readExactlyOneArgument(COUNT_ARG).longValue @@ -620,52 +549,42 @@ class MacroEvaluator { val repeated = thisExpansion.expansionDelegate!! return when (val maybeNext = repeated.produceNext()) { is DataModelExpression -> maybeNext - EndOfExpansion -> thisExpansion.dropDelegateAndContinue() + EndOfExpansion -> thisExpansion.closeDelegateAndContinue() } } }, ; - abstract fun produceNext(thisExpansion: ExpansionFrame): ExpansionOutputExpressionOrContinue + /** + * Produces the next value, [EndOfExpansion], or [ContinueExpansion]. + * Each enum variant must implement this method. + */ + abstract fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue - internal inline fun checkExpansionSize(thisExpansion: ExpansionFrame, condition: (Int) -> Boolean): ContinueExpansion { + /** Helper function for the `if_*` macros */ + inline fun ExpansionInfo.branchIf(condition: (Int) -> Boolean): ContinueExpansion { val argToTest = VariableRef(0) val trueBranch = VariableRef(1) val falseBranch = VariableRef(2) - val testArg = thisExpansion.readArgument(argToTest) + val testArg = readArgument(argToTest) var n = 0 while (n < 2) { if (testArg.produceNext() is EndOfExpansion) break n++ } - testArg.drop() + testArg.close() val branch = if (condition(n)) trueBranch else falseBranch - val branchExpansion = thisExpansion.readArgument(branch) - thisExpansion.tailCall(branchExpansion) + tailCall(readArgument(branch)) return ContinueExpansion } - internal fun VariableRef.readFrom(environment: Environment, session: MacroEvaluationSession): ExpansionFrame { - val argIndex = environment.argumentIndices[signatureIndex] - if (argIndex < 0) { - // Argument was elided. - return session.getExpander(Empty, emptyList(), 0, 0, Environment.EMPTY) - } - val firstArgExpression = environment.arguments[argIndex] - - return session.getExpander( - expanderKind = Stream, - expressions = environment.arguments, - startInclusive = if (firstArgExpression is ExpressionGroup) firstArgExpression.startInclusive else argIndex, - endExclusive = if (firstArgExpression is HasStartAndEnd) firstArgExpression.endExclusive else argIndex + 1, - environment = environment.parentEnvironment!! - ) - } - - internal fun ExpansionFrame.readArgument(variableRef: VariableRef): ExpansionFrame { + /** + * Returns an expansion for the given variable. + */ + fun ExpansionInfo.readArgument(variableRef: VariableRef): ExpansionInfo { val argIndex = environment.argumentIndices[variableRef.signatureIndex] if (argIndex < 0) { // Argument was elided. @@ -673,7 +592,7 @@ class MacroEvaluator { } val firstArgExpression = environment.arguments[argIndex] return session.getExpander( - expanderKind = Stream, + expanderKind = Variable, expressions = environment.arguments, startInclusive = if (firstArgExpression is ExpressionGroup) firstArgExpression.startInclusive else argIndex, endExclusive = if (firstArgExpression is HasStartAndEnd) firstArgExpression.endExclusive else argIndex + 1, @@ -681,7 +600,10 @@ class MacroEvaluator { ) } - internal inline fun ExpansionFrame.forEach(variableRef: VariableRef, action: (DataModelExpression) -> Unit) { + /** + * Performs the given [action] for each value produced by the expansion of [variableRef]. + */ + inline fun ExpansionInfo.forEach(variableRef: VariableRef, action: (DataModelExpression) -> Unit) { val variableExpansion = readArgument(variableRef) while (true) { when (val next = variableExpansion.produceNext()) { @@ -691,18 +613,27 @@ class MacroEvaluator { } } - internal inline fun ExpansionFrame.map(variableRef: VariableRef, action: (DataModelExpression) -> T): List { + /** + * Performs the given [transform] on each value produced by the expansion of [variableRef], returning a list + * of the results. + */ + inline fun ExpansionInfo.map(variableRef: VariableRef, transform: (DataModelExpression) -> T): List { val variableExpansion = readArgument(variableRef) val result = mutableListOf() while (true) { when (val next = variableExpansion.produceNext()) { EndOfExpansion -> return result - is DataModelExpression -> result.add(action(next)) + is DataModelExpression -> result.add(transform(next)) } } } - internal inline fun ExpansionFrame.readZeroOrOneArgument(variableRef: VariableRef): T? { + /** + * Reads and returns zero or one values from the expansion of the given [variableRef]. + * Throws an [IonException] if more than one value is present in the variable expansion. + * Throws an [IonException] if the value is not the expected type [T]. + */ + inline fun ExpansionInfo.readZeroOrOneArgument(variableRef: VariableRef): T? { val argExpansion = readArgument(variableRef) var argValue: T? = null while (true) { @@ -717,43 +648,58 @@ class MacroEvaluator { is FieldName -> unreachable("Unreachable without stepping into a container") } } + argExpansion.close() return argValue } - internal inline fun ExpansionFrame.readExactlyOneArgument(variableRef: VariableRef): T { + /** + * Reads and returns exactly one value from the expansion of the given [variableRef]. + * Throws an [IonException] if the expansion of [variableRef] does not produce exactly one value. + * Throws an [IonException] if the value is not the expected type [T]. + */ + inline fun ExpansionInfo.readExactlyOneArgument(variableRef: VariableRef): T { return readZeroOrOneArgument(variableRef) ?: throw IonException("invalid argument; no value when one is expected") } companion object { + /** + * Gets the [ExpanderKind] for the given [macro]. + */ @JvmStatic - fun getExpanderKindForSystemMacro(systemMacro: SystemMacro) = when (systemMacro) { - SystemMacro.Annotate -> Annotate - SystemMacro.MakeString -> MakeString - SystemMacro.MakeSymbol -> MakeSymbol - SystemMacro.MakeDecimal -> MakeDecimal - SystemMacro.Repeat -> Repeat - SystemMacro.Sum -> Sum - SystemMacro.Delta -> Delta - SystemMacro.MakeBlob -> MakeBlob - SystemMacro.Flatten -> Flatten - SystemMacro._Private_FlattenStruct -> _Private_FlattenStruct - SystemMacro.MakeTimestamp -> MakeTimestamp - SystemMacro._Private_MakeFieldNameAndValue -> _Private_MakeFieldNameAndValue - SystemMacro.IfNone -> IfNone - SystemMacro.IfSome -> IfSome - SystemMacro.IfSingle -> IfSingle - SystemMacro.IfMulti -> IfMulti - else -> if (systemMacro.body != null) { - throw IllegalStateException("SystemMacro ${systemMacro.name} should be using its template body.") - } else { - TODO("Not implemented yet: ${systemMacro.name}") + fun forMacro(macro: Macro): ExpanderKind { + return if (macro.body != null) { + TemplateBody + } else when (macro as SystemMacro) { + SystemMacro.Annotate -> Annotate + SystemMacro.MakeString -> MakeString + SystemMacro.MakeSymbol -> MakeSymbol + SystemMacro.MakeDecimal -> MakeDecimal + SystemMacro.Repeat -> Repeat + SystemMacro.Sum -> Sum + SystemMacro.Delta -> Delta + SystemMacro.MakeBlob -> MakeBlob + SystemMacro.Flatten -> Flatten + SystemMacro._Private_FlattenStruct -> _Private_FlattenStruct + SystemMacro.MakeTimestamp -> MakeTimestamp + SystemMacro._Private_MakeFieldNameAndValue -> _Private_MakeFieldNameAndValue + SystemMacro.IfNone -> IfNone + SystemMacro.IfSome -> IfSome + SystemMacro.IfSingle -> IfSingle + SystemMacro.IfMulti -> IfMulti + else -> TODO("Not implemented yet: ${macro.name}") } } } } - class ExpansionFrame( - @JvmField val session: MacroEvaluationSession, + /** + * Represents a frame in the expansion stack for a particular container. + * + * TODO: "info" is very non-specific; rename to ExpansionFrame next time there's a + * non-functional refactoring in this class. + */ + private class ExpansionInfo( + @JvmField val session: Session, @JvmField var expanderKind: ExpanderKind = Uninitialized, /** * The [Expression]s being expanded. This MUST be the original list, not a sublist because @@ -763,56 +709,56 @@ class MacroEvaluator { @JvmField var expressions: List = emptyList(), /** Current position within [expressions] of this expansion */ @JvmField var i: Int = 0, - /** End of [expressions] that are applicable for this [ExpansionFrame] */ + /** End of [expressions] that are applicable for this [ExpansionInfo] */ @JvmField var endExclusive: Int = 0, /** The evaluation [Environment]—i.e. variable bindings. */ @JvmField var environment: Environment = Environment.EMPTY, - @JvmField var _expansionDelegate: ExpansionFrame? = null, + _expansionDelegate: ExpansionInfo? = null, @JvmField var additionalState: Any? = null, ) { - var expansionDelegate: ExpansionFrame? - get() = _expansionDelegate + // TODO: if expansionDelegate == this, it will cause an infinite loop or stack overflow somewhere. + // In practice, it should never happen, so we may wish to remove the custom setter to avoid any performance impact. + var expansionDelegate: ExpansionInfo? = _expansionDelegate set(value) { check(value != this) - _expansionDelegate = value + field = value } - fun dropDelegateAndContinue(): ContinueExpansion { - expansionDelegate?.drop() + /** + * Convenience function to close the [expansionDelegate] and return it to the pool. + */ + fun closeDelegateAndContinue(): ContinueExpansion { + expansionDelegate?.close() expansionDelegate = null return ContinueExpansion } - fun top(): ExpansionFrame = expansionDelegate?.top() ?: this + /** + * Gets the [ExpansionInfo] at the top of the stack of [expansionDelegate]s. + */ + fun top(): ExpansionInfo = expansionDelegate?.top() ?: this - fun drop() { + /** + * Returns this [ExpansionInfo] to the expander pool, recursively closing [expansionDelegate]s in the process. + * Could also be thought of as a `free` function. + */ + fun close() { expanderKind = Uninitialized - additionalState = null environment = Environment.EMPTY expressions = emptyList() - expansionDelegate?.drop() - expansionDelegate = null - session.returnExpander(this) - } - - fun initExpansion( - expanderKind: ExpanderKind, - expressions: List, - startInclusive: Int, - endExclusive: Int, - environment: Environment, - ) { - this.expanderKind = expanderKind - this.expressions = expressions - this.i = startInclusive - this.endExclusive = endExclusive - this.environment = environment + additionalState?.let { if (it is ExpansionInfo) it.close() } additionalState = null + expansionDelegate?.close() expansionDelegate = null + session.reclaimExpander(this) } - fun tailCall(other: ExpansionFrame) { + /** + * Replaces the state of `this` [ExpansionInfo] with the state of [other]—effectively a tail-call optimization. + * After transferring the state, `other` is returned to the expansion pool. + */ + fun tailCall(other: ExpansionInfo) { this.expanderKind = other.expanderKind this.expressions = other.expressions this.i = other.i @@ -820,26 +766,35 @@ class MacroEvaluator { this.expansionDelegate = other.expansionDelegate this.additionalState = other.additionalState this.environment = other.environment - // Drop `other` + // Close `other` other.expansionDelegate = null - other.drop() + other.close() } + /** + * Produces the next value from this expansion. + */ fun produceNext(): ExpansionOutputExpression { while (true) { val next = expanderKind.produceNext(this) if (next is ContinueExpansion) continue + // This the only place where we count the expansion steps. + // It is theoretically possible to have macro expansions that are millions of levels deep because this + // only counts macro invocations at the end of their expansion, but this will still work to catch things + // like a billion laughs attack because it does place a limit on the number of _values_ produced. + // This counts every value _at every level_, so most values will be counted multiple times. If possible + // without impacting performance, count values only once in order to have more predictable behavior. session.incrementStepCounter() return next as ExpansionOutputExpression } } override fun toString() = """ - |ExpansionFrame( + |ExpansionInfo( | expansionKind: $expanderKind, | environment: ${environment.toString().lines().joinToString("\n| ")}, | expressions: [ - | ${expressions.mapIndexed { index, expression -> "$index. $expression" }.joinToString(",\n| ") { it.toString() } } + | ${expressions.mapIndexed { i, expr -> "$i. $expr" }.joinToString(",\n| ") } | ], | endExclusive: $endExclusive, | i: $i, @@ -848,6 +803,87 @@ class MacroEvaluator { |) """.trimMargin() } + + private val session = Session(expansionLimit = 1_000_000) + private val containerStack = _Private_RecyclingStack(8) { ContainerInfo() } + private var currentExpr: DataModelExpression? = null + + /** + * Returns the e-expression argument expressions that this MacroEvaluator would evaluate. + */ + fun getArguments(): List { + return containerStack.iterator().next().expansion.expressions + } + + /** + * Initialize the macro evaluator with an E-Expression. + */ + fun initExpansion(encodingExpressions: List) { + session.reset() + containerStack.push { ci -> + ci.type = ContainerInfo.Type.TopLevel + ci.expansion = session.getExpander(Stream, encodingExpressions, 0, encodingExpressions.size, Environment.EMPTY) + } + } + + /** + * Evaluate the macro expansion until the next [DataModelExpression] can be returned. + * Returns null if at the end of a container or at the end of the expansion. + */ + fun expandNext(): DataModelExpression? { + currentExpr = null + val currentContainer = containerStack.peek() + when (val nextExpansionOutput = currentContainer.produceNext()) { + is DataModelExpression -> currentExpr = nextExpansionOutput + EndOfExpansion -> { + if (currentContainer.type == ContainerInfo.Type.TopLevel) { + currentContainer.close() + containerStack.pop() + } + } + } + return currentExpr + } + + /** + * Steps out of the current [DataModelContainer]. + */ + fun stepOut() { + // TODO: We should be able to step out of a "TopLevel" container and/or we need some way to close the evaluation early. + if (containerStack.size() <= 1) throw IonException("Nothing to step out of.") + val popped = containerStack.pop() + popped.close() + } + + /** + * Steps in to the current [DataModelContainer]. + * Throws [IonException] if not positioned on a container. + */ + fun stepIn() { + val expression = requireNotNull(currentExpr) { "Not positioned on a value" } + if (expression is DataModelContainer) { + val currentContainer = containerStack.peek() + val topExpansion = currentContainer.expansion.top() + containerStack.push { ci -> + ci.type = when (expression.type) { + IonType.LIST -> ContainerInfo.Type.List + IonType.SEXP -> ContainerInfo.Type.Sexp + IonType.STRUCT -> ContainerInfo.Type.Struct + else -> unreachable() + } + ci.expansion = session.getExpander( + expanderKind = Stream, + expressions = topExpansion.expressions, + startInclusive = expression.startInclusive, + endExclusive = expression.endExclusive, + environment = topExpansion.environment, + ) + } + currentExpr = null + } else { + throw IonException("Not positioned on a container.") + } + } } /** @@ -859,17 +895,18 @@ class MacroEvaluator { * This function also validates that the correct number of parameters are present. If there are * too many parameters or too few parameters, this will throw [IonException]. */ -private fun Macro.calculateArgumentIndices( +private fun calculateArgumentIndices( + macro: Macro, encodingExpressions: List, argsStartInclusive: Int, argsEndExclusive: Int ): List { // TODO: For TDL macro invocations, see if we can calculate this during the "compile" step. var numArgs = 0 - val argsIndices = IntArray(signature.size) + val argsIndices = IntArray(macro.signature.size) var currentArgIndex = argsStartInclusive - for (p in signature) { + for (p in macro.signature) { if (currentArgIndex >= argsEndExclusive) { if (!p.cardinality.canBeVoid) throw IonException("No value provided for parameter ${p.variableName}") // Elided rest parameter. @@ -890,8 +927,8 @@ private fun Macro.calculateArgumentIndices( } numArgs++ } - if (numArgs > signature.size) { - throw IonException("Too many arguments. Expected ${signature.size}, but found $numArgs") + if (numArgs > macro.signature.size) { + throw IonException("Too many arguments. Expected ${macro.signature.size}, but found $numArgs") } return argsIndices.toList() } diff --git a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt index 67cead5b3..8d3c997a4 100644 --- a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt @@ -15,7 +15,7 @@ import com.amazon.ion.impl.macro.ParameterFactory.zeroToManyTagged */ enum class SystemMacro( val id: Byte, - val systemSymbol: SystemSymbols_1_1?, + private val _systemSymbol: SystemSymbols_1_1?, override val signature: List, override val body: List? = null ) : Macro { @@ -27,8 +27,9 @@ enum class SystemMacro( IfMulti(-1, IF_MULTI, listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), // Unnameable, unaddressable macros used for the internals of certain other system macros - _Private_FlattenStruct(-1, systemSymbol = null, listOf(zeroToManyTagged("structs"))), - _Private_MakeFieldNameAndValue(-1, systemSymbol = null, listOf(exactlyOneTagged("fieldName"), exactlyOneTagged("value"))), + // TODO: See if we can move these somewhere else so that they are not visible + _Private_FlattenStruct(-1, _systemSymbol = null, listOf(zeroToManyTagged("structs"))), + _Private_MakeFieldNameAndValue(-1, _systemSymbol = null, listOf(exactlyOneTagged("fieldName"), exactlyOneTagged("value"))), // The real macros Values(1, VALUES, listOf(zeroToManyTagged("values")), templateBody { variable(0) }), @@ -260,7 +261,10 @@ enum class SystemMacro( ), ; - val macroName: String get() = this.systemSymbol?.text ?: throw IllegalStateException("Attempt to get name for unaddressable macro $name") + val systemSymbol: SystemSymbols_1_1 + get() = _systemSymbol ?: throw IllegalStateException("Attempted to get name for unaddressable macro $name") + + val macroName: String get() = this.systemSymbol.text override val dependencies: List get() = body @@ -272,7 +276,7 @@ enum class SystemMacro( companion object : MacroTable { private val MACROS_BY_NAME: Map = SystemMacro.entries - .filter { it.systemSymbol != null } + .filter { it._systemSymbol != null } .associateBy { it.macroName } // TODO: Once all of the macros are implemented, replace this with an array as in SystemSymbols_1_1 diff --git a/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt b/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt index 4a9d68ecb..ec5092817 100644 --- a/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt +++ b/src/test/java/com/amazon/ion/impl/IonRawTextWriterTest_1_1.kt @@ -713,7 +713,11 @@ class IonRawTextWriterTest_1_1 { @ParameterizedTest @EnumSource(SystemMacro::class) fun `write system macro E-expression by name`(systemMacro: SystemMacro) { - if (systemMacro.systemSymbol == null) throw TestAbortedException("Skip this test for unaddressable macros") + try { + systemMacro.systemSymbol + } catch (e: IllegalStateException) { + throw TestAbortedException("Skip this test for unaddressable macros") + } assertWriterOutputEquals("(:\$ion::${systemMacro.macroName})") { stepInEExp(systemMacro) stepOut() From a9fc6ea70b205879dd0f015a8be83a2ca4ddf8dc Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Tue, 17 Dec 2024 15:17:38 -0800 Subject: [PATCH 08/10] Rearrange some lines to minimize the diff --- .../amazon/ion/impl/macro/MacroEvaluator.kt | 158 +++++++++--------- 1 file changed, 82 insertions(+), 76 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index af3ecf589..52c3229a0 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -4,13 +4,10 @@ package com.amazon.ion.impl.macro import com.amazon.ion.* import com.amazon.ion.impl._Private_RecyclingStack -import com.amazon.ion.impl._Private_Utils.* +import com.amazon.ion.impl._Private_Utils.newSymbolToken import com.amazon.ion.impl.macro.Expression.* -import com.amazon.ion.impl.macro.MacroEvaluator.ExpanderKind.Stream -import com.amazon.ion.impl.macro.MacroEvaluator.ExpanderKind.Uninitialized -import com.amazon.ion.util.* +import com.amazon.ion.util.unreachable import java.io.ByteArrayOutputStream -import java.lang.StringBuilder import java.math.BigDecimal import java.math.BigInteger @@ -65,15 +62,15 @@ class MacroEvaluator { private val expanderPool: ArrayList = ArrayList(32) /** Gets an [ExpansionInfo] from the pool (or allocates a new one if necessary), initializing it with the provided values. */ - fun getExpander(expanderKind: ExpanderKind, expressions: List, startInclusive: Int, endExclusive: Int, environment: Environment): ExpansionInfo { + fun getExpander(expansionKind: ExpansionKind, expressions: List, startInclusive: Int, endExclusive: Int, environment: Environment): ExpansionInfo { val expansion = expanderPool.removeLastOrNull() ?: ExpansionInfo(this) - expansion.expanderKind = expanderKind + expansion.expansionKind = expansionKind expansion.expressions = expressions expansion.i = startInclusive expansion.endExclusive = endExclusive expansion.environment = environment expansion.additionalState = null - expansion.expansionDelegate = null + expansion.childExpansion = null return expansion } @@ -124,7 +121,7 @@ class MacroEvaluator { */ // TODO(PERF): It might be possible to optimize this by changing it to an enum without any methods (or even a set of // integer constants) and converting all their implementations to static methods. - private enum class ExpanderKind { + private enum class ExpansionKind { Uninitialized { override fun produceNext(thisExpansion: ExpansionInfo): Nothing = throw IllegalStateException("ExpansionInfo not initialized.") }, @@ -134,21 +131,21 @@ class MacroEvaluator { Stream { override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { // If there's a delegate, we'll try that first. - val delegate = thisExpansion.expansionDelegate + val delegate = thisExpansion.childExpansion check(thisExpansion != delegate) if (delegate != null) { return when (val result = delegate.produceNext()) { is DataModelExpression -> result EndOfExpansion -> { delegate.close() - thisExpansion.expansionDelegate = null + thisExpansion.childExpansion = null ContinueExpansion } } } if (thisExpansion.i >= thisExpansion.endExclusive) { - thisExpansion.expanderKind = Empty + thisExpansion.expansionKind = Empty return ContinueExpansion } @@ -162,9 +159,9 @@ class MacroEvaluator { val macro = next.macro val argIndices = calculateArgumentIndices(macro, thisExpansion.expressions, next.startInclusive, next.endExclusive) val newEnvironment = thisExpansion.environment.createChild(thisExpansion.expressions, argIndices) - val expanderKind = ExpanderKind.forMacro(macro) - thisExpansion.expansionDelegate = thisExpansion.session.getExpander( - expanderKind = expanderKind, + val expansionKind = ExpansionKind.forMacro(macro) + thisExpansion.childExpansion = thisExpansion.session.getExpander( + expansionKind = expansionKind, expressions = macro.body ?: emptyList(), startInclusive = 0, endExclusive = macro.body?.size ?: 0, @@ -173,8 +170,8 @@ class MacroEvaluator { ContinueExpansion } is ExpressionGroup -> { - thisExpansion.expansionDelegate = thisExpansion.session.getExpander( - expanderKind = ExprGroup, + thisExpansion.childExpansion = thisExpansion.session.getExpander( + expansionKind = ExprGroup, expressions = thisExpansion.expressions, startInclusive = next.startInclusive, endExclusive = next.endExclusive, @@ -185,7 +182,7 @@ class MacroEvaluator { } is VariableRef -> { - thisExpansion.expansionDelegate = thisExpansion.readArgument(next) + thisExpansion.childExpansion = thisExpansion.readArgument(next) ContinueExpansion } Placeholder -> unreachable() @@ -286,7 +283,7 @@ class MacroEvaluator { is FieldName -> unreachable() } } - thisExpansion.expanderKind = Empty + thisExpansion.expansionKind = Empty return StringValue(value = sb.toString()) } }, @@ -321,7 +318,7 @@ class MacroEvaluator { is FieldName -> unreachable() } } - thisExpansion.expanderKind = Empty + thisExpansion.expansionKind = Empty return BlobValue(value = baos.toByteArray()) } }, @@ -332,7 +329,7 @@ class MacroEvaluator { override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { val coefficient = thisExpansion.readExactlyOneArgument(COEFFICIENT_ARG).bigIntegerValue val exponent = thisExpansion.readExactlyOneArgument(EXPONENT_ARG).bigIntegerValue - thisExpansion.expanderKind = Empty + thisExpansion.expansionKind = Empty return DecimalValue(value = BigDecimal(coefficient, -1 * exponent.intValueExact())) } }, @@ -386,7 +383,7 @@ class MacroEvaluator { Timestamp.forYear(year) } } - thisExpansion.expanderKind = Empty + thisExpansion.expansionKind = Empty return TimestampValue(value = ts) } catch (e: IllegalArgumentException) { throw IonException(e.message) @@ -410,7 +407,7 @@ class MacroEvaluator { return fieldNameExpression.also { thisExpansion.tailCall(valueExpansion) - thisExpansion.expanderKind = ExactlyOneValueStream + thisExpansion.expansionKind = ExactlyOneValueStream } } }, @@ -425,7 +422,7 @@ class MacroEvaluator { thisExpansion.additionalState = argumentExpansion } - val currentChildExpansion = thisExpansion.expansionDelegate + val currentChildExpansion = thisExpansion.childExpansion return when (val next = currentChildExpansion?.produceNext()) { is DataModelExpression -> next @@ -433,8 +430,8 @@ class MacroEvaluator { // Only possible if expansionDelegate is null null -> when (val nextSequence = argumentExpansion.produceNext()) { is StructValue -> { - thisExpansion.expansionDelegate = thisExpansion.session.getExpander( - expanderKind = Stream, + thisExpansion.childExpansion = thisExpansion.session.getExpander( + expansionKind = Stream, expressions = argumentExpansion.top().expressions, startInclusive = nextSequence.startInclusive, endExclusive = nextSequence.endExclusive, @@ -464,7 +461,7 @@ class MacroEvaluator { thisExpansion.additionalState = argumentExpansion } - val currentChildExpansion = thisExpansion.expansionDelegate + val currentChildExpansion = thisExpansion.childExpansion return when (val next = currentChildExpansion?.produceNext()) { is DataModelExpression -> next @@ -473,8 +470,8 @@ class MacroEvaluator { null -> when (val nextSequence = argumentExpansion.produceNext()) { is StructValue -> throw IonException("invalid argument; flatten expects sequences") is DataModelContainer -> { - thisExpansion.expansionDelegate = thisExpansion.session.getExpander( - expanderKind = Stream, + thisExpansion.childExpansion = thisExpansion.session.getExpander( + expansionKind = Stream, expressions = argumentExpansion.top().expressions, startInclusive = nextSequence.startInclusive, endExclusive = nextSequence.endExclusive, @@ -497,7 +494,7 @@ class MacroEvaluator { // TODO(PERF): consider checking whether the value would fit in a long and returning a `LongIntValue`. val a = thisExpansion.readExactlyOneArgument(ARG_A).bigIntegerValue val b = thisExpansion.readExactlyOneArgument(ARG_B).bigIntegerValue - thisExpansion.expanderKind = Empty + thisExpansion.expansionKind = Empty return BigIntValue(value = a + b) } }, @@ -506,11 +503,11 @@ class MacroEvaluator { override fun produceNext(thisExpansion: ExpansionInfo): ExpansionOutputExpressionOrContinue { // TODO(PERF): Optimize to use LongIntValue when possible - var delegate = thisExpansion.expansionDelegate + var delegate = thisExpansion.childExpansion val runningTotal = thisExpansion.additionalState as? BigInteger ?: BigInteger.ZERO if (delegate == null) { delegate = thisExpansion.readArgument(ARGS) - thisExpansion.expansionDelegate = delegate + thisExpansion.childExpansion = delegate } when (val nextExpandedArg = delegate.produceNext()) { @@ -537,16 +534,16 @@ class MacroEvaluator { thisExpansion.additionalState = n } - if (thisExpansion.expansionDelegate == null) { + if (thisExpansion.childExpansion == null) { if (n > 0) { - thisExpansion.expansionDelegate = thisExpansion.readArgument(THING_TO_REPEAT) + thisExpansion.childExpansion = thisExpansion.readArgument(THING_TO_REPEAT) thisExpansion.additionalState = n - 1 } else { return EndOfExpansion } } - val repeated = thisExpansion.expansionDelegate!! + val repeated = thisExpansion.childExpansion!! return when (val maybeNext = repeated.produceNext()) { is DataModelExpression -> maybeNext EndOfExpansion -> thisExpansion.closeDelegateAndContinue() @@ -592,7 +589,7 @@ class MacroEvaluator { } val firstArgExpression = environment.arguments[argIndex] return session.getExpander( - expanderKind = Variable, + expansionKind = Variable, expressions = environment.arguments, startInclusive = if (firstArgExpression is ExpressionGroup) firstArgExpression.startInclusive else argIndex, endExclusive = if (firstArgExpression is HasStartAndEnd) firstArgExpression.endExclusive else argIndex + 1, @@ -663,29 +660,29 @@ class MacroEvaluator { companion object { /** - * Gets the [ExpanderKind] for the given [macro]. + * Gets the [ExpansionKind] for the given [macro]. */ @JvmStatic - fun forMacro(macro: Macro): ExpanderKind { + fun forMacro(macro: Macro): ExpansionKind { return if (macro.body != null) { TemplateBody } else when (macro as SystemMacro) { + SystemMacro.IfNone -> IfNone + SystemMacro.IfSome -> IfSome + SystemMacro.IfSingle -> IfSingle + SystemMacro.IfMulti -> IfMulti SystemMacro.Annotate -> Annotate SystemMacro.MakeString -> MakeString SystemMacro.MakeSymbol -> MakeSymbol SystemMacro.MakeDecimal -> MakeDecimal + SystemMacro.MakeTimestamp -> MakeTimestamp + SystemMacro.MakeBlob -> MakeBlob SystemMacro.Repeat -> Repeat SystemMacro.Sum -> Sum SystemMacro.Delta -> Delta - SystemMacro.MakeBlob -> MakeBlob SystemMacro.Flatten -> Flatten SystemMacro._Private_FlattenStruct -> _Private_FlattenStruct - SystemMacro.MakeTimestamp -> MakeTimestamp SystemMacro._Private_MakeFieldNameAndValue -> _Private_MakeFieldNameAndValue - SystemMacro.IfNone -> IfNone - SystemMacro.IfSome -> IfSome - SystemMacro.IfSingle -> IfSingle - SystemMacro.IfMulti -> IfMulti else -> TODO("Not implemented yet: ${macro.name}") } } @@ -698,59 +695,68 @@ class MacroEvaluator { * TODO: "info" is very non-specific; rename to ExpansionFrame next time there's a * non-functional refactoring in this class. */ - private class ExpansionInfo( - @JvmField val session: Session, - @JvmField var expanderKind: ExpanderKind = Uninitialized, + private class ExpansionInfo(@JvmField val session: Session) { + + /** The [ExpansionKind]. */ + @JvmField var expansionKind: ExpansionKind = ExpansionKind.Uninitialized + /** + * The evaluation [Environment]—i.e. variable bindings. + */ + @JvmField var environment: Environment = Environment.EMPTY /** * The [Expression]s being expanded. This MUST be the original list, not a sublist because * (a) we don't want to be allocating new sublists all the time, and (b) the * start and end indices of the expressions may be incorrect if a sublist is taken. */ - @JvmField var expressions: List = emptyList(), - /** Current position within [expressions] of this expansion */ - @JvmField var i: Int = 0, + @JvmField var expressions: List = emptyList() /** End of [expressions] that are applicable for this [ExpansionInfo] */ - @JvmField var endExclusive: Int = 0, - /** The evaluation [Environment]—i.e. variable bindings. */ - @JvmField var environment: Environment = Environment.EMPTY, - _expansionDelegate: ExpansionInfo? = null, - @JvmField var additionalState: Any? = null, - ) { + @JvmField var endExclusive: Int = 0 + /** Current position within [expressions] of this expansion */ + @JvmField var i: Int = 0 - // TODO: if expansionDelegate == this, it will cause an infinite loop or stack overflow somewhere. - // In practice, it should never happen, so we may wish to remove the custom setter to avoid any performance impact. - var expansionDelegate: ExpansionInfo? = _expansionDelegate + /** + * Field for storing any additional state required by an ExpansionKind. + */ + @JvmField + var additionalState: Any? = null + + /** + * Additional state in the form of a child [ExpansionInfo]. + */ + var childExpansion: ExpansionInfo? = null + // TODO: if childExpansion == this, it will cause an infinite loop or stack overflow somewhere. + // In practice, it should never happen, so we may wish to remove the custom setter to avoid any performance impact. set(value) { check(value != this) field = value } /** - * Convenience function to close the [expansionDelegate] and return it to the pool. + * Convenience function to close the [childExpansion] and return it to the pool. */ fun closeDelegateAndContinue(): ContinueExpansion { - expansionDelegate?.close() - expansionDelegate = null + childExpansion?.close() + childExpansion = null return ContinueExpansion } /** - * Gets the [ExpansionInfo] at the top of the stack of [expansionDelegate]s. + * Gets the [ExpansionInfo] at the top of the stack of [childExpansion]s. */ - fun top(): ExpansionInfo = expansionDelegate?.top() ?: this + fun top(): ExpansionInfo = childExpansion?.top() ?: this /** - * Returns this [ExpansionInfo] to the expander pool, recursively closing [expansionDelegate]s in the process. + * Returns this [ExpansionInfo] to the expander pool, recursively closing [childExpansion]s in the process. * Could also be thought of as a `free` function. */ fun close() { - expanderKind = Uninitialized + expansionKind = ExpansionKind.Uninitialized environment = Environment.EMPTY expressions = emptyList() additionalState?.let { if (it is ExpansionInfo) it.close() } additionalState = null - expansionDelegate?.close() - expansionDelegate = null + childExpansion?.close() + childExpansion = null session.reclaimExpander(this) } @@ -759,15 +765,15 @@ class MacroEvaluator { * After transferring the state, `other` is returned to the expansion pool. */ fun tailCall(other: ExpansionInfo) { - this.expanderKind = other.expanderKind + this.expansionKind = other.expansionKind this.expressions = other.expressions this.i = other.i this.endExclusive = other.endExclusive - this.expansionDelegate = other.expansionDelegate + this.childExpansion = other.childExpansion this.additionalState = other.additionalState this.environment = other.environment // Close `other` - other.expansionDelegate = null + other.childExpansion = null other.close() } @@ -776,7 +782,7 @@ class MacroEvaluator { */ fun produceNext(): ExpansionOutputExpression { while (true) { - val next = expanderKind.produceNext(this) + val next = expansionKind.produceNext(this) if (next is ContinueExpansion) continue // This the only place where we count the expansion steps. // It is theoretically possible to have macro expansions that are millions of levels deep because this @@ -791,14 +797,14 @@ class MacroEvaluator { override fun toString() = """ |ExpansionInfo( - | expansionKind: $expanderKind, + | expansionKind: $expansionKind, | environment: ${environment.toString().lines().joinToString("\n| ")}, | expressions: [ | ${expressions.mapIndexed { i, expr -> "$i. $expr" }.joinToString(",\n| ") } | ], | endExclusive: $endExclusive, | i: $i, - | child: ${expansionDelegate?.expanderKind} + | child: ${childExpansion?.expansionKind} | additionalState: $additionalState, |) """.trimMargin() @@ -822,7 +828,7 @@ class MacroEvaluator { session.reset() containerStack.push { ci -> ci.type = ContainerInfo.Type.TopLevel - ci.expansion = session.getExpander(Stream, encodingExpressions, 0, encodingExpressions.size, Environment.EMPTY) + ci.expansion = session.getExpander(ExpansionKind.Stream, encodingExpressions, 0, encodingExpressions.size, Environment.EMPTY) } } @@ -872,7 +878,7 @@ class MacroEvaluator { else -> unreachable() } ci.expansion = session.getExpander( - expanderKind = Stream, + expansionKind = ExpansionKind.Stream, expressions = topExpansion.expressions, startInclusive = expression.startInclusive, endExclusive = expression.endExclusive, From 75e012db9470af7721d11ad9c87c3c16ccc7646b Mon Sep 17 00:00:00 2001 From: Matthew Pope <81593196+popematt@users.noreply.github.com> Date: Thu, 2 Jan 2025 11:14:02 -0800 Subject: [PATCH 09/10] Update src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt Co-authored-by: Tyler Gregg --- src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index 52c3229a0..46015fb8d 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -250,7 +250,7 @@ class MacroEvaluator { when (it) { is StringValue -> newSymbolToken(it.value) is SymbolValue -> it.value - is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}") + is DataModelValue -> throw IonException("Invalid argument type for 'annotate': ${it.type}") else -> unreachable("Unreachable without stepping in to a container") } } From 14f315d14aae1498ab50f07acf896ea601fdb7e3 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Thu, 2 Jan 2025 13:08:40 -0800 Subject: [PATCH 10/10] Adds suggested changes --- .../amazon/ion/impl/macro/MacroEvaluator.kt | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index 46015fb8d..47bc52798 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -10,6 +10,7 @@ import com.amazon.ion.util.unreachable import java.io.ByteArrayOutputStream import java.math.BigDecimal import java.math.BigInteger +import java.util.IdentityHashMap /** * Evaluates an EExpression from a List of [EExpressionBodyExpression] and the [TemplateBodyExpression]s @@ -26,24 +27,16 @@ import java.math.BigInteger * * ### Implementation Overview: * - * The macro evaluator can be thought of as a stack of containers where each container has a stack of expansion frames - * (i.e. [ExpansionInfo]). To get the next value at the current depth, the macro evaluator starts with the bottom frame - * of the top container. Expansion frames may delegate to and/or intercept other expansions that are further up the stack. + * The macro evaluator consists of a stack of containers, each of which has an implicit stream (i.e. the + * expressions in that container) which is modeled as an expansion frame ([ExpansionInfo]). * - * One might visualize it like this: - * ``` - * 3. List : Stream --> Delta --> Variable - * 2. List : Stream --> Flatten --> Stream - * 1. Struct : Stream --> Variable --> TemplateBody --> Stream --> TemplateBody - * 0. TopLevel : Stream --> TemplateBody --> TemplateBody - * ``` - * - * When calling [expandNext], the evaluator looks at the first expansion frame of the top container in the stack. - * Then it calls `produceNext` for the first expansion in that container. That expansion may produce a result all on its - * own, or it may call the next expansion in the chain and return that value (with or without further modification). - * - * In practice, it is a little more complex. A single expansion frame may hold more than one child frame sequentially - * (e.g. `repeat`, `annotate`) or concurrently (e.g. `for`, `flatten`). + * When calling [expandNext], the evaluator looks at the top container in the stack and requests the next value from + * its expansion frame. That expansion frame may produce a result all on its own (i.e. if the next value is a literal + * value), or it may create and delegate to a child expansion frame if the next source expression is something that + * needs to be expanded (e.g. macro invocation, variable expansion, etc.). When delegating to a child expansion frame, + * the value returned by the child could be intercepted and inspected, modified, or consumed. + * In this way, the expansion frames model a lazily constructed expression tree over the flat list of expressions in the + * input to the macro evaluator. */ class MacroEvaluator { @@ -60,10 +53,13 @@ class MacroEvaluator { private var numExpandedExpressions = 0 /** Pool of [ExpansionInfo] to minimize allocation and garbage collection. */ private val expanderPool: ArrayList = ArrayList(32) + /** Negative view of pool so that we can have O(1) membership checks */ + private val vendedExpanders: IdentityHashMap = IdentityHashMap(32) /** Gets an [ExpansionInfo] from the pool (or allocates a new one if necessary), initializing it with the provided values. */ fun getExpander(expansionKind: ExpansionKind, expressions: List, startInclusive: Int, endExclusive: Int, environment: Environment): ExpansionInfo { val expansion = expanderPool.removeLastOrNull() ?: ExpansionInfo(this) + vendedExpanders[expansion] = Unit expansion.expansionKind = expansionKind expansion.expressions = expressions expansion.i = startInclusive @@ -76,8 +72,8 @@ class MacroEvaluator { /** Reclaims an [ExpansionInfo] to the available pool. */ fun reclaimExpander(ex: ExpansionInfo) { - // TODO: This check is O(n). Consider removing this when confident there are no double frees. - check(ex !in expanderPool) + // Ensure that we are not doubly-adding an ExpansionInfo instance to the pool. + check(vendedExpanders.remove(ex, Unit)) expanderPool.add(ex) } @@ -694,6 +690,8 @@ class MacroEvaluator { * * TODO: "info" is very non-specific; rename to ExpansionFrame next time there's a * non-functional refactoring in this class. + * Alternately, consider ExpansionOperator to reflect the fact that these are + * like operators in an expression tree. */ private class ExpansionInfo(@JvmField val session: Session) {