diff --git a/src/main/java/com/amazon/ion/impl/macro/Expression.kt b/src/main/java/com/amazon/ion/impl/macro/Expression.kt index 9bff69879..79263eff0 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Expression.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Expression.kt @@ -60,6 +60,8 @@ sealed interface Expression { sealed interface DataModelValue : DataModelExpression { val annotations: List val type: IonType + + fun withAnnotations(annotations: List): DataModelValue } /** Expressions that represent Ion container types */ @@ -82,32 +84,44 @@ sealed interface Expression { data class ExpressionGroup(override val selfIndex: Int, override val endExclusive: Int) : EExpressionBodyExpression, TemplateBodyExpression, HasStartAndEnd // Scalars - data class NullValue(override val annotations: List = emptyList(), override val type: IonType) : DataModelValue + data class NullValue(override val annotations: List = emptyList(), override val type: IonType) : DataModelValue { + override fun withAnnotations(annotations: List) = copy(annotations = annotations) + } data class BoolValue(override val annotations: List = emptyList(), val value: Boolean) : DataModelValue { override val type: IonType get() = IonType.BOOL + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } - sealed interface IntValue : DataModelValue + sealed interface IntValue : DataModelValue { + val bigIntegerValue: BigInteger + } data class LongIntValue(override val annotations: List = emptyList(), val value: Long) : IntValue { override val type: IonType get() = IonType.INT + override fun withAnnotations(annotations: List) = copy(annotations = annotations) + override val bigIntegerValue: BigInteger get() = BigInteger.valueOf(value) } data class BigIntValue(override val annotations: List = emptyList(), val value: BigInteger) : IntValue { override val type: IonType get() = IonType.INT + override fun withAnnotations(annotations: List) = copy(annotations = annotations) + override val bigIntegerValue: BigInteger get() = value } data class FloatValue(override val annotations: List = emptyList(), val value: Double) : DataModelValue { override val type: IonType get() = IonType.FLOAT + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } data class DecimalValue(override val annotations: List = emptyList(), val value: BigDecimal) : DataModelValue { override val type: IonType get() = IonType.DECIMAL + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } data class TimestampValue(override val annotations: List = emptyList(), val value: Timestamp) : DataModelValue { override val type: IonType get() = IonType.TIMESTAMP + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } sealed interface TextValue : DataModelValue { @@ -117,11 +131,13 @@ sealed interface Expression { data class StringValue(override val annotations: List = emptyList(), val value: String) : TextValue { override val type: IonType get() = IonType.STRING override val stringValue: String get() = value + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } data class SymbolValue(override val annotations: List = emptyList(), val value: SymbolToken) : TextValue { override val type: IonType get() = IonType.SYMBOL override val stringValue: String get() = value.assumeText() + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } sealed interface LobValue : DataModelValue { @@ -133,6 +149,7 @@ sealed interface Expression { // We must override hashcode and equals in the lob types because `value` is a `byte[]` data class BlobValue(override val annotations: List = emptyList(), override val value: ByteArray) : LobValue { override val type: IonType get() = IonType.BLOB + override fun withAnnotations(annotations: List) = copy(annotations = annotations) override fun hashCode(): Int = annotations.hashCode() * 31 + value.contentHashCode() override fun equals(other: Any?): Boolean { if (this === other) return true @@ -144,6 +161,7 @@ sealed interface Expression { data class ClobValue(override val annotations: List = emptyList(), override val value: ByteArray) : LobValue { override val type: IonType get() = IonType.CLOB + override fun withAnnotations(annotations: List) = copy(annotations = annotations) override fun hashCode(): Int = annotations.hashCode() * 31 + value.contentHashCode() override fun equals(other: Any?): Boolean { if (this === other) return true @@ -165,6 +183,7 @@ sealed interface Expression { override val endExclusive: Int ) : DataModelContainer { override val type: IonType get() = IonType.LIST + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } /** @@ -176,6 +195,7 @@ sealed interface Expression { override val endExclusive: Int ) : DataModelContainer { override val type: IonType get() = IonType.SEXP + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } /** @@ -188,6 +208,7 @@ sealed interface Expression { val templateStructIndex: Map> ) : DataModelContainer { override val type: IonType get() = IonType.STRUCT + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } data class FieldName(val value: SymbolToken) : DataModelExpression diff --git a/src/main/java/com/amazon/ion/impl/macro/Macro.kt b/src/main/java/com/amazon/ion/impl/macro/Macro.kt index c4601f4c2..61e34c00c 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Macro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Macro.kt @@ -3,6 +3,7 @@ package com.amazon.ion.impl.macro import com.amazon.ion.impl.* +import com.amazon.ion.impl.macro.Macro.Parameter.Companion.exactlyOneTagged import com.amazon.ion.impl.macro.Macro.Parameter.Companion.zeroToManyTagged /** @@ -115,9 +116,20 @@ data class TemplateMacro(override val signature: List, val body /** * Macros that are built in, rather than being defined by a template. */ -enum class SystemMacro(val macroName: String, override val signature: List) : Macro { +enum class SystemMacro(val macroName: String, override val signature: List, val body: List? = null) : Macro { + None("none", emptyList()), Values("values", listOf(zeroToManyTagged("values"))), + Annotate("annotate", listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))), MakeString("make_string", listOf(zeroToManyTagged("text"))), + MakeSymbol("make_symbol", listOf(zeroToManyTagged("text"))), + MakeDecimal( + "make_decimal", + listOf( + Macro.Parameter("coefficient", Macro.ParameterEncoding.CompactInt, Macro.ParameterCardinality.ExactlyOne), + Macro.Parameter("exponent", Macro.ParameterEncoding.CompactInt, Macro.ParameterCardinality.ExactlyOne), + ) + ), + // TODO: Other system macros ; diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index f995ccaa6..f0cee7509 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -3,8 +3,10 @@ package com.amazon.ion.impl.macro import com.amazon.ion.* +import com.amazon.ion.SymbolTable.* import com.amazon.ion.impl.* import com.amazon.ion.impl.macro.Expression.* +import java.math.BigDecimal /** * Evaluates an EExpression from a List of [EExpressionBodyExpression] and the [TemplateBodyExpression]s @@ -26,6 +28,56 @@ class MacroEvaluator { */ private fun interface Expander { fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression + + /** + * Read the expanded values from one argument, returning exactly one value. + * Throws an exception if there is not exactly one expanded value. + */ + fun readExactlyOneArgumentValue(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, argName: String): DataModelExpression { + return readZeroOrOneArgumentValues(expansionInfo, macroEvaluator, argName) + ?: throw IonException("Argument $argName expanded to nothing.") + } + + /** + * Read the expanded values from one argument, returning zero or one values. + * Throws an exception if there is more than one expanded value. + */ + fun readZeroOrOneArgumentValues(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, argName: String): DataModelExpression? { + var value: DataModelExpression? = null + readArgumentValues(expansionInfo, macroEvaluator) { + if (value == null) { + value = it + } else { + throw IonException("Too many values for argument $argName") + } + } + return value + } + + /** + * Reads the expanded values from one argument. + */ + fun readArgumentValues(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, callback: (DataModelExpression) -> Unit) { + val i = expansionInfo.i + expansionInfo.nextSourceExpression() + + macroEvaluator.pushExpansion( + expansionKind = ExpansionKind.Values, + argsStartInclusive = i, + // There can only be one top-level expression for an argument (it's either a value, macro, or + // expression group) so we can set the end to one more than the start. + argsEndExclusive = i + 1, + environment = expansionInfo.environment ?: Environment.EMPTY, + expressions = expansionInfo.expressions!!, + ) + + val depth = macroEvaluator.expansionStack.size() + var expr = macroEvaluator.expandNext(depth) + while (expr != null) { + callback(expr) + expr = macroEvaluator.expandNext(depth) + } + } } private object SimpleExpander : Expander { @@ -34,24 +86,58 @@ class MacroEvaluator { } } - private object MakeStringExpander : Expander { + private object AnnotateExpander : Expander { + // TODO: Handle edge cases mentioned in https://github.com/amazon-ion/ion-docs/issues/347 + override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { + val annotations = mutableListOf() + + readArgumentValues(expansionInfo, macroEvaluator) { + when (it) { + is StringValue -> annotations.add(_Private_Utils.newSymbolToken(it.value, UNKNOWN_SYMBOL_ID)) + is SymbolValue -> annotations.add(it.value) + is DataModelValue -> throw IonException("Annotation arguments must be string or symbol; found: ${it.type}") + is FieldName -> TODO("Unreachable. Must encounter a StructValue first.") + } + } + + val valueToAnnotate = readExactlyOneArgumentValue(expansionInfo, macroEvaluator, SystemMacro.Annotate.signature[1].variableName) + + // It cannot be a FieldName expression because we haven't stepped into a struct, so it must be DataModelValue + valueToAnnotate as DataModelValue + // Combine the annotations + annotations.addAll(valueToAnnotate.annotations) + return valueToAnnotate.withAnnotations(annotations) + } + } + + private class MakeTextExpander(private val constructor: (String) -> Expression) : Expander { override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - // Tell the macro evaluator to treat this as a values expansion... - macroEvaluator.expansionStack.peek().expansionKind = ExpansionKind.Values - val minDepth = macroEvaluator.expansionStack.size() - // ...But capture the output and turn it into a String val sb = StringBuilder() - while (true) { - when (val expr: DataModelExpression? = macroEvaluator.expandNext(minDepth)) { - is StringValue -> sb.append(expr.value) - is SymbolValue -> sb.append(expr.value.assumeText()) + readArgumentValues(expansionInfo, macroEvaluator) { + when (it) { + is StringValue -> sb.append(it.value) + is SymbolValue -> sb.append(it.value.assumeText()) is NullValue -> {} - null -> break - is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${expr.type}") + is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}") is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.") } } - return StringValue(value = sb.toString()) + return constructor(sb.toString()) + } + } + + private object MakeDecimalExpander : Expander { + override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { + val coefficient = readExactlyOneArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeDecimal.signature[0].variableName) + .let { it as? IntValue } + ?.bigIntegerValue + ?: throw IonException("Coefficient must be an integer") + val exponent = readExactlyOneArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeDecimal.signature[1].variableName) + .let { it as? IntValue } + ?.bigIntegerValue + ?: throw IonException("Exponent must be an integer") + + return DecimalValue(value = BigDecimal(coefficient, -1 * exponent.intValueExact())) } } @@ -59,15 +145,22 @@ class MacroEvaluator { Container(SimpleExpander), TemplateBody(SimpleExpander), Values(SimpleExpander), - MakeString(MakeStringExpander), + MakeString(MakeTextExpander { StringValue(value = it) }), + MakeSymbol(MakeTextExpander { SymbolValue(value = _Private_Utils.newSymbolToken(it, UNKNOWN_SYMBOL_ID)) }), + MakeDecimal(MakeDecimalExpander), + Annotate(AnnotateExpander), ; companion object { @JvmStatic fun forSystemMacro(macro: SystemMacro): ExpansionKind { return when (macro) { + SystemMacro.None -> Values // "none" takes no args, so we can treat it as an empty "values" expansion SystemMacro.Values -> Values SystemMacro.MakeString -> MakeString + SystemMacro.Annotate -> Annotate + SystemMacro.MakeSymbol -> MakeSymbol + SystemMacro.MakeDecimal -> MakeDecimal } } } diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt index 04c9c4b17..4d45312cb 100644 --- a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt +++ b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt @@ -8,8 +8,13 @@ import com.amazon.ion.impl.macro.Expression.* import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.eExpBody import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.templateBody import com.amazon.ion.impl.macro.SystemMacro.* +import java.math.BigDecimal +import java.math.BigInteger +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.contract import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class MacroEvaluatorTest { @@ -39,6 +44,47 @@ class MacroEvaluatorTest { val evaluator = MacroEvaluator() + @Test + fun `the 'none' system macro`() { + // Given: + // When: + // (:none) + // Then: + // + + evaluator.initExpansion { + eexp(None) {} + } + + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `the 'none' system macro, invoked in TDL`() { + // Given: + // (macro blackhole (any*) (.none)) + // When: + // (:blackhole "abc" 123 true) + // Then: + // + + val blackholeMacro = template("any*") { + macro(None) {} + } + + evaluator.initExpansion { + eexp(blackholeMacro) { + expressionGroup { + string("abc") + int(123) + bool(true) + } + } + } + + assertEquals(null, evaluator.expandNext()) + } + @Test fun `a trivial constant macro evaluation`() { // Given: @@ -384,6 +430,237 @@ class MacroEvaluatorTest { assertEquals(null, evaluator.expandNext()) } + @Test + fun `simple make_symbol`() { + // Given: + // When: + // (:make_symbol "a" "b" "c") + // Then: + // abc + + evaluator.initExpansion { + eexp(MakeSymbol) { + expressionGroup { + string("a") + string("b") + string("c") + } + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals("abc", expr.value.text) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `simple make_decimal`() { + // Given: + // When: + // (:make_decimal 2 4) + // Then: + // 2d4 + + evaluator.initExpansion { + eexp(MakeDecimal) { + int(2) + int(4) + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertTrue(BigDecimal.valueOf(20000).compareTo(expr.value) == 0) + assertEquals(BigInteger.valueOf(2), expr.value.unscaledValue()) + assertEquals(-4, expr.value.scale()) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `make_decimal from nested expressions`() { + // Given: + // (macro fixed_point (x) (.make_decimal x (.values -2))) + // When: + // (:fixed_point (:identity 123)) + // Then: + // 1.23 + + val fixedPointMacro = template("x") { + macro(MakeDecimal) { + variable(0) + macro(Values) { + expressionGroup { + int(-2) + } + } + } + } + + evaluator.initExpansion { + eexp(fixedPointMacro) { + eexp(IDENTITY_MACRO) { + int(123) + } + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(BigDecimal.valueOf(123, 2), expr.value) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `simple annotate`() { + // Given: + // When: + // (:annotate (:: "a" "b" "c") 1) + // Then: + // a::b::c::1 + + evaluator.initExpansion { + eexp(Annotate) { + expressionGroup { + string("a") + string("b") + string("c") + } + int(1) + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("a", "b", "c"), expr.annotations.map { it.text }) + assertEquals(1, expr.value) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `annotate a container`() { + // Given: + // When: + // (:annotate (:: "a" "b" "c") [1]) + // Then: + // a::b::c::[1] + + evaluator.initExpansion { + eexp(Annotate) { + expressionGroup { + string("a") + string("b") + string("c") + } + list { + int(1) + } + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("a", "b", "c"), expr.annotations.map { it.text }) + evaluator.stepIn() + assertEquals(LongIntValue(emptyList(), 1), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `annotate with nested make_string`() { + // Given: + // When: + // (:annotate (:make_string (:: "a" "b" "c")) 1) + // Then: + // abc::1 + + evaluator.initExpansion { + eexp(Annotate) { + eexp(MakeString) { + expressionGroup { + string("a") + string("b") + string("c") + } + } + int(1) + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("abc"), expr.annotations.map { it.text }) + assertEquals(1, expr.value) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `annotate an e-expression result`() { + // Given: + // When: + // (:annotate (:: "a" "b" "c") (:make_string "d" "e" "f")) + // Then: + // a::b::c::"def" + + evaluator.initExpansion { + eexp(Annotate) { + expressionGroup { + string("a") + string("b") + string("c") + } + + eexp(MakeString) { + expressionGroup { + string("d") + string("e") + string("f") + } + } + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("a", "b", "c"), expr.annotations.map { it.text }) + assertEquals("def", expr.value) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `annotate an TDL macro invocation result`() { + // Given: + // (macro pi () 3.14159) + // (macro annotate_pi (x) (.annotate (..x) (.pi))) + // When: + // (:annotate_pi "foo") + // Then: + // foo::3.14159 + + val annotatePi = template("x") { + macro(Annotate) { + expressionGroup { + variable(0) + } + macro(PI_MACRO) {} + } + } + + evaluator.initExpansion { + eexp(annotatePi) { + string("foo") + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("foo"), expr.annotations.map { it.text }) + assertEquals(3.14159, expr.value) + assertEquals(null, evaluator.expandNext()) + } + @Test fun `macro with a variable substitution in struct field position`() { // Given: @@ -501,7 +778,9 @@ class MacroEvaluatorTest { /** Helper function to use Expression DSL for evaluator inputs */ fun MacroEvaluator.initExpansion(eExpression: EExpDsl.() -> Unit) = initExpansion(eExpBody(eExpression)) + @OptIn(ExperimentalContracts::class) private inline fun assertIsInstance(value: Any?) { + contract { returns() implies (value is T) } if (value !is T) { val message = if (value == null) { "Expected instance of ${T::class.qualifiedName}; was null"