diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/DataType.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/DataType.java index 44852a36ab..c6e899a6f4 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/DataType.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/DataType.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; +import java.math.BigDecimal; import java.util.Arrays; import java.util.ArrayList; import java.util.Map; @@ -49,6 +50,13 @@ public enum DataType { */ DOUBLE("double"), + /** + * Type of BigDecimal. No precision loss possible type. Compatible with the Java BigDecimal primitive data type. + * + * @since 2.8 + */ + BIG_DECIMAL("big_decimal"), + /** * Type of map. Compatible with the Java map primitive data type. * @@ -96,20 +104,22 @@ public static boolean isSameType(final Object object, final String option) { if (type == null) throw new IllegalArgumentException("Unknown DataType"); switch (type) { - case MAP: - return (object instanceof Map); - case ARRAY: - return (object instanceof ArrayList || object.getClass().isArray()); - case DOUBLE: - return (object instanceof Double); - case BOOLEAN: - return (object instanceof Boolean); - case INTEGER: - return (object instanceof Integer); - case LONG: - return (object instanceof Long); - default: // STRING - return (object instanceof String); + case MAP: + return (object instanceof Map); + case ARRAY: + return (object instanceof ArrayList || object.getClass().isArray()); + case DOUBLE: + return (object instanceof Double); + case BOOLEAN: + return (object instanceof Boolean); + case INTEGER: + return (object instanceof Integer); + case LONG: + return (object instanceof Long); + case BIG_DECIMAL: + return (object instanceof BigDecimal); + default: // STRING + return (object instanceof String); } } } diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java index e135f7e9db..9ef34bb82c 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java @@ -11,6 +11,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.cfg.JsonNodeFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.type.TypeFactory; @@ -61,11 +63,14 @@ public class JacksonEvent implements Event { private static final String SEPARATOR = "/"; - private static final ObjectMapper mapper = new ObjectMapper() + private static final ObjectMapper mapper = JsonMapper.builder() + .disable(JsonNodeFeature.STRIP_TRAILING_BIGDECIMAL_ZEROES) + .build() .registerModule(new JavaTimeModule()) .registerModule(new Jdk8Module()); // required for using Optional with Jackson. Ref: https://github.com/FasterXML/jackson-modules-java8 - private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() { + + private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference<>() { }; private final EventMetadata eventMetadata; diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/source/coordinator/SourceCoordinator.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/source/coordinator/SourceCoordinator.java index fa78880925..71c0232669 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/source/coordinator/SourceCoordinator.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/source/coordinator/SourceCoordinator.java @@ -122,6 +122,7 @@ public interface SourceCoordinator { /** * Should be called by the source when it is shutting down to indicate that it will no longer be able to perform work on partitions, * or can be called to give up ownership of its partitions in order to pick up new ones with {@link #getNextPartition(Function)} ()}. + * @param partitionKey - Key used as the partition key. * @param priorityTimestamp - A timestamp that will determine the order that UNASSIGNED partitions are acquired after they are given up. * @throws org.opensearch.dataprepper.model.source.coordinator.exceptions.PartitionUpdateException if the partition could not be given up due to some failure * @since 2.8 diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/BigDecimalConverter.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/BigDecimalConverter.java new file mode 100644 index 0000000000..8935e55e71 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/BigDecimalConverter.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + package org.opensearch.dataprepper.typeconverter; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +/** + * Converter class for BigDecimal data type. By default, it applies zero scaling keeping the original value as it is. + * If required, the scale can be set using the setScale method. + */ +public class BigDecimalConverter implements TypeConverter { + + public BigDecimal convert(Object source) throws IllegalArgumentException { + return this.convert(source, 0); + } + + public BigDecimal convert(Object source, ConverterArguments arguments) throws IllegalArgumentException { + return this.convert(source, arguments.getScale()); + } + + public BigDecimal convert(Object source, int scale) throws IllegalArgumentException { + BigDecimal result = null; + if (source instanceof String) { + result = new BigDecimal((String)source); + } + else if (source instanceof Float) { + result = BigDecimal.valueOf((Float)source); + } + else if (source instanceof Double) { + result = BigDecimal.valueOf((Double)source); + } + else if (source instanceof Boolean) { + result = ((Boolean)source) ? BigDecimal.valueOf(1L) : BigDecimal.valueOf(0L); + } + else if (source instanceof Integer) { + result = BigDecimal.valueOf((Integer)source); + } + else if (source instanceof Long) { + result = BigDecimal.valueOf((Long)source); + } + else if (source instanceof BigDecimal) { + result = ((BigDecimal)source); + } + + if(result!=null) { + if(scale!=0) { + result = result.setScale(scale, RoundingMode.HALF_EVEN); + } + return result; + } + throw new IllegalArgumentException("Unsupported type conversion. From Source class: " + source.getClass() + " to BigDecimal"); + } +} + \ No newline at end of file diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/BooleanConverter.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/BooleanConverter.java index a24c0bdc64..3da4669624 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/BooleanConverter.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/BooleanConverter.java @@ -6,18 +6,23 @@ package org.opensearch.dataprepper.typeconverter; public class BooleanConverter implements TypeConverter { + + public Boolean convert(Object source, ConverterArguments arguments) throws IllegalArgumentException { + return this.convert(source); + } + public Boolean convert(Object source) throws IllegalArgumentException { if (source instanceof String) { return Boolean.parseBoolean((String)source); } if (source instanceof Number) { Number number = (Number)source; - return ((number.intValue() != 0) || - (number.longValue() != 0) || + return ((number instanceof Integer && number.intValue() != 0) || + (number instanceof Long && number.longValue() != 0) || + (number instanceof Short && number.shortValue() != 0) || + (number instanceof Byte && number.byteValue() != 0)) || (number.floatValue() != 0) || - (number.doubleValue() != 0) || - (number.shortValue() != 0) || - (number.byteValue() != 0)) ? true : false; + (number.doubleValue() != 0); } if (source instanceof Boolean) { return (Boolean)source; diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/ConverterArguments.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/ConverterArguments.java new file mode 100644 index 0000000000..3fad9e3a0e --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/ConverterArguments.java @@ -0,0 +1,10 @@ +package org.opensearch.dataprepper.typeconverter; + +/** + * Interface for arguments passed to the {@link TypeConverter} + * + * @since 1.2 + */ +public interface ConverterArguments { + int getScale(); +} diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/DoubleConverter.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/DoubleConverter.java index db57bbe9eb..aa80c1360e 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/DoubleConverter.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/DoubleConverter.java @@ -6,6 +6,11 @@ package org.opensearch.dataprepper.typeconverter; public class DoubleConverter implements TypeConverter { + + public Double convert(Object source, ConverterArguments arguments) throws IllegalArgumentException { + return convert(source); + } + public Double convert(Object source) throws IllegalArgumentException { if (source instanceof String) { return Double.parseDouble((String)source); @@ -17,7 +22,7 @@ public Double convert(Object source) throws IllegalArgumentException { return (((Number)source).doubleValue()); } if (source instanceof Boolean) { - return (double)(((Boolean)source) ? 1.0 : 0.0); + return ((Boolean)source) ? 1.0 : 0.0; } throw new IllegalArgumentException("Unsupported type conversion. Source class: " + source.getClass()); } diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/IntegerConverter.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/IntegerConverter.java index 664c7f463f..41420b9dde 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/IntegerConverter.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/IntegerConverter.java @@ -7,6 +7,11 @@ import java.math.BigDecimal; public class IntegerConverter implements TypeConverter { + + public Integer convert(Object source, ConverterArguments arguments) throws IllegalArgumentException { + return convert(source); + } + public Integer convert(Object source) throws IllegalArgumentException { if (source instanceof String) { return Integer.parseInt((String)source); diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/LongConverter.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/LongConverter.java index 809bec9280..e3c1ee2b08 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/LongConverter.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/LongConverter.java @@ -7,6 +7,11 @@ import java.math.BigDecimal; public class LongConverter implements TypeConverter { + + public Long convert(Object source, ConverterArguments arguments) throws IllegalArgumentException { + return this.convert(source); + } + public Long convert(Object source) throws IllegalArgumentException { if (source instanceof String) { return Long.parseLong((String)source); diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/StringConverter.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/StringConverter.java index c2ad1a1eb6..acc9d08fce 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/StringConverter.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/StringConverter.java @@ -6,6 +6,11 @@ package org.opensearch.dataprepper.typeconverter; public class StringConverter implements TypeConverter { + + public String convert(Object source, ConverterArguments arguments) throws IllegalArgumentException { + return this.convert(source); + } + public String convert(Object source) throws IllegalArgumentException { if (source instanceof Number || source instanceof Boolean || source instanceof String) { return source.toString(); diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/TypeConverter.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/TypeConverter.java index 9d597d38a9..4256b8571f 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/TypeConverter.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/TypeConverter.java @@ -7,4 +7,5 @@ public interface TypeConverter { T convert(Object source); + T convert(Object source, ConverterArguments arguments); } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DataTypeTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DataTypeTest.java index 015fb91873..ac7a5bf613 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DataTypeTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DataTypeTest.java @@ -15,6 +15,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -37,15 +40,18 @@ void test_isSameType(Object object, String type, boolean expectedResult) { } private static Stream getSameTypeTestData() { - int testArray[] = {1,2}; + int[] testArray = {1,2}; + List testList = new ArrayList<>(); return Stream.of( Arguments.of(2, "integer", true), Arguments.of("testString", "string", true), Arguments.of(2L, "long", true), Arguments.of(2.0, "double", true), + Arguments.of(BigDecimal.valueOf(2.34567), "big_decimal", true), Arguments.of(true, "boolean", true), Arguments.of(Map.of("k","v"), "map", true), Arguments.of(testArray, "array", true), + Arguments.of(testList, "array", true), Arguments.of(2.0, "integer", false), Arguments.of(2, "string", false), Arguments.of("testString", "long", false), diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DefaultEventMetadataTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DefaultEventMetadataTest.java index 057ce4d1a6..7e91a86eb3 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DefaultEventMetadataTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DefaultEventMetadataTest.java @@ -176,6 +176,20 @@ public void testAttributes_without_attributes_is_empty() { } + @Test + public void testAttributes_with_attributes_of_not_map_type() { + Object v1 = new Object(); + eventMetadata = DefaultEventMetadata.builder() + .withEventType(testEventType) + .withTimeReceived(testTimeReceived) + .withAttributes(Map.of("key1", v1)) + .build(); + assertThat(eventMetadata.getAttribute("key1"), equalTo(v1)); + assertThat(eventMetadata.getAttribute("key1/key2/"), equalTo(null)); + assertThat(eventMetadata.getAttribute("key3"), equalTo(null)); + + } + @Test public void testBuild_withoutTimeReceived() { @@ -303,6 +317,40 @@ void setUp() { @Test void equals_returns_false_for_null() { assertThat(event.equals(null), equalTo(false)); + assertThat(event.equals(new Object()), equalTo(false)); + } + + @Test + void equals_returns_false_when_timeinstance_not_match() { + DefaultEventMetadata newEvent = DefaultEventMetadata.builder() + .withEventType(eventType) + .withTimeReceived(Instant.now()) + .withAttributes(Collections.singletonMap(attributeKey, attributeValue)) + .build(); + assertThat(event.equals(newEvent), equalTo(false)); + } + + @Test + void equals_returns_false_when_attributes_not_match() { + String newAttributeKey = UUID.randomUUID().toString(); + String newAttributeValue = UUID.randomUUID().toString(); + DefaultEventMetadata newEvent = DefaultEventMetadata.builder() + .withEventType(eventType) + .withTimeReceived(timeReceived) + .withAttributes(Collections.singletonMap(newAttributeKey, newAttributeValue)) + .build(); + assertThat(event.equals(newEvent), equalTo(false)); + } + + @Test + void equals_returns_false_when_tags_not_match() { + DefaultEventMetadata newEvent = DefaultEventMetadata.builder() + .withEventType(eventType) + .withTimeReceived(timeReceived) + .withAttributes(Collections.singletonMap(attributeKey, attributeValue)) + .withTags(Set.of("some","new","tag")) + .build(); + assertThat(event.equals(newEvent), equalTo(false)); } @Test diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java index e6fdceecfc..1a7efb7467 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java @@ -9,11 +9,14 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.model.event.exceptions.EventKeyNotFoundException; +import java.math.BigDecimal; import java.time.Instant; import java.util.Arrays; import java.util.Collections; @@ -22,6 +25,7 @@ import java.util.Map; import java.util.Random; import java.util.UUID; +import java.util.stream.Stream; import static org.hamcrest.CoreMatchers.containsStringIgnoringCase; import static org.hamcrest.CoreMatchers.equalTo; @@ -776,6 +780,13 @@ void testJsonStringBuilderWithIncludeKeys() { .getThis() .build(); + // Include Keys must start with / and also ordered, This is pre-processed in SinkModel + List includeNullKey = null; + assertThat(event.jsonBuilder().rootKey(null).includeKeys(includeNullKey).toJsonString(), equalTo(jsonString)); + + List includeEmptyKey = List.of(); + assertThat(event.jsonBuilder().rootKey(null).includeKeys(includeEmptyKey).toJsonString(), equalTo(jsonString)); + // Include Keys must start with / and also ordered, This is pre-processed in SinkModel List includeKeys1 = Arrays.asList("foo", "info"); final String expectedJsonString1 = "{\"foo\":\"bar\",\"info\":{\"name\":\"hello\",\"foo\":\"bar\"}}"; @@ -861,7 +872,16 @@ void testJsonStringBuilderWithExcludeKeys() { } @ParameterizedTest - @CsvSource(value = {"test_key, true", "/test_key, true", "inv(alid, false", "getMetadata(\"test_key\"), false"}) + @CsvSource(value = {"test_key, true", + "/test_key, true", + "inv(alid, false", + "getMetadata(\"test_key\"), false", + "key.with.dot, true", + "key-with-hyphen, true", + "key_with_underscore, true", + "key@with@at, true", + "key[with]brackets, true" + }) void isValidEventKey_returns_expected_result(final String key, final boolean isValid) { assertThat(JacksonEvent.isValidEventKey(key), equalTo(isValid)); } @@ -882,4 +902,24 @@ private static Map createComplexDataMap() { return dataObject; } + @ParameterizedTest + @MethodSource("getBigDecimalPutTestData") + void testPutAndGet_withBigDecimal(final String value) { + final String key = "bigDecimalKey"; + event.put(key, new BigDecimal(value)); + final Object result = event.get(key, Object.class); + assertThat(result, is(notNullValue())); + assertThat(result.toString(), is(equalTo(value))); + } + + private static Stream getBigDecimalPutTestData() { + return Stream.of( + Arguments.of("702062202420"), + Arguments.of("1.23345E+9"), + Arguments.of("1.2345E+60"), + Arguments.of("1.2345E+6"), + Arguments.of("1.000") + ); + } + } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/BigDecimalConverterTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/BigDecimalConverterTests.java new file mode 100644 index 0000000000..5aebb426a2 --- /dev/null +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/BigDecimalConverterTests.java @@ -0,0 +1,109 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.typeconverter; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.Collections; +import java.util.Map; +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class BigDecimalConverterTests { + @Test + void testStringToBigDecimalConversion() { + BigDecimalConverter converter = new BigDecimalConverter(); + final String stringConstant = "12345678912.12345"; + assertThat(converter.convert(stringConstant), equalTo(new BigDecimal(stringConstant))); + assertThat(converter.convert(stringConstant, () -> 0), equalTo(new BigDecimal(stringConstant))); + } + + @Test + void testIntegerToBigDecimalConversion() { + BigDecimalConverter converter = new BigDecimalConverter(); + final int intConstant = 12345; + assertThat(converter.convert(intConstant), equalTo(BigDecimal.valueOf(intConstant))); + assertThat(converter.convert(intConstant, () -> 0), equalTo(new BigDecimal(intConstant))); + } + + @Test + void testLongToBigDecimalConversion() { + BigDecimalConverter converter = new BigDecimalConverter(); + final long longConstant = 123456789012L; + assertThat(converter.convert(longConstant).longValue(), equalTo(longConstant)); + assertThat(converter.convert(longConstant, () -> 0).longValue(), equalTo(longConstant)); + } + + @Test + void testBooleanToBigDecimalConversion() { + BigDecimalConverter converter = new BigDecimalConverter(); + final Boolean boolFalseConstant = false; + assertThat(converter.convert(boolFalseConstant), equalTo(BigDecimal.valueOf(0))); + final Boolean boolTrueConstant = true; + assertThat(converter.convert(boolTrueConstant), equalTo(BigDecimal.valueOf(1))); + assertThat(converter.convert(boolTrueConstant, () -> 0), equalTo(BigDecimal.valueOf(1))); + } + + @Test + void testFloatToBigDecimalConversion() { + BigDecimalConverter converter = new BigDecimalConverter(); + final float fval = 12345.6789f; + assertThat(converter.convert(fval).floatValue(), equalTo(fval)); + assertThat(converter.convert(fval, () -> 0).floatValue(), equalTo(fval)); + } + + @Test + void testBigDecimalToBigDecimalConversion() { + BigDecimalConverter converter = new BigDecimalConverter(); + BigDecimal bigDecimal = new BigDecimal("12345.6789"); + assertThat(converter.convert(bigDecimal), equalTo(bigDecimal)); + assertThat(converter.convert(bigDecimal, () -> 0), equalTo(bigDecimal)); + } + + @ParameterizedTest + @MethodSource("decimalToBigDecimalValueProvider") + void testDoubleToBigDecimalConversion(BigDecimal expectedBigDecimal, double actualValue, int scale) { + BigDecimalConverter converter = new BigDecimalConverter(); + if(scale!=0) { + expectedBigDecimal = expectedBigDecimal.setScale(scale, RoundingMode.HALF_EVEN); + } + assertThat(converter.convert(actualValue, scale), equalTo(expectedBigDecimal)); + assertThat(converter.convert(actualValue, () -> scale), equalTo(expectedBigDecimal)); + } + + private static Stream decimalToBigDecimalValueProvider() { + return Stream.of( + Arguments.of(new BigDecimal ("0.0"), 0, 1), + Arguments.of(new BigDecimal ("0.0"), 0.0, 1), + Arguments.of(new BigDecimal ("0.00000000000000000000000"), 0.00000000000000000000000, 1), + Arguments.of(BigDecimal.ZERO, BigDecimal.ZERO.doubleValue(), 1), + Arguments.of(new BigDecimal ("1"), (double)1, 1), + Arguments.of(new BigDecimal ("1703908514.045833"), 1703908514.045833, 6), + Arguments.of(new BigDecimal ("1.00000000000000000000000"), 1.00000000000000000000000, 1), + Arguments.of(new BigDecimal ("-12345678912.12345"), -12345678912.12345, 1), + Arguments.of(BigDecimal.ONE, BigDecimal.ONE.doubleValue(), 1), + Arguments.of(new BigDecimal("1.7976931348623157E+308"), 1.7976931348623157E+308, 0), + Arguments.of(new BigDecimal("1702062202420"), 1.70206220242E+12, 12), + Arguments.of(BigDecimal.valueOf(Double.MAX_VALUE), Double.MAX_VALUE, 0), + Arguments.of(BigDecimal.valueOf(Double.MIN_VALUE), Double.MIN_VALUE, 0) + ); + } + + @Test + void testInvalidBigDecimalConversion() { + BigDecimalConverter converter = new BigDecimalConverter(); + final Map map = Collections.emptyMap(); + assertThrows(IllegalArgumentException.class, () -> converter.convert(map)); + } +} diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/BooleanConverterTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/BooleanConverterTests.java index c29bcddb0c..354f7a610f 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/BooleanConverterTests.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/BooleanConverterTests.java @@ -21,11 +21,19 @@ import java.util.stream.Stream; public class BooleanConverterTests { + + @Test + void testStringToBooleanConversionWithArguments() { + BooleanConverter converter = new BooleanConverter(); + final String stringConstant = "12345678912.12345"; + assertThat(converter.convert(stringConstant, () -> 0), equalTo(false)); + } + @Test void testStringToBooleanConversion() { BooleanConverter converter = new BooleanConverter(); final String stringConstant = "12345678912.12345"; - assertThat(converter.convert(stringConstant), equalTo(Boolean.parseBoolean(stringConstant))); + assertThat(converter.convert(stringConstant), equalTo(false)); } @Test void testIntegerToBooleanConversion() { @@ -46,7 +54,7 @@ void testDoubleToBooleanConversion() { @Test void testLongToBooleanConversion() { BooleanConverter converter = new BooleanConverter(); - final Long longTrueConstant = (long)1234578912; + final Long longTrueConstant = 1234578912345L; assertThat(converter.convert(longTrueConstant), equalTo(true)); final Long longFalseConstant = (long)0; assertThat(converter.convert(longFalseConstant), equalTo(false)); diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/DoubleConverterTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/DoubleConverterTests.java index e10121e2c6..8008bfc226 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/DoubleConverterTests.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/DoubleConverterTests.java @@ -25,6 +25,7 @@ void testStringToDoubleConversion() { DoubleConverter converter = new DoubleConverter(); final String stringConstant = "12345678912.12345"; assertThat(converter.convert(stringConstant), equalTo(Double.parseDouble(stringConstant))); + assertThat(converter.convert(stringConstant, () -> 0), equalTo(Double.parseDouble(stringConstant))); } @Test void testIntegerToDoubleConversion() { @@ -43,7 +44,7 @@ void testBooleanToDoubleConversion() { @Test void testDoubleToDoubleConversion() { DoubleConverter converter = new DoubleConverter(); - final Double doubleConstant = (double)12345.123; + final Double doubleConstant = 12345.123; assertThat(converter.convert(doubleConstant), equalTo(doubleConstant)); } @ParameterizedTest @@ -56,16 +57,16 @@ private static Stream BigDecimalValueProvider() { return Stream.of( Arguments.of(new BigDecimal ("0"), (double)0), Arguments.of(new BigDecimal ("0.0"), (double)0), - Arguments.of(new BigDecimal ("0.00000000000000000000000"), (double)0.00000000000000000000000), + Arguments.of(new BigDecimal ("0.00000000000000000000000"), 0.00000000000000000000000), Arguments.of(BigDecimal.ZERO, BigDecimal.ZERO.doubleValue()), Arguments.of(new BigDecimal ("1"), (double)1), - Arguments.of(new BigDecimal ("1703908514.045833"), (double)1703908514.045833), - Arguments.of(new BigDecimal ("1.00000000000000000000000"), (double)1.00000000000000000000000), - Arguments.of(new BigDecimal ("-12345678912.12345"), (double)-12345678912.12345), + Arguments.of(new BigDecimal ("1703908514.045833"), 1703908514.045833), + Arguments.of(new BigDecimal ("1.00000000000000000000000"), 1.00000000000000000000000), + Arguments.of(new BigDecimal ("-12345678912.12345"), -12345678912.12345), Arguments.of(BigDecimal.ONE, BigDecimal.ONE.doubleValue()), - Arguments.of(new BigDecimal("1.7976931348623157E+308"), (double)1.7976931348623157E+308), - Arguments.of(new BigDecimal(Double.MAX_VALUE), (double)Double.MAX_VALUE), - Arguments.of(new BigDecimal(Double.MIN_VALUE), (double)Double.MIN_VALUE) + Arguments.of(new BigDecimal("1.7976931348623157E+308"), 1.7976931348623157E+308), + Arguments.of(new BigDecimal(Double.MAX_VALUE), Double.MAX_VALUE), + Arguments.of(new BigDecimal(Double.MIN_VALUE), Double.MIN_VALUE) ); } @Test diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/IntegerConverterTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/IntegerConverterTests.java index 958c24b8d5..ed993a9ba0 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/IntegerConverterTests.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/IntegerConverterTests.java @@ -23,6 +23,7 @@ void testStringToIntegerConversion() { IntegerConverter converter = new IntegerConverter(); final String stringConstant = "1234"; assertThat(converter.convert(stringConstant), equalTo(Integer.parseInt(stringConstant))); + assertThat(converter.convert(stringConstant, () -> 0), equalTo(Integer.parseInt(stringConstant))); } @Test void testFloatToIntegerConversion() { @@ -41,7 +42,7 @@ void testBooleanToIntegerConversion() { @Test void testIntegerToIntegerConversion() { IntegerConverter converter = new IntegerConverter(); - final Integer intConstant = (int)1234; + final Integer intConstant = 1234; assertThat(converter.convert(intConstant), equalTo(intConstant)); } @ParameterizedTest @@ -52,18 +53,18 @@ void testBigDecimalToIntegerConversion(BigDecimal bigDecimalConstant, int expect } private static Stream BigDecimalValueProvider() { return Stream.of( - Arguments.of(new BigDecimal ("0"), (int)0), - Arguments.of(new BigDecimal ("0.0"), (int)0), - Arguments.of(new BigDecimal ("0.00000000000000000000000"), (int)0), + Arguments.of(new BigDecimal ("0"), 0), + Arguments.of(new BigDecimal ("0.0"), 0), + Arguments.of(new BigDecimal ("0.00000000000000000000000"), 0), Arguments.of(BigDecimal.ZERO, BigDecimal.ZERO.intValue()), - Arguments.of(new BigDecimal ("1"), (int)1), - Arguments.of(new BigDecimal ("1703908514.045833"), (int)1703908514), - Arguments.of(new BigDecimal ("1.00000000000000000000000"), (int)1), - Arguments.of(new BigDecimal ("-12345678.12345"), (int)-12345678), + Arguments.of(new BigDecimal ("1"), 1), + Arguments.of(new BigDecimal ("1703908514.045833"), 1703908514), + Arguments.of(new BigDecimal ("1.00000000000000000000000"), 1), + Arguments.of(new BigDecimal ("-12345678.12345"), -12345678), Arguments.of(BigDecimal.ONE, BigDecimal.ONE.intValue()), - Arguments.of(new BigDecimal("1.7976931348623157E+308"), (int)0), - Arguments.of(new BigDecimal(Integer.MAX_VALUE), (int)Integer.MAX_VALUE), - Arguments.of(new BigDecimal(Integer.MIN_VALUE), (int)Integer.MIN_VALUE) + Arguments.of(new BigDecimal("1.7976931348623157E+308"), 0), + Arguments.of(new BigDecimal(Integer.MAX_VALUE), Integer.MAX_VALUE), + Arguments.of(new BigDecimal(Integer.MIN_VALUE), Integer.MIN_VALUE) ); } @Test diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/LongConverterTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/LongConverterTests.java index 5b72fdf0d9..e1e8ebc424 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/LongConverterTests.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/LongConverterTests.java @@ -24,18 +24,19 @@ public class LongConverterTests { void testStringToLongConversion(String stringValue) { LongConverter converter = new LongConverter(); assertThat(converter.convert(stringValue), equalTo(Long.parseLong(stringValue))); + assertThat(converter.convert(stringValue, () -> 0), equalTo(Long.parseLong(stringValue))); } @ParameterizedTest @ValueSource(floats = {(float)1234.56789, Float.MAX_VALUE, Float.MIN_VALUE}) void testfloatToLongConversion(float floatValue) { LongConverter converter = new LongConverter(); - assertThat(converter.convert(floatValue), equalTo((long)(float)floatValue)); + assertThat(converter.convert(floatValue), equalTo((long) floatValue)); } @ParameterizedTest @ValueSource(doubles = {12345678.12345678, 2.0 * Integer.MAX_VALUE, Double.MAX_VALUE, Double.MIN_VALUE}) void testDoubleToLongConversion(double doubleValue) { LongConverter converter = new LongConverter(); - assertThat(converter.convert(doubleValue), equalTo((long)(double)doubleValue)); + assertThat(converter.convert(doubleValue), equalTo((long) doubleValue)); } @ParameterizedTest @ValueSource(booleans = {false,true}) @@ -77,8 +78,8 @@ private static Stream BigDecimalValueProvider() { Arguments.of(new BigDecimal("1.7976931348623157E+308"), (long)0), Arguments.of(new BigDecimal(Integer.MAX_VALUE), (long)Integer.MAX_VALUE), Arguments.of(new BigDecimal(Integer.MIN_VALUE), (long)Integer.MIN_VALUE), - Arguments.of(new BigDecimal(Long.MAX_VALUE), (long)Long.MAX_VALUE), - Arguments.of(new BigDecimal(Long.MIN_VALUE), (long)Long.MIN_VALUE), + Arguments.of(new BigDecimal(Long.MAX_VALUE), Long.MAX_VALUE), + Arguments.of(new BigDecimal(Long.MIN_VALUE), Long.MIN_VALUE), Arguments.of(new BigDecimal("267694723"), (long)267694723) ); diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/StringConverterTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/StringConverterTests.java index b9ad5c8f7d..b67648976a 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/StringConverterTests.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/StringConverterTests.java @@ -25,6 +25,7 @@ void testLongToStringConversion() { StringConverter converter = new StringConverter(); final Long longConstant = (long)100000000 * (long)10000000; assertThat(converter.convert(longConstant), equalTo(longConstant.toString())); + assertThat(converter.convert(longConstant, () -> 0), equalTo(longConstant.toString())); } @Test void testDoubleToStringConversion() { diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializerTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializerTest.java index d0edd69049..8756e97134 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializerTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializerTest.java @@ -16,6 +16,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; +import java.math.BigDecimal; import java.time.Duration; import java.util.stream.Stream; @@ -53,6 +54,7 @@ private static Stream getScalarTypeArguments() { Arguments.of(Long.class, 200L), Arguments.of(Double.class, 1.23d), Arguments.of(Float.class, 2.15f), - Arguments.of(Character.class, 'c')); + Arguments.of(Character.class, 'c'), + Arguments.of(BigDecimal.class, 1.2345E+5)); } } \ No newline at end of file diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java index ab5a125321..386b7fd826 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java @@ -21,6 +21,7 @@ import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; import java.io.IOException; +import java.math.BigDecimal; import java.time.Duration; import java.util.Collections; import java.util.Map; @@ -113,6 +114,7 @@ private static Stream getNonStringTypeArguments() { Arguments.of(Long.class, "200", 200L), Arguments.of(Double.class, "1.23", 1.23d), Arguments.of(Float.class, "2.15", 2.15f), + Arguments.of(BigDecimal.class, "2.15", BigDecimal.valueOf(2.15)), Arguments.of(Map.class, "{}", Collections.emptyMap())); } @@ -127,6 +129,7 @@ private static Stream getStringTypeArguments() { Arguments.of(Long.class, "\"200\"", 200L), Arguments.of(Double.class, "\"1.23\"", 1.23d), Arguments.of(Float.class, "\"2.15\"", 2.15f), + Arguments.of(BigDecimal.class, "\"2.15\"", BigDecimal.valueOf(2.15)), Arguments.of(Character.class, "\"c\"", 'c')); } } \ No newline at end of file diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java index 13c28d3345..24f56ef2ba 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java @@ -13,6 +13,7 @@ import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.typeconverter.ConverterArguments; import org.opensearch.dataprepper.typeconverter.TypeConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,17 +34,22 @@ public class ConvertEntryTypeProcessor extends AbstractProcessor, private final List nullValues; private final String type; private final List tagsOnFailure; + private int scale = 0; private final ExpressionEvaluator expressionEvaluator; + private final ConverterArguments converterArguments; @DataPrepperPluginConstructor public ConvertEntryTypeProcessor(final PluginMetrics pluginMetrics, final ConvertEntryTypeProcessorConfig convertEntryTypeProcessorConfig, final ExpressionEvaluator expressionEvaluator) { super(pluginMetrics); + this.converterArguments = convertEntryTypeProcessorConfig; this.convertEntryKeys = getKeysToConvert(convertEntryTypeProcessorConfig); - this.type = convertEntryTypeProcessorConfig.getType().name(); - this.converter = convertEntryTypeProcessorConfig.getType().getTargetConverter(); + TargetType targetType = convertEntryTypeProcessorConfig.getType(); + this.type = targetType.name(); + this.converter = targetType.getTargetConverter(); + this.scale = convertEntryTypeProcessorConfig.getScale(); this.convertWhen = convertEntryTypeProcessorConfig.getConvertWhen(); this.nullValues = convertEntryTypeProcessorConfig.getNullValues() .orElse(List.of()); @@ -67,7 +73,7 @@ public Collection> doExecute(final Collection> recor if (keyVal != null) { if (!nullValues.contains(keyVal.toString())) { try { - recordEvent.put(key, converter.convert(keyVal)); + recordEvent.put(key, converter.convert(keyVal, converterArguments)); } catch (final RuntimeException e) { LOG.error(EVENT, "Unable to convert key: {} with value: {} to {}", key, keyVal, type, e); recordEvent.getMetadata().addTags(tagsOnFailure); diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java index 07183f7bcf..448d9bb0a4 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java @@ -6,11 +6,12 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import com.fasterxml.jackson.annotation.JsonProperty; +import org.opensearch.dataprepper.typeconverter.ConverterArguments; import java.util.List; import java.util.Optional; -public class ConvertEntryTypeProcessorConfig { +public class ConvertEntryTypeProcessorConfig implements ConverterArguments { @JsonProperty("key") private String key; @@ -20,6 +21,12 @@ public class ConvertEntryTypeProcessorConfig { @JsonProperty("type") private TargetType type = TargetType.INTEGER; + /** + * Optional scale value used only in the case of BigDecimal converter + */ + @JsonProperty("scale") + private int scale = 0; + @JsonProperty("convert_when") private String convertWhen; @@ -35,9 +42,10 @@ public String getKey() { public List getKeys() { return keys; } - public TargetType getType() { - return type; - } + public TargetType getType() { return type; } + + @Override + public int getScale() { return scale; } public String getConvertWhen() { return convertWhen; } diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/TargetType.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/TargetType.java index 74e516f27c..acf67e8702 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/TargetType.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/TargetType.java @@ -7,15 +7,16 @@ import com.fasterxml.jackson.annotation.JsonCreator; import org.opensearch.dataprepper.model.event.DataType; -import org.opensearch.dataprepper.typeconverter.TypeConverter; -import org.opensearch.dataprepper.typeconverter.IntegerConverter; -import org.opensearch.dataprepper.typeconverter.StringConverter; -import org.opensearch.dataprepper.typeconverter.DoubleConverter; +import org.opensearch.dataprepper.typeconverter.BigDecimalConverter; import org.opensearch.dataprepper.typeconverter.BooleanConverter; +import org.opensearch.dataprepper.typeconverter.DoubleConverter; +import org.opensearch.dataprepper.typeconverter.IntegerConverter; import org.opensearch.dataprepper.typeconverter.LongConverter; +import org.opensearch.dataprepper.typeconverter.StringConverter; +import org.opensearch.dataprepper.typeconverter.TypeConverter; -import java.util.Map; import java.util.Arrays; +import java.util.Map; import java.util.stream.Collectors; public enum TargetType { @@ -23,7 +24,8 @@ public enum TargetType { STRING(DataType.STRING, new StringConverter()), DOUBLE(DataType.DOUBLE, new DoubleConverter()), BOOLEAN(DataType.BOOLEAN, new BooleanConverter()), - LONG(DataType.LONG, new LongConverter()); + LONG(DataType.LONG, new LongConverter()), + BIG_DECIMAL(DataType.BIG_DECIMAL, new BigDecimalConverter()); private static final Map OPTIONS_MAP = Arrays.stream(TargetType.values()) .collect(Collectors.toMap( diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java index 0e9ffd9502..5f8b66a6a6 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java @@ -8,6 +8,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.expression.ExpressionEvaluator; @@ -16,17 +19,18 @@ import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; +import java.math.BigDecimal; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.math.BigDecimal; +import java.util.stream.Stream; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; @@ -53,21 +57,21 @@ static Record buildRecordWithEvent(final Map data) { } @BeforeEach - private void setup() { + void setup() { lenient().when(mockConfig.getKey()).thenReturn(TEST_KEY); lenient().when(mockConfig.getKeys()).thenReturn(null); lenient().when(mockConfig.getConvertWhen()).thenReturn(null); } - private Record getMessage(String message, String key, Object value) { - final Map testData = new HashMap(); + private Record getMessage(String message, Object value) { + final Map testData = new HashMap<>(); testData.put("message", message); - testData.put(key, value); + testData.put(ConvertEntryTypeProcessorTests.TEST_KEY, value); return buildRecordWithEvent(testData); } private Event executeAndGetProcessedEvent(final Object testValue) { - final Record record = getMessage(UUID.randomUUID().toString(), TEST_KEY, testValue); + final Record record = getMessage(UUID.randomUUID().toString(), testValue); final List> processedRecords = (List>) typeConversionProcessor.doExecute(Collections.singletonList(record)); assertThat(processedRecords.size(), equalTo(1)); assertThat(processedRecords.get(0), notNullValue()); @@ -103,6 +107,50 @@ void testBigDecimalToIntegerConvertEntryTypeProcessor() { assertThat(event.get(TEST_KEY, Integer.class), equalTo(testValue.intValue())); } + @Test + void testDecimalToBigDecimalConvertEntryTypeProcessor() { + BigDecimal testValue = new BigDecimal(Integer.MAX_VALUE); + when(mockConfig.getType()).thenReturn(TargetType.fromOptionValue("big_decimal")); + typeConversionProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); + Event event = executeAndGetProcessedEvent(testValue.toString()); + assertThat(event.get(TEST_KEY, BigDecimal.class), equalTo(testValue)); + } + + @Test + void testDecimalToBigDecimalWithScaleConvertEntryTypeProcessor() { + String testValue = "2147483647"; + TargetType bigdecimalTargetType = TargetType.fromOptionValue("big_decimal"); + when(mockConfig.getType()).thenReturn(bigdecimalTargetType); + when(mockConfig.getScale()).thenReturn(5); + typeConversionProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); + Event event = executeAndGetProcessedEvent(testValue); + //As we set the scale to 5, we expect to see 5 positions filled with zeros + assertThat(event.get(TEST_KEY, BigDecimal.class), equalTo(new BigDecimal(testValue+".00000"))); + } + + @ParameterizedTest + @MethodSource("decimalFormatKeysArgumentProvider") + void testDecimalToBigDecimalWithRoundingConvertEntryTypeProcessor(String source, String target) { + + TargetType bigdecimalTargetType = TargetType.fromOptionValue("big_decimal"); + when(mockConfig.getType()).thenReturn(bigdecimalTargetType); + when(mockConfig.getScale()).thenReturn(5); + typeConversionProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); + //Default HALF_ROUND_UP applied for all the conversions + Event event1 = executeAndGetProcessedEvent(source); + assertThat(event1.get(TEST_KEY, BigDecimal.class), equalTo(new BigDecimal(target))); + } + + private static Stream decimalFormatKeysArgumentProvider() { + //Default HALF_ROUND_UP applied for all the conversions + return Stream.of( + Arguments.of("1703908412.707011", "1703908412.70701"), + Arguments.of("1703908412.707016", "1703908412.70702"), + Arguments.of("1703908412.707015", "1703908412.70702"), + Arguments.of("1703908412.707014", "1703908412.70701") + ); + } + @Test void testBooleanToIntegerConvertEntryTypeProcessor() { int testValue = 1; @@ -203,7 +251,7 @@ void testBigDecimalToStringConvertEntryTypeProcessor() { @Test void testDoubleToStringConvertEntryTypeProcessor() { - Double testValue = (double)123.456; + Double testValue = 123.456; String expectedValue = testValue.toString(); when(mockConfig.getType()).thenReturn(TargetType.fromOptionValue("string")); typeConversionProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); @@ -242,7 +290,7 @@ void testNoConversionWhenConvertWhenIsFalse() { when(mockConfig.getType()).thenReturn(TargetType.fromOptionValue("integer")); when(mockConfig.getConvertWhen()).thenReturn(convertWhen); - final Record record = getMessage(UUID.randomUUID().toString(), TEST_KEY, testValue); + final Record record = getMessage(UUID.randomUUID().toString(), testValue); when(expressionEvaluator.evaluateConditional(convertWhen, record.getData())).thenReturn(false); typeConversionProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); Event event = executeAndGetProcessedEvent(record); @@ -258,11 +306,11 @@ void testMultipleKeysConvertEntryTypeProcessor() { when(mockConfig.getKey()).thenReturn(null); when(mockConfig.getKeys()).thenReturn(List.of(testKey1, testKey2)); when(mockConfig.getType()).thenReturn(TargetType.fromOptionValue("string")); - final Map testData = new HashMap(); + final Map testData = new HashMap<>(); testData.put("message", "testMessage"); testData.put(testKey1, testValue); testData.put(testKey2, testValue); - Record record = buildRecordWithEvent(testData); + Record record = buildRecordWithEvent(testData); typeConversionProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); Event event = executeAndGetProcessedEvent(record); assertThat(event.get(testKey1, String.class), equalTo(expectedValue)); diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/TargetTypeTest.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/TargetTypeTest.java index e2565f7e04..0b653fc766 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/TargetTypeTest.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/TargetTypeTest.java @@ -39,7 +39,8 @@ public Stream provideArguments(final ExtensionContext exten arguments(DataType.BOOLEAN.getTypeName(), TargetType.BOOLEAN), arguments(DataType.INTEGER.getTypeName(), TargetType.INTEGER), arguments(DataType.LONG.getTypeName(), TargetType.LONG), - arguments(DataType.DOUBLE.getTypeName(), TargetType.DOUBLE) + arguments(DataType.DOUBLE.getTypeName(), TargetType.DOUBLE), + arguments(DataType.BIG_DECIMAL.getTypeName(), TargetType.BIG_DECIMAL) ); } }