From 382b8ec81dc59e72da1b6394cefa1d4665cf2b1f Mon Sep 17 00:00:00 2001 From: Hai Yan Date: Thu, 16 Jan 2025 15:44:25 -0600 Subject: [PATCH] Support null values Signed-off-by: Hai Yan --- .../mutateevent/MapToListProcessor.java | 14 ++-- .../mutateevent/MapToListProcessorTest.java | 66 +++++++++++++++++++ 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java index af291ebc05..d5af649c84 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java @@ -6,7 +6,6 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; @@ -22,6 +21,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -33,7 +33,6 @@ @DataPrepperPlugin(name = "map_to_list", pluginType = Processor.class, pluginConfigurationType = MapToListProcessorConfig.class) public class MapToListProcessor extends AbstractProcessor, Record> { private static final Logger LOG = LoggerFactory.getLogger(MapToListProcessor.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final MapToListProcessorConfig config; private final ExpressionEvaluator expressionEvaluator; private final Set excludeKeySet = new HashSet<>(); @@ -72,9 +71,8 @@ public Collection> doExecute(final Collection> recor for (final Map.Entry entry : sourceMap.entrySet()) { if (!excludeKeySet.contains(entry.getKey())) { - targetNestedList.add(List.of(entry.getKey(), entry.getValue())); + targetNestedList.add(Arrays.asList(entry.getKey(), entry.getValue())); } - } removeProcessedFields(sourceMap, recordEvent); recordEvent.put(config.getTarget(), targetNestedList); @@ -82,10 +80,10 @@ public Collection> doExecute(final Collection> recor final List> targetList = new ArrayList<>(); for (final Map.Entry entry : sourceMap.entrySet()) { if (!excludeKeySet.contains(entry.getKey())) { - targetList.add(Map.of( - config.getKeyName(), entry.getKey(), - config.getValueName(), entry.getValue() - )); + final Map listItem = new HashMap<>(); + listItem.put(config.getKeyName(), entry.getKey()); + listItem.put(config.getValueName(), entry.getValue()); + targetList.add(listItem); } } removeProcessedFields(sourceMap, recordEvent); diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java index 1b2ca68833..6e9717c103 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java @@ -18,7 +18,9 @@ import org.opensearch.dataprepper.model.record.Record; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -27,6 +29,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; @@ -357,6 +360,51 @@ void testFailureTagsAreAddedWhenException() { assertThat(resultEvent.getMetadata().getTags(), is(new HashSet<>(testTags))); } + @Test + void testMapToListSuccessWithNullValuesInMap() { + + final MapToListProcessor processor = createObjectUnderTest(); + final Record testRecord = createTestRecordWithNullValues(); + final List> resultRecord = (List>) processor.doExecute(Collections.singletonList(testRecord)); + + assertThat(resultRecord.size(), is(1)); + + final Event resultEvent = resultRecord.get(0).getData(); + List> resultList = resultEvent.get("my-list", List.class); + + assertThat(resultList.size(), is(2)); + Map resultMapWithNullValue = new HashMap<>(); + resultMapWithNullValue.put("key", "key2"); + resultMapWithNullValue.put("value", null); + assertThat(resultList, containsInAnyOrder( + Map.of("key", "key1", "value", "value1"), + resultMapWithNullValue + )); + assertThat(resultEvent.containsKey("my-map"), is(true)); + assertSourceMapUnchangedWithNullValues(resultEvent); + } + + @Test + public void testConvertFieldToListSuccessWithNullValuesInMap() { + when(mockConfig.getConvertFieldToList()).thenReturn(true); + + final MapToListProcessor processor = createObjectUnderTest(); + final Record testRecord = createTestRecordWithNullValues(); + final List> resultRecord = (List>) processor.doExecute(Collections.singletonList(testRecord)); + + assertThat(resultRecord.size(), is(1)); + + final Event resultEvent = resultRecord.get(0).getData(); + List> resultList = resultEvent.get("my-list", List.class); + + assertThat(resultList.size(), is(2)); + assertThat(resultList, containsInAnyOrder( + Arrays.asList("key1", "value1"), + Arrays.asList("key2", null) + )); + assertSourceMapUnchangedWithNullValues(resultEvent); + } + private MapToListProcessor createObjectUnderTest() { return new MapToListProcessor(pluginMetrics, mockConfig, expressionEvaluator); } @@ -396,6 +444,18 @@ private Record createTestRecordWithNestedMap() { return new Record<>(event); } + private Record createTestRecordWithNullValues() { + final Map mapData = new HashMap<>(); + mapData.put("key1", "value1"); + mapData.put("key2", null); + final Map> data = Map.of("my-map", mapData); + final Event event = JacksonEvent.builder() + .withData(data) + .withEventType("event") + .build(); + return new Record<>(event); + } + private void assertSourceMapUnchanged(final Event resultEvent) { assertThat(resultEvent.containsKey("my-map"), is(true)); assertThat(resultEvent.get("my-map/key1", String.class), is("value1")); @@ -408,4 +468,10 @@ private void assertSourceMapUnchangedForFlatRecord(final Event resultEvent) { assertThat(resultEvent.get("key2", String.class), is("value2")); assertThat(resultEvent.get("key3", String.class), is("value3")); } + + private void assertSourceMapUnchangedWithNullValues(final Event resultEvent) { + assertThat(resultEvent.containsKey("my-map"), is(true)); + assertThat(resultEvent.get("my-map/key1", String.class), is("value1")); + assertThat(resultEvent.get("my-map/key2", String.class), nullValue()); + } }