From f10867fa0a8d88c3c24170e809ecae72a40d2677 Mon Sep 17 00:00:00 2001 From: David Venable Date: Wed, 23 Oct 2024 10:57:26 -0500 Subject: [PATCH] ExampleValues annotation for schema generation along with date examples (#5088) Introduces ExampleValues annotation for schema and documentation generation. Adds a new annotation ExampleValues. Updates the schema generation with an InstanceAttributeOverride to add the examples to the schemas. Adds ExampleValues to the date processor. Signed-off-by: David Venable --- .../model/annotations/ExampleValues.java | 52 +++++ ...xampleValuesInstanceAttributeOverride.java | 50 +++++ .../schemas/JsonSchemaConverter.java | 1 + ...leValuesInstanceAttributeOverrideTest.java | 187 ++++++++++++++++++ .../schemas/JsonSchemaConverterIT.java | 48 +++++ .../processor/date/DateProcessorConfig.java | 23 +++ 6 files changed, 361 insertions(+) create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ExampleValues.java create mode 100644 data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/ExampleValuesInstanceAttributeOverride.java create mode 100644 data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/ExampleValuesInstanceAttributeOverrideTest.java diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ExampleValues.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ExampleValues.java new file mode 100644 index 0000000000..5ab7593cd3 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ExampleValues.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.model.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * Use this annotation to provide example values for plugin configuration. + * + * @since 2.11 + */ +@Documented +@Retention(RUNTIME) +@Target({FIELD}) +public @interface ExampleValues { + /** + * One or more examples. + * @return the examples. + * @since 2.11 + */ + Example[] value(); + + /** + * A single example. + * + * @since 2.11 + */ + @interface Example { + /** + * The example value + * @return The example value + * + * @since 2.11 + */ + String value(); + + /** + * A description of the example value. + * + * @since 2.11 + */ + String description() default ""; + } +} diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/ExampleValuesInstanceAttributeOverride.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/ExampleValuesInstanceAttributeOverride.java new file mode 100644 index 0000000000..49d332e403 --- /dev/null +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/ExampleValuesInstanceAttributeOverride.java @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.schemas; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.victools.jsonschema.generator.FieldScope; +import com.github.victools.jsonschema.generator.InstanceAttributeOverrideV2; +import com.github.victools.jsonschema.generator.SchemaGenerationContext; +import org.opensearch.dataprepper.model.annotations.ExampleValues; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +class ExampleValuesInstanceAttributeOverride implements InstanceAttributeOverrideV2 { + @Override + public void overrideInstanceAttributes(final ObjectNode fieldSchema, final FieldScope fieldScope, final SchemaGenerationContext context) { + final ExampleValues exampleValuesAnnotation = fieldScope.getAnnotationConsideringFieldAndGetterIfSupported(ExampleValues.class); + if(exampleValuesAnnotation != null && exampleValuesAnnotation.value().length > 0) { + final ObjectMapper objectMapper = context.getGeneratorConfig().getObjectMapper(); + + addExampleSchema(fieldSchema, objectMapper, exampleValuesAnnotation); + } + } + + private void addExampleSchema(final ObjectNode fieldSchema, final ObjectMapper objectMapper, final ExampleValues exampleValuesAnnotation) { + final List> exampleValues = Arrays.stream(exampleValuesAnnotation.value()) + .map(ExampleValuesInstanceAttributeOverride::createExampleMap).collect(Collectors.toList()); + final ArrayNode exampleNode = objectMapper.convertValue(exampleValues, ArrayNode.class); + + fieldSchema.putArray("examples") + .addAll(exampleNode); + } + + private static Map createExampleMap(final ExampleValues.Example example) { + final HashMap exampleMap = new HashMap<>(); + exampleMap.put("example", example.value()); + if(example.description() != null && !example.description().isEmpty()) { + exampleMap.put("description", example.description()); + } + return exampleMap; + } +} diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java index 9bff7b2aa7..78e79c9fa0 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java @@ -53,6 +53,7 @@ public ObjectNode convertIntoJsonSchema( resolveDependentRequiresFields(scopeSchemaGeneratorConfigPart); overrideDataPrepperPluginTypeAttribute(configBuilder.forTypesInGeneral(), schemaVersion, optionPreset); resolveDataPrepperTypes(scopeSchemaGeneratorConfigPart); + scopeSchemaGeneratorConfigPart.withInstanceAttributeOverride(new ExampleValuesInstanceAttributeOverride()); final SchemaGeneratorConfig config = configBuilder.build(); final SchemaGenerator generator = new SchemaGenerator(config); diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/ExampleValuesInstanceAttributeOverrideTest.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/ExampleValuesInstanceAttributeOverrideTest.java new file mode 100644 index 0000000000..152cab2bd3 --- /dev/null +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/ExampleValuesInstanceAttributeOverrideTest.java @@ -0,0 +1,187 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.schemas; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.JsonNodeType; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.victools.jsonschema.generator.FieldScope; +import com.github.victools.jsonschema.generator.SchemaGenerationContext; +import com.github.victools.jsonschema.generator.SchemaGeneratorConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.annotations.ExampleValues; + +import java.util.Map; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ExampleValuesInstanceAttributeOverrideTest { + + @Mock + private ObjectNode fieldSchema; + + @Mock + private FieldScope fieldScope; + + @Mock + private SchemaGenerationContext context; + private ObjectMapper objectMapper; + + @BeforeEach + void setUp() { + objectMapper = new ObjectMapper(); + fieldSchema = spy(objectMapper.convertValue(Map.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()), ObjectNode.class)); + } + + private ExampleValuesInstanceAttributeOverride createObjectUnderTest() { + return new ExampleValuesInstanceAttributeOverride(); + } + + @Test + void overrideInstanceAttributes_does_not_modify_fieldSchema_if_no_ExampleValues_annotation() { + createObjectUnderTest().overrideInstanceAttributes(fieldSchema, fieldScope, context); + + verifyNoInteractions(fieldSchema); + } + + @Nested + class WithExampleValuesAnnotation { + @Mock + private ExampleValues exampleValuesAnnotation; + + @Mock + private SchemaGeneratorConfig schemaGeneratorConfig; + + @BeforeEach + void setUp() { + when(fieldScope.getAnnotationConsideringFieldAndGetterIfSupported(ExampleValues.class)) + .thenReturn(exampleValuesAnnotation); + } + + @Test + void overrideInstanceAttributes_does_not_modify_fieldSchema_if_no_ExampleValues_annotation_is_empty() { + when(exampleValuesAnnotation.value()).thenReturn(new ExampleValues.Example[]{}); + + createObjectUnderTest().overrideInstanceAttributes(fieldSchema, fieldScope, context); + + verifyNoInteractions(fieldSchema); + } + + @Test + void overrideInstanceAttributes_adds_examples_when_one_ExampleValue() { + when(context.getGeneratorConfig()).thenReturn(schemaGeneratorConfig); + when(schemaGeneratorConfig.getObjectMapper()).thenReturn(objectMapper); + + final ExampleValues.Example example = mock(ExampleValues.Example.class); + final String value = UUID.randomUUID().toString(); + final String description = UUID.randomUUID().toString(); + when(example.value()).thenReturn(value); + when(example.description()).thenReturn(description); + final ExampleValues.Example[] examples = {example}; + when(exampleValuesAnnotation.value()).thenReturn(examples); + + createObjectUnderTest().overrideInstanceAttributes(fieldSchema, fieldScope, context); + + final JsonNode examplesNode = fieldSchema.get("examples"); + assertThat(examplesNode, notNullValue()); + + assertThat(examplesNode.getNodeType(), equalTo(JsonNodeType.ARRAY)); + assertThat(examplesNode.size(), equalTo(1)); + final JsonNode firstExampleNode = examplesNode.get(0); + assertThat(firstExampleNode, notNullValue()); + assertThat(firstExampleNode.getNodeType(), equalTo(JsonNodeType.OBJECT)); + assertThat(firstExampleNode.get("example"), notNullValue()); + assertThat(firstExampleNode.get("example").getNodeType(), equalTo(JsonNodeType.STRING)); + assertThat(firstExampleNode.get("example").textValue(), equalTo(value)); + assertThat(firstExampleNode.get("description"), notNullValue()); + assertThat(firstExampleNode.get("description").getNodeType(), equalTo(JsonNodeType.STRING)); + assertThat(firstExampleNode.get("description").textValue(), equalTo(description)); + } + + @Test + void overrideInstanceAttributes_adds_examples_when_one_ExampleValue_with_no_description() { + when(context.getGeneratorConfig()).thenReturn(schemaGeneratorConfig); + when(schemaGeneratorConfig.getObjectMapper()).thenReturn(objectMapper); + + final ExampleValues.Example example = mock(ExampleValues.Example.class); + final String value = UUID.randomUUID().toString(); + final String description = UUID.randomUUID().toString(); + when(example.value()).thenReturn(value); + final ExampleValues.Example[] examples = {example}; + when(exampleValuesAnnotation.value()).thenReturn(examples); + + createObjectUnderTest().overrideInstanceAttributes(fieldSchema, fieldScope, context); + + final JsonNode examplesNode = fieldSchema.get("examples"); + assertThat(examplesNode, notNullValue()); + + assertThat(examplesNode.getNodeType(), equalTo(JsonNodeType.ARRAY)); + assertThat(examplesNode.size(), equalTo(1)); + final JsonNode firstExampleNode = examplesNode.get(0); + assertThat(firstExampleNode, notNullValue()); + assertThat(firstExampleNode.getNodeType(), equalTo(JsonNodeType.OBJECT)); + assertThat(firstExampleNode.get("example"), notNullValue()); + assertThat(firstExampleNode.get("example").getNodeType(), equalTo(JsonNodeType.STRING)); + assertThat(firstExampleNode.get("example").textValue(), equalTo(value)); + assertThat(firstExampleNode.has("description"), equalTo(false)); + } + + @ParameterizedTest + @ValueSource(ints = {2, 3, 5}) + void overrideInstanceAttributes_adds_examples_when_multiple_ExampleValue(final int numberOfExamples) { + when(context.getGeneratorConfig()).thenReturn(schemaGeneratorConfig); + when(schemaGeneratorConfig.getObjectMapper()).thenReturn(objectMapper); + + final ExampleValues.Example[] examples = new ExampleValues.Example[numberOfExamples]; + for (int i = 0; i < numberOfExamples; i++) { + final ExampleValues.Example example = mock(ExampleValues.Example.class); + final String value = UUID.randomUUID().toString(); + final String description = UUID.randomUUID().toString(); + when(example.value()).thenReturn(value); + when(example.description()).thenReturn(description); + + examples[i] = example; + } + when(exampleValuesAnnotation.value()).thenReturn(examples); + + createObjectUnderTest().overrideInstanceAttributes(fieldSchema, fieldScope, context); + + final JsonNode examplesNode = fieldSchema.get("examples"); + assertThat(examplesNode, notNullValue()); + + assertThat(examplesNode.getNodeType(), equalTo(JsonNodeType.ARRAY)); + assertThat(examplesNode.size(), equalTo(numberOfExamples)); + + for (int i = 0; i < numberOfExamples; i++) { + final JsonNode exampleNode = examplesNode.get(0); + assertThat(exampleNode, notNullValue()); + assertThat(exampleNode.getNodeType(), equalTo(JsonNodeType.OBJECT)); + assertThat(exampleNode.get("example"), notNullValue()); + assertThat(exampleNode.get("example").getNodeType(), equalTo(JsonNodeType.STRING)); + assertThat(exampleNode.get("example").textValue(), notNullValue()); + assertThat(exampleNode.get("description"), notNullValue()); + assertThat(exampleNode.get("description").getNodeType(), equalTo(JsonNodeType.STRING)); + assertThat(exampleNode.get("description").textValue(), notNullValue()); + } + } + } +} \ No newline at end of file diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java index 7a3dca5991..fcd0bddebc 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java @@ -1,10 +1,12 @@ package org.opensearch.dataprepper.schemas; import com.fasterxml.jackson.annotation.JsonClassDescription; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.victools.jsonschema.generator.Module; import com.github.victools.jsonschema.generator.OptionPreset; @@ -14,6 +16,7 @@ import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.annotations.ExampleValues; import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.plugin.ClasspathPluginProvider; @@ -23,8 +26,10 @@ import java.util.List; import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; public class JsonSchemaConverterIT { @@ -59,6 +64,38 @@ void testSubTypes() throws JsonProcessingException { anyOfNode.forEach(aggregateActionNode -> assertThat(aggregateActionNode.has(PROPERTIES_KEY), is(true))); } + @Test + void test_examples() throws JsonProcessingException { + final ObjectNode jsonSchemaNode = objectUnderTest.convertIntoJsonSchema( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); + assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); + final JsonNode propertiesNode = jsonSchemaNode.at("/" + PROPERTIES_KEY); + assertThat(propertiesNode, instanceOf(ObjectNode.class)); + assertThat(propertiesNode.has("string_value_with_two_examples"), is(true)); + final JsonNode propertyNode = propertiesNode.at("/string_value_with_two_examples"); + assertThat(propertyNode.has("type"), equalTo(true)); + assertThat(propertyNode.get("type").getNodeType(), equalTo(JsonNodeType.STRING)); + + assertThat(propertyNode.has("examples"), equalTo(true)); + assertThat(propertyNode.get("examples").getNodeType(), equalTo(JsonNodeType.ARRAY)); + assertThat(propertyNode.get("examples").size(), equalTo(2)); + assertThat(propertyNode.get("examples").get(0), notNullValue()); + assertThat(propertyNode.get("examples").get(0).getNodeType(), equalTo(JsonNodeType.OBJECT)); + assertThat(propertyNode.get("examples").get(0).has("example"), equalTo(true)); + assertThat(propertyNode.get("examples").get(0).get("example").getNodeType(), equalTo(JsonNodeType.STRING)); + assertThat(propertyNode.get("examples").get(0).get("example").textValue(), equalTo("some example value")); + assertThat(propertyNode.get("examples").get(0).has("description"), equalTo(false)); + + assertThat(propertyNode.get("examples").get(1), notNullValue()); + assertThat(propertyNode.get("examples").get(1).getNodeType(), equalTo(JsonNodeType.OBJECT)); + assertThat(propertyNode.get("examples").get(1).has("example"), equalTo(true)); + assertThat(propertyNode.get("examples").get(1).get("example").getNodeType(), equalTo(JsonNodeType.STRING)); + assertThat(propertyNode.get("examples").get(1).get("example").textValue(), equalTo("second example value")); + assertThat(propertyNode.get("examples").get(1).has("description"), equalTo(true)); + assertThat(propertyNode.get("examples").get(1).get("description").getNodeType(), equalTo(JsonNodeType.STRING)); + assertThat(propertyNode.get("examples").get(1).get("description").textValue(), equalTo("This is the second value.")); + } + @JsonClassDescription("test config") static class TestConfig { @JsonPropertyDescription("The aggregate action description") @@ -68,5 +105,16 @@ static class TestConfig { public PluginModel getAction() { return action; } + + @JsonProperty("string_value_with_two_examples") + @ExampleValues({ + @ExampleValues.Example("some example value"), + @ExampleValues.Example(value = "second example value", description = "This is the second value.") + }) + private String stringValueWithTwoExamples; + + public String getStringValueWithTwoExamples() { + return stringValueWithTwoExamples; + } } } diff --git a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java index 901bde4c9d..c81a3a1ffb 100644 --- a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java +++ b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java @@ -12,6 +12,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import jakarta.validation.constraints.AssertTrue; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import java.time.ZoneId; import java.util.List; @@ -47,6 +49,11 @@ public static class DateMatch { "The timestamp value also supports epoch_second, epoch_milli, and epoch_nano values, " + "which represent the timestamp as the number of seconds, milliseconds, and nanoseconds since the epoch. " + "Epoch values always use the UTC time zone.") + @ExampleValues({ + @Example(value = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", description = "Matches ISO-8601 formatted strings."), + @Example(value = "dd/MMM/yyyy:HH:mm:ss Z", description = "Matches Apache Common Log Format."), + @Example(value = "epoch_second", description = "Matches against strings that represent seconds since Unix epoch time.") + }) private List patterns; public DateMatch() { @@ -129,6 +136,10 @@ public static boolean isValidPattern(final String pattern) { @JsonProperty(value = "output_format", defaultValue = DEFAULT_OUTPUT_FORMAT) @JsonPropertyDescription("Determines the format of the timestamp added to an event.") + @ExampleValues({ + @Example(value = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", description = "Outputs ISO-8601 formatted strings."), + @Example(value = "dd/MMM/yyyy:HH:mm:ss Z", description = "Outputs in Apache Common Log Format.") + }) private String outputFormat = DEFAULT_OUTPUT_FORMAT; @JsonProperty("to_origination_metadata") @@ -141,12 +152,20 @@ public static boolean isValidPattern(final String pattern) { "from the value. If the zone or offset are part of the value, then the time zone is ignored. " + "A list of all the available time zones is contained in the TZ database name column of " + "this table.") + @ExampleValues({ + @Example(value = "UTC", description = "Coordinated Universal Time (UTC)."), + @Example(value = "US/Pacific", description = "United States Pacific time zone.") + }) private String sourceTimezone = DEFAULT_SOURCE_TIMEZONE; @JsonProperty("destination_timezone") @JsonPropertyDescription("The time zone used for storing the timestamp in the destination field. " + "A list of all the available time zones is contained in the TZ database name column of " + "this table.") + @ExampleValues({ + @Example(value = "UTC", description = "Coordinated Universal Time (UTC)."), + @Example(value = "US/Pacific", description = "United States Pacific time zone.") + }) private String destinationTimezone = DEFAULT_DESTINATION_TIMEZONE; @JsonProperty("locale") @@ -157,6 +176,10 @@ public static boolean isValidPattern(final String pattern) { "A full list of locale fields, including language, country, and variant, can be found " + "here." + "Default is Locale.ROOT.") + @ExampleValues({ + @Example("en-US"), + @Example("fr-FR") + }) private String locale; @JsonProperty("date_when")