From 4df8500afb75a56b0ad57ab7798fe79abe8ca18b Mon Sep 17 00:00:00 2001 From: Abigael-JT <130055999+Abigael-JT@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:41:23 +0200 Subject: [PATCH] updated schema assertions (#456) * updated schema assertions * removed format changes from spotless * Run spotless --- .../pth10/FormatTransformationTest.java | 200 +++++++++--------- .../pth10/StrcatTransformationTest.java | 110 ++++++++-- .../pth10/TableTransformationTest.java | 45 +++- .../pth10/TimechartStreamingTest.java | 80 ++++++- .../pth10/XmlkvTransformationTest.java | 57 +++-- .../java/com/teragrep/pth10/whereTest.java | 27 ++- 6 files changed, 373 insertions(+), 146 deletions(-) diff --git a/src/test/java/com/teragrep/pth10/FormatTransformationTest.java b/src/test/java/com/teragrep/pth10/FormatTransformationTest.java index 55ffaae9..cb17097a 100644 --- a/src/test/java/com/teragrep/pth10/FormatTransformationTest.java +++ b/src/test/java/com/teragrep/pth10/FormatTransformationTest.java @@ -54,7 +54,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -102,49 +101,49 @@ void tearDown() { void testFormatTransformation() { String q = "index=index_A | format "; - streamingTestUtil - .performDPLTest( - q, testFile, res -> { - // Check if result contains the column that was created for format result - Assertions.assertTrue(Arrays.toString(res.columns()).contains("search")); - - // List of expected values for the format destination field - List expectedValues = Collections - .singletonList( - "( " + "( " + "_time=\"2023-09-06 11:22:31.0\" " + "AND " + "id=\"1\" " - + "AND " + "_raw=\"raw 01\" " + "AND " + "index=\"index_A\" " - + "AND " + "sourcetype=\"A:X:0\" " + "AND " + "host=\"host\" " - + "AND " + "source=\"input\" " + "AND " + "partition=\"0\" " - + "AND offset=\"1\"" + " ) " + "OR " + "( " - + "_time=\"2023-09-06 12:22:31.0\" " + "AND " + "id=\"2\" " + "AND " - + "_raw=\"raw 02\" " + "AND " + "index=\"index_A\" " + "AND " - + "sourcetype=\"A:X:0\" " + "AND " + "host=\"host\" " + "AND " - + "source=\"input\" " + "AND " + "partition=\"0\" " + "AND " - + "offset=\"2\" " + ") " + "OR " + "( " - + "_time=\"2023-09-06 13:22:31.0\" " + "AND id=\"3\" " + "AND " - + "_raw=\"raw 03\" " + "AND " + "index=\"index_A\" " + "AND " - + "sourcetype=\"A:Y:0\" " + "AND " + "host=\"host\" " + "AND " - + "source=\"input\" " + "AND " + "partition=\"0\" " + "AND " - + "offset=\"3\" " + ") " + "OR " + "( " - + "_time=\"2023-09-06 14:22:31.0\" " + "AND " + "id=\"4\" " + "AND " - + "_raw=\"raw 04\" " + "AND " + "index=\"index_A\" " + "AND " - + "sourcetype=\"A:Y:0\" " + "AND " + "host=\"host\" " + "AND " - + "source=\"input\" " + "AND " + "partition=\"0\" " + "AND " - + "offset=\"4\" " + ") " + "OR " + "( " - + "_time=\"2023-09-06 15:22:31.0\" " + "AND " + "id=\"5\" " + "AND " - + "_raw=\"raw 05\" " + "AND " + "index=\"index_A\" " + "AND " - + "sourcetype=\"A:Y:0\" " + "AND " + "host=\"host\" " + "AND " - + "source=\"input\" " + "AND " + "partition=\"0\" " + "AND " - + "offset=\"5\" " + ") " + ")" - ); - - // Destination field from result dataset - List searchAsList = res.select("search").collectAsList().stream().map(r -> r.getString(0)).collect(Collectors.toList()); - - // Assert search field contents as equals with expected contents - Assertions.assertEquals(expectedValues, searchAsList); - } - ); + streamingTestUtil.performDPLTest(q, testFile, res -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("search", DataTypes.StringType, false, new MetadataBuilder().build()) + }); + // Check if result contains the column that was created for format result + Assertions.assertEquals(expectedSchema, res.schema()); + + // List of expected values for the format destination field + List expectedValues = Collections + .singletonList( + "( " + "( " + "_time=\"2023-09-06 11:22:31.0\" " + "AND " + "id=\"1\" " + "AND " + + "_raw=\"raw 01\" " + "AND " + "index=\"index_A\" " + "AND " + + "sourcetype=\"A:X:0\" " + "AND " + "host=\"host\" " + "AND " + "source=\"input\" " + + "AND " + "partition=\"0\" " + "AND offset=\"1\"" + " ) " + "OR " + "( " + + "_time=\"2023-09-06 12:22:31.0\" " + "AND " + "id=\"2\" " + "AND " + + "_raw=\"raw 02\" " + "AND " + "index=\"index_A\" " + "AND " + + "sourcetype=\"A:X:0\" " + "AND " + "host=\"host\" " + "AND " + "source=\"input\" " + + "AND " + "partition=\"0\" " + "AND " + "offset=\"2\" " + ") " + "OR " + "( " + + "_time=\"2023-09-06 13:22:31.0\" " + "AND id=\"3\" " + "AND " + "_raw=\"raw 03\" " + + "AND " + "index=\"index_A\" " + "AND " + "sourcetype=\"A:Y:0\" " + "AND " + + "host=\"host\" " + "AND " + "source=\"input\" " + "AND " + "partition=\"0\" " + + "AND " + "offset=\"3\" " + ") " + "OR " + "( " + + "_time=\"2023-09-06 14:22:31.0\" " + "AND " + "id=\"4\" " + "AND " + + "_raw=\"raw 04\" " + "AND " + "index=\"index_A\" " + "AND " + + "sourcetype=\"A:Y:0\" " + "AND " + "host=\"host\" " + "AND " + "source=\"input\" " + + "AND " + "partition=\"0\" " + "AND " + "offset=\"4\" " + ") " + "OR " + "( " + + "_time=\"2023-09-06 15:22:31.0\" " + "AND " + "id=\"5\" " + "AND " + + "_raw=\"raw 05\" " + "AND " + "index=\"index_A\" " + "AND " + + "sourcetype=\"A:Y:0\" " + "AND " + "host=\"host\" " + "AND " + "source=\"input\" " + + "AND " + "partition=\"0\" " + "AND " + "offset=\"5\" " + ") " + ")" + ); + + // Destination field from result dataset + List searchAsList = res + .select("search") + .collectAsList() + .stream() + .map(r -> r.getString(0)) + .collect(Collectors.toList()); + + // Assert search field contents as equals with expected contents + Assertions.assertEquals(expectedValues, searchAsList); + }); } @Test @@ -155,30 +154,34 @@ void testFormatTransformation() { void testFormatWithMultiValue() { String q = "index=index_A | eval a=mvappend(\"1\", \"2\") | format maxresults=1 "; - streamingTestUtil - .performDPLTest( - q, testFile, res -> { - // Check if result contains the column that was created for format result - Assertions.assertTrue(Arrays.toString(res.columns()).contains("search")); - - // List of expected values for the format destination field - List expectedValues = Collections - .singletonList( - "( " + "( " + "_time=\"2023-09-06 11:22:31.0\" " + "AND " + "id=\"1\" " - + "AND " + "_raw=\"raw 01\" " + "AND " + "index=\"index_A\" " - + "AND " + "sourcetype=\"A:X:0\" " + "AND " + "host=\"host\" " - + "AND " + "source=\"input\" " + "AND " + "partition=\"0\" " - + "AND " + "offset=\"1\" " + "AND " + "( " + "a=\"1\" " + "OR " - + "a=\"2\" " + ") " + ") " + ")" - ); - - // Destination field from result dataset - List searchAsList = res.select("search").collectAsList().stream().map(r -> r.getString(0)).collect(Collectors.toList()); - - // Assert search field contents as equals with expected contents - Assertions.assertEquals(expectedValues, searchAsList); - } - ); + streamingTestUtil.performDPLTest(q, testFile, res -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("search", DataTypes.StringType, false, new MetadataBuilder().build()) + }); + // Check if result contains the column that was created for format result + Assertions.assertEquals(expectedSchema, res.schema()); + + // List of expected values for the format destination field + List expectedValues = Collections + .singletonList( + "( " + "( " + "_time=\"2023-09-06 11:22:31.0\" " + "AND " + "id=\"1\" " + "AND " + + "_raw=\"raw 01\" " + "AND " + "index=\"index_A\" " + "AND " + + "sourcetype=\"A:X:0\" " + "AND " + "host=\"host\" " + "AND " + "source=\"input\" " + + "AND " + "partition=\"0\" " + "AND " + "offset=\"1\" " + "AND " + "( " + + "a=\"1\" " + "OR " + "a=\"2\" " + ") " + ") " + ")" + ); + + // Destination field from result dataset + List searchAsList = res + .select("search") + .collectAsList() + .stream() + .map(r -> r.getString(0)) + .collect(Collectors.toList()); + + // Assert search field contents as equals with expected contents + Assertions.assertEquals(expectedValues, searchAsList); + }); } @Test @@ -189,34 +192,37 @@ void testFormatWithMultiValue() { void testFormatTransformationWithParameters() { String q = "index=index_A | format maxresults=2 \"ROWPRE\" \"COLPRE\" \"COLSEP\" \"COLSUF\"\"ROWSEP\" \"ROWSUF\" "; - streamingTestUtil - .performDPLTest( - q, testFile, res -> { - // Check if result contains the column that was created for format result - Assertions.assertTrue(Arrays.toString(res.columns()).contains("search")); - - // List of expected values for the format destination field - List expectedValues = Collections - .singletonList( - "ROWPRE " + "COLPRE " + "_time=\"2023-09-06 11:22:31.0\" " + "COLSEP " - + "id=\"1\" " + "COLSEP " + "_raw=\"raw 01\" " + "COLSEP " - + "index=\"index_A\" " + "COLSEP " + "sourcetype=\"A:X:0\" " - + "COLSEP " + "host=\"host\" " + "COLSEP " + "source=\"input\" " - + "COLSEP " + "partition=\"0\" " + "COLSEP " + "offset=\"1\"" - + " COLSUF " + "ROWSEP " + "COLPRE " - + "_time=\"2023-09-06 12:22:31.0\" " + "COLSEP " + "id=\"2\" " - + "COLSEP " + "_raw=\"raw 02\" " + "COLSEP " + "index=\"index_A\" " - + "COLSEP " + "sourcetype=\"A:X:0\" " + "COLSEP " + "host=\"host\" " - + "COLSEP " + "source=\"input\" " + "COLSEP " + "partition=\"0\" " - + "COLSEP " + "offset=\"2\" " + "COLSUF " + "ROWSUF" - ); - - // Destination field from result dataset - List searchAsList = res.select("search").collectAsList().stream().map(r -> r.getString(0)).collect(Collectors.toList()); - - // Assert search field contents as equals with expected contents - Assertions.assertEquals(expectedValues, searchAsList); - } - ); + streamingTestUtil.performDPLTest(q, testFile, res -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("search", DataTypes.StringType, false, new MetadataBuilder().build()) + }); + // Check if result contains the column that was created for format result + Assertions.assertEquals(res.schema(), expectedSchema); + + // List of expected values for the format destination field + List expectedValues = Collections + .singletonList( + "ROWPRE " + "COLPRE " + "_time=\"2023-09-06 11:22:31.0\" " + "COLSEP " + "id=\"1\" " + + "COLSEP " + "_raw=\"raw 01\" " + "COLSEP " + "index=\"index_A\" " + "COLSEP " + + "sourcetype=\"A:X:0\" " + "COLSEP " + "host=\"host\" " + "COLSEP " + + "source=\"input\" " + "COLSEP " + "partition=\"0\" " + "COLSEP " + "offset=\"1\"" + + " COLSUF " + "ROWSEP " + "COLPRE " + "_time=\"2023-09-06 12:22:31.0\" " + + "COLSEP " + "id=\"2\" " + "COLSEP " + "_raw=\"raw 02\" " + "COLSEP " + + "index=\"index_A\" " + "COLSEP " + "sourcetype=\"A:X:0\" " + "COLSEP " + + "host=\"host\" " + "COLSEP " + "source=\"input\" " + "COLSEP " + + "partition=\"0\" " + "COLSEP " + "offset=\"2\" " + "COLSUF " + "ROWSUF" + ); + + // Destination field from result dataset + List searchAsList = res + .select("search") + .collectAsList() + .stream() + .map(r -> r.getString(0)) + .collect(Collectors.toList()); + + // Assert search field contents as equals with expected contents + Assertions.assertEquals(expectedValues, searchAsList); + }); } } diff --git a/src/test/java/com/teragrep/pth10/StrcatTransformationTest.java b/src/test/java/com/teragrep/pth10/StrcatTransformationTest.java index ff8f543d..a81407a6 100644 --- a/src/test/java/com/teragrep/pth10/StrcatTransformationTest.java +++ b/src/test/java/com/teragrep/pth10/StrcatTransformationTest.java @@ -107,8 +107,20 @@ void strcatTransformTest() { String q = "index=index_A | strcat _raw sourcetype \"literal\" dest"; streamingTestUtil.performDPLTest(q, testFile, res -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("dest", DataTypes.StringType, false, new MetadataBuilder().build()) + }); // check if result contains the column that was created for strcat result - Assertions.assertTrue(Arrays.toString(res.columns()).contains("dest")); + Assertions.assertEquals(res.schema(), expectedSchema); // List of expected values for the strcat destination field List expectedValues = new ArrayList<>( @@ -145,8 +157,20 @@ void strcatTransformAllRequiredTrueTest() { String q = "index=index_A | strcat allrequired=t _raw \"literal\" sourcetype dest"; streamingTestUtil.performDPLTest(q, testFile, res -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("dest", DataTypes.StringType, false, new MetadataBuilder().build()) + }); // check if result contains the column that was created for strcat result - Assertions.assertTrue(Arrays.toString(res.columns()).contains("dest")); + Assertions.assertEquals(res.schema(), expectedSchema); // List of expected values for the strcat destination field List expectedValues = new ArrayList<>( @@ -183,8 +207,20 @@ void strcatTransformAllRequiredFalseTest() { String q = "index=index_A | strcat allrequired=f _raw sourcetype \"hello world\" dest"; streamingTestUtil.performDPLTest(q, testFile, res -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("dest", DataTypes.StringType, false, new MetadataBuilder().build()) + }); // check if result contains the column that was created for strcat result - Assertions.assertTrue(Arrays.toString(res.columns()).contains("dest")); + Assertions.assertEquals(res.schema(), expectedSchema); // List of expected values for the strcat destination field List expectedValues = new ArrayList<>( @@ -221,22 +257,36 @@ void strcatTransformAllRequiredFalseTest() { void strcatTransformAllRequiredTrueWithMissingFieldTest() { String q = "index=index_A | strcat allrequired=t _raw sourcetype NOT_A_REAL_FIELD \"literal\" dest"; - streamingTestUtil - .performDPLTest( - q, testFile, res -> { - // check if result contains the column that was created for strcat result - Assertions.assertTrue(Arrays.toString(res.columns()).contains("dest")); + streamingTestUtil.performDPLTest(q, testFile, res -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("dest", DataTypes.NullType, true, new MetadataBuilder().build()) + }); + // check if result contains the column that was created for strcat result + Assertions.assertEquals(res.schema(), expectedSchema); - // List of expected values for the strcat destination field - List expectedValues = new ArrayList<>(Arrays.asList(null, null, null, null, null)); + // List of expected values for the strcat destination field + List expectedValues = new ArrayList<>(Arrays.asList(null, null, null, null, null)); - // Destination field from result dataset - List destAsList = res.select("dest").collectAsList().stream().map(r -> r.getString(0)).collect(Collectors.toList()); + // Destination field from result dataset + List destAsList = res + .select("dest") + .collectAsList() + .stream() + .map(r -> r.getString(0)) + .collect(Collectors.toList()); - // assert dest field contents as equals with expected contents - Assertions.assertEquals(expectedValues, destAsList); - } - ); + // assert dest field contents as equals with expected contents + Assertions.assertEquals(expectedValues, destAsList); + }); } // strcat with allRequired=False AND missing(incorrect) field @@ -249,8 +299,20 @@ void strcatTransformAllRequiredFalseWithMissingFieldTest() { String q = "index=index_A | strcat allrequired=f _raw sourcetype \"literal\" NOT_A_REAL_FIELD dest"; streamingTestUtil.performDPLTest(q, testFile, res -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("dest", DataTypes.StringType, false, new MetadataBuilder().build()) + }); // check if result contains the column that was created for strcat result - Assertions.assertTrue(Arrays.toString(res.columns()).contains("dest")); + Assertions.assertEquals(res.schema(), expectedSchema); // List of expected values for the strcat destination field List expectedValues = new ArrayList<>( @@ -287,8 +349,20 @@ void strcatTransformWithMoreThanTwoFields() { String q = "index=index_A | strcat allrequired=f _raw \",\" sourcetype \",\" index dest"; streamingTestUtil.performDPLTest(q, testFile, res -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("dest", DataTypes.StringType, false, new MetadataBuilder().build()) + }); // check if result contains the column that was created for strcat result - Assertions.assertTrue(Arrays.toString(res.columns()).contains("dest")); + Assertions.assertEquals(res.schema(), expectedSchema); // List of expected values for the strcat destination field List expectedValues = new ArrayList<>( diff --git a/src/test/java/com/teragrep/pth10/TableTransformationTest.java b/src/test/java/com/teragrep/pth10/TableTransformationTest.java index 67456a30..451a812f 100644 --- a/src/test/java/com/teragrep/pth10/TableTransformationTest.java +++ b/src/test/java/com/teragrep/pth10/TableTransformationTest.java @@ -54,8 +54,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; - /** * Tests for the new ProcessingStack implementation Uses streaming datasets * @@ -108,8 +106,15 @@ void tearDown() { ) public void testTableWithWildcard() { streamingTestUtil.performDPLTest("index=index_A | strcat _time \"\" _time2 | table _time*", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("_time2", DataTypes.StringType, false, new MetadataBuilder().build()) + }); Assertions - .assertEquals("[_time, _time2]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); }); } @@ -120,8 +125,15 @@ public void testTableWithWildcard() { ) public void testTableMultipleFieldsWithComma() { streamingTestUtil.performDPLTest("index=index_A | table index, offset", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()) + }); Assertions - .assertEquals("[index, offset]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); }); } @@ -132,8 +144,16 @@ public void testTableMultipleFieldsWithComma() { ) public void testTableMultipleFieldsWithSpace() { streamingTestUtil.performDPLTest("index=index_A | table _time offset index", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()) + }); Assertions - .assertEquals("[_time, offset, index]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); }); } @@ -145,8 +165,12 @@ public void testTableMultipleFieldsWithSpace() { ) public void testTableWithSpecialCharactersFieldName() { streamingTestUtil.performDPLTest("index=index_A | table \"?????\"", testFile, ds -> { + final StructType expectedSchema = new StructType(); Assertions - .assertEquals("[]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); }); } @@ -157,8 +181,15 @@ public void testTableWithSpecialCharactersFieldName() { ) public void testTableWithMultipleWildcards() { streamingTestUtil.performDPLTest("index=index_A | strcat _time \"\" _time2 | table *ime*", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("_time2", DataTypes.StringType, false, new MetadataBuilder().build()) + }); Assertions - .assertEquals("[_time, _time2]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); }); } } diff --git a/src/test/java/com/teragrep/pth10/TimechartStreamingTest.java b/src/test/java/com/teragrep/pth10/TimechartStreamingTest.java index 82f41e94..dfff6ee6 100644 --- a/src/test/java/com/teragrep/pth10/TimechartStreamingTest.java +++ b/src/test/java/com/teragrep/pth10/TimechartStreamingTest.java @@ -55,7 +55,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -113,8 +112,26 @@ public void testTimechartBinSizeForMonthSpan() { .performDPLTest( "index=index_A earliest=2020-01-01T00:00:00z latest=2021-01-01T00:00:00z | timechart span=1mon count(_raw) as craw by sourcetype", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField( + "_time", + DataTypes.TimestampType, + false, + new MetadataBuilder().build() + ), + new StructField( + "sourcetype", + DataTypes.StringType, + false, + new MetadataBuilder().build() + ), + new StructField("craw", DataTypes.LongType, false, new MetadataBuilder().build()) + }); Assertions - .assertEquals("[_time, sourcetype, craw]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); List listOfTime = ds.select("_time").collectAsList(); @@ -134,8 +151,26 @@ public void testTimechartBinSizeForMinuteSpan() { .performDPLTest( "index=index_A earliest=2020-12-12T00:00:00z latest=2020-12-12T00:30:00z | timechart span=1min count(_raw) as craw by sourcetype", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField( + "_time", + DataTypes.TimestampType, + false, + new MetadataBuilder().build() + ), + new StructField( + "sourcetype", + DataTypes.StringType, + false, + new MetadataBuilder().build() + ), + new StructField("craw", DataTypes.LongType, false, new MetadataBuilder().build()) + }); Assertions - .assertEquals("[_time, sourcetype, craw]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); List listOfTime = ds.select("_time").collectAsList(); @@ -154,8 +189,26 @@ public void testTimechartSpanWithSplitBY() { streamingTestUtil .performDPLTest( "index=index_A | timechart span=1min count(_raw) as craw by sourcetype", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField( + "_time", + DataTypes.TimestampType, + false, + new MetadataBuilder().build() + ), + new StructField( + "sourcetype", + DataTypes.StringType, + false, + new MetadataBuilder().build() + ), + new StructField("craw", DataTypes.LongType, false, new MetadataBuilder().build()) + }); Assertions - .assertEquals("[_time, sourcetype, craw]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); List listOfSourcetype = ds .select("sourcetype") @@ -183,8 +236,16 @@ public void testTimechartSpanWithSplitBY() { ) public void testTimechartSplitBy() { streamingTestUtil.performDPLTest("index=index_A | timechart count by host", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, false, new MetadataBuilder().build()), + new StructField("count", DataTypes.LongType, false, new MetadataBuilder().build()) + }); Assertions - .assertEquals("[_time, host, count]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); List listOfHosts = ds .select("host") @@ -206,8 +267,15 @@ public void testTimechartSplitBy() { ) public void testTimechartBasicCount() { streamingTestUtil.performDPLTest("index=index_A | timechart count", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()), + new StructField("count", DataTypes.LongType, false, new MetadataBuilder().build()) + }); Assertions - .assertEquals("[_time, count]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); List listOfCount = ds .select("count") diff --git a/src/test/java/com/teragrep/pth10/XmlkvTransformationTest.java b/src/test/java/com/teragrep/pth10/XmlkvTransformationTest.java index b08602ab..09f37485 100644 --- a/src/test/java/com/teragrep/pth10/XmlkvTransformationTest.java +++ b/src/test/java/com/teragrep/pth10/XmlkvTransformationTest.java @@ -54,7 +54,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; import java.util.stream.Collectors; /** @@ -68,15 +67,15 @@ public class XmlkvTransformationTest { private static final Logger LOGGER = LoggerFactory.getLogger(XmlkvTransformationTest.class); private final StructType testSchema = new StructType(new StructField[] { - new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()), - new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()), - new StructField("_raw", DataTypes.StringType, false, new MetadataBuilder().build()), - new StructField("index", DataTypes.StringType, false, new MetadataBuilder().build()), - new StructField("sourcetype", DataTypes.StringType, false, new MetadataBuilder().build()), - new StructField("host", DataTypes.StringType, false, new MetadataBuilder().build()), - new StructField("source", DataTypes.StringType, false, new MetadataBuilder().build()), - new StructField("partition", DataTypes.StringType, false, new MetadataBuilder().build()), - new StructField("offset", DataTypes.LongType, false, new MetadataBuilder().build()) + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()) }); private StreamingTestUtil streamingTestUtil; @@ -113,10 +112,23 @@ void tearDown() { ) public void testXmlkvWithNestedXml() { streamingTestUtil.performDPLTest("index=index_A | xmlkv _raw", XML_DATA_2, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("item", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("something", DataTypes.StringType, true, new MetadataBuilder().build()) + }); Assertions .assertEquals( - "[_time, id, _raw, index, sourcetype, host, source, partition, offset, item, something]", - Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !" + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" ); String result = ds @@ -138,10 +150,21 @@ public void testXmlkvWithNestedXml() { ) public void testXmlkvWithSimpleXml() { streamingTestUtil.performDPLTest("index=index_A | xmlkv _raw", XML_DATA_1, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("item", DataTypes.StringType, true, new MetadataBuilder().build()) + }); Assertions .assertEquals( - "[_time, id, _raw, index, sourcetype, host, source, partition, offset, item]", Arrays - .toString(ds.columns()), + expectedSchema, ds.schema(), "Batch handler dataset contained an unexpected column arrangement !" ); @@ -168,7 +191,11 @@ public void testXmlkvInvalidDataSchema() { "index=index_A | xmlkv _raw", INVALID_DATA, ds -> { // invalid data does not generate a result; only checking column arrangement // to be the same as the input data. - Assertions.assertEquals(Arrays.toString(testSchema.fieldNames()), Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + Assertions + .assertEquals( + testSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); } ); } diff --git a/src/test/java/com/teragrep/pth10/whereTest.java b/src/test/java/com/teragrep/pth10/whereTest.java index 85c01372..ff374d1a 100644 --- a/src/test/java/com/teragrep/pth10/whereTest.java +++ b/src/test/java/com/teragrep/pth10/whereTest.java @@ -272,10 +272,20 @@ void whereWithLikeAndWildcardTest() { ) public void whereTestIntegerColumnLessThan() { streamingTestUtil.performDPLTest("index=index_A | where offset < 3", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField("_time", DataTypes.TimestampType, true, new MetadataBuilder().build()), + new StructField("id", DataTypes.LongType, true, new MetadataBuilder().build()), + new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("index", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("sourcetype", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("host", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("source", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("partition", DataTypes.StringType, true, new MetadataBuilder().build()), + new StructField("offset", DataTypes.LongType, true, new MetadataBuilder().build()) + }); Assertions .assertEquals( - "[_time, id, _raw, index, sourcetype, host, source, partition, offset]", Arrays - .toString(ds.columns()), + expectedSchema, ds.schema(), "Batch handler dataset contained an unexpected column arrangement !" ); @@ -294,8 +304,19 @@ public void whereTestIntegerColumnGreaterThanAfterChart() { "index=index_A " + "| chart avg(offset) as aoffset" + "| chart values(aoffset) as voffset" + "| chart sum(voffset) as soffset" + "| where soffset > 3", testFile, ds -> { + final StructType expectedSchema = new StructType(new StructField[] { + new StructField( + "soffset", + DataTypes.StringType, + true, + new MetadataBuilder().build() + ) + }); Assertions - .assertEquals("[soffset]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + .assertEquals( + expectedSchema, ds.schema(), + "Batch handler dataset contained an unexpected column arrangement !" + ); Assertions.assertEquals(1, ds.collectAsList().size()); }