From e53a4b55aedc1b680f55e5e8e744a5083d379b54 Mon Sep 17 00:00:00 2001 From: Shivam Malhotra Date: Mon, 17 Jun 2024 10:54:10 -0500 Subject: [PATCH] Resolving review comments --- .../parquet/base/PageMaterializer.java | 12 +-- .../InstantFromInt96Materializer.java | 14 +++- ...> InstantNanosFromMicrosMaterializer.java} | 18 ++-- ...> InstantNanosFromMillisMaterializer.java} | 20 +++-- .../LocalDateTimeFromMicrosMaterializer.java | 49 +++++++++++ .../LocalDateTimeFromMillisMaterializer.java | 53 ++++++++++++ .../LocalDateTimeFromNanosMaterializer.java | 53 ++++++++++++ .../LocalDateTimeMaterializer.java | 83 ------------------- .../LocalDateTimePageMaterializerBase.java | 36 ++++++++ .../LocalTimeFromMicrosMaterializer.java | 6 +- .../LocalTimeFromMillisMaterializer.java | 6 +- .../LocalTimeFromNanosMaterializer.java | 6 +- .../LocalTimePageMaterializerBase.java | 12 --- .../LongFromUnsignedIntMaterializer.java | 6 +- .../base/materializers/LongMaterializer.java | 6 +- .../LongPageMaterializerBase.java | 12 --- .../ReplicatePageMaterializers.java | 77 ++++++++++++----- 17 files changed, 304 insertions(+), 165 deletions(-) rename extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/{TimestampNanosFromMicrosMaterializer.java => InstantNanosFromMicrosMaterializer.java} (55%) rename extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/{TimestampNanosFromMillisMaterializer.java => InstantNanosFromMillisMaterializer.java} (54%) create mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromMicrosMaterializer.java create mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromMillisMaterializer.java create mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromNanosMaterializer.java delete mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeMaterializer.java create mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimePageMaterializerBase.java diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PageMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PageMaterializer.java index 808ba26c5f0..c4adfe39069 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PageMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PageMaterializer.java @@ -57,9 +57,9 @@ static PageMaterializerFactory factoryForType(@NotNull final PrimitiveType primi // The column will store nanoseconds elapsed since epoch as long values switch (timestampLogicalType.getUnit()) { case MILLIS: - return TimestampNanosFromMillisMaterializer.Factory; + return InstantNanosFromMillisMaterializer.Factory; case MICROS: - return TimestampNanosFromMicrosMaterializer.Factory; + return InstantNanosFromMicrosMaterializer.Factory; case NANOS: return LongMaterializer.Factory; } @@ -68,11 +68,11 @@ static PageMaterializerFactory factoryForType(@NotNull final PrimitiveType primi // Ref:https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#local-semantics-timestamps-not-normalized-to-utc switch (timestampLogicalType.getUnit()) { case MILLIS: - return LocalDateTimeMaterializer.FromMillisFactory; + return LocalDateTimeFromMillisMaterializer.Factory; case MICROS: - return LocalDateTimeMaterializer.FromMicrosFactory; + return LocalDateTimeFromMicrosMaterializer.Factory; case NANOS: - return LocalDateTimeMaterializer.FromNanosFactory; + return LocalDateTimeFromNanosMaterializer.Factory; } } } else if (logicalTypeAnnotation instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { @@ -101,7 +101,7 @@ static PageMaterializerFactory factoryForType(@NotNull final PrimitiveType primi if (logicalTypeAnnotation instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation) { return StringMaterializer.Factory; } - case FIXED_LEN_BYTE_ARRAY: + case FIXED_LEN_BYTE_ARRAY: // fall through return BlobMaterializer.Factory; default: throw new RuntimeException("Unexpected type name:" + primitiveTypeName); diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantFromInt96Materializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantFromInt96Materializer.java index a9a77361150..b4eddb6cd88 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantFromInt96Materializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantFromInt96Materializer.java @@ -29,8 +29,8 @@ public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int num }; /** - * {@link PageMaterializer} implementation for {@link Instant}s stored as Int96s representing an Impala format - * Timestamp (nanoseconds of day and Julian date encoded as 8 bytes and 4 bytes, respectively) + * {@link PageMaterializer} implementation for {@link Instant Instants} stored as Int96s representing an Impala + * format Timestamp (nanoseconds of day and Julian date encoded as 8 bytes and 4 bytes, respectively) */ private static final class InstantFromInt96PageMaterializer extends LongPageMaterializerBase implements PageMaterializer { @@ -68,11 +68,17 @@ private InstantFromInt96PageMaterializer(ValuesReader dataReader, long nullValue */ private static void setReferenceTimeZone(@NotNull final String timeZone) { offset = DateTimeUtils.nanosOfDay(DateTimeUtils.parseInstant("1970-01-01T00:00:00 " + timeZone), - ZoneId.of("UTC")); + ZoneId.of("UTC"), false); } @Override - long readLong() { + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = readInstantNanos(); + } + } + + long readInstantNanos() { final ByteBuffer resultBuffer = ByteBuffer.wrap(dataReader.readBytes().getBytesUnsafe()); resultBuffer.order(java.nio.ByteOrder.LITTLE_ENDIAN); final long nanos = resultBuffer.getLong(); diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/TimestampNanosFromMicrosMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantNanosFromMicrosMaterializer.java similarity index 55% rename from extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/TimestampNanosFromMicrosMaterializer.java rename to extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantNanosFromMicrosMaterializer.java index a38387852f7..1f40ad9fc1c 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/TimestampNanosFromMicrosMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantNanosFromMicrosMaterializer.java @@ -8,37 +8,39 @@ import io.deephaven.time.DateTimeUtils; import org.apache.parquet.column.values.ValuesReader; -public class TimestampNanosFromMicrosMaterializer { +public class InstantNanosFromMicrosMaterializer { public static final PageMaterializerFactory Factory = new PageMaterializerFactory() { @Override public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { - return new TimestampNanosFromMicrosPageMaterializer(dataReader, (long) nullValue, numValues); + return new InstantNanosFromMicrosPageMaterializer(dataReader, (long) nullValue, numValues); } @Override public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { - return new TimestampNanosFromMicrosPageMaterializer(dataReader, numValues); + return new InstantNanosFromMicrosPageMaterializer(dataReader, numValues); } }; - private static final class TimestampNanosFromMicrosPageMaterializer extends LongPageMaterializerBase + private static final class InstantNanosFromMicrosPageMaterializer extends LongPageMaterializerBase implements PageMaterializer { final ValuesReader dataReader; - private TimestampNanosFromMicrosPageMaterializer(ValuesReader dataReader, int numValues) { + private InstantNanosFromMicrosPageMaterializer(ValuesReader dataReader, int numValues) { this(dataReader, 0, numValues); } - private TimestampNanosFromMicrosPageMaterializer(ValuesReader dataReader, long nullValue, int numValues) { + private InstantNanosFromMicrosPageMaterializer(ValuesReader dataReader, long nullValue, int numValues) { super(nullValue, numValues); this.dataReader = dataReader; } @Override - long readLong() { - return DateTimeUtils.microsToNanos(dataReader.readLong()); + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = DateTimeUtils.microsToNanos(dataReader.readLong()); + } } } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/TimestampNanosFromMillisMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantNanosFromMillisMaterializer.java similarity index 54% rename from extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/TimestampNanosFromMillisMaterializer.java rename to extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantNanosFromMillisMaterializer.java index 9d9f48581c5..0f2767f9fe4 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/TimestampNanosFromMillisMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/InstantNanosFromMillisMaterializer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit TimestampNanosFromMicrosMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// ****** Edit InstantNanosFromMicrosMaterializer and run "./gradlew replicatePageMaterializers" to regenerate // // @formatter:off package io.deephaven.parquet.base.materializers; @@ -12,37 +12,39 @@ import io.deephaven.time.DateTimeUtils; import org.apache.parquet.column.values.ValuesReader; -public class TimestampNanosFromMillisMaterializer { +public class InstantNanosFromMillisMaterializer { public static final PageMaterializerFactory Factory = new PageMaterializerFactory() { @Override public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { - return new TimestampNanosFromMillisPageMaterializer(dataReader, (long) nullValue, numValues); + return new InstantNanosFromMillisPageMaterializer(dataReader, (long) nullValue, numValues); } @Override public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { - return new TimestampNanosFromMillisPageMaterializer(dataReader, numValues); + return new InstantNanosFromMillisPageMaterializer(dataReader, numValues); } }; - private static final class TimestampNanosFromMillisPageMaterializer extends LongPageMaterializerBase + private static final class InstantNanosFromMillisPageMaterializer extends LongPageMaterializerBase implements PageMaterializer { final ValuesReader dataReader; - private TimestampNanosFromMillisPageMaterializer(ValuesReader dataReader, int numValues) { + private InstantNanosFromMillisPageMaterializer(ValuesReader dataReader, int numValues) { this(dataReader, 0, numValues); } - private TimestampNanosFromMillisPageMaterializer(ValuesReader dataReader, long nullValue, int numValues) { + private InstantNanosFromMillisPageMaterializer(ValuesReader dataReader, long nullValue, int numValues) { super(nullValue, numValues); this.dataReader = dataReader; } @Override - long readLong() { - return DateTimeUtils.millisToNanos(dataReader.readLong()); + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = DateTimeUtils.millisToNanos(dataReader.readLong()); + } } } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromMicrosMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromMicrosMaterializer.java new file mode 100644 index 00000000000..998b3012024 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromMicrosMaterializer.java @@ -0,0 +1,49 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import io.deephaven.parquet.base.ParquetTimeUtils; +import org.apache.parquet.column.values.ValuesReader; + +import java.time.LocalDateTime; + +public class LocalDateTimeFromMicrosMaterializer { + + public static final PageMaterializerFactory Factory = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new LocalDateTimeFromMicrosPageMaterializer(dataReader, (LocalDateTime) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new LocalDateTimeFromMicrosPageMaterializer(dataReader, numValues); + } + }; + + private static final class LocalDateTimeFromMicrosPageMaterializer extends LocalDateTimePageMaterializerBase + implements PageMaterializer { + + final ValuesReader dataReader; + + private LocalDateTimeFromMicrosPageMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, null, numValues); + } + + private LocalDateTimeFromMicrosPageMaterializer(ValuesReader dataReader, LocalDateTime nullValue, + int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = ParquetTimeUtils.epochMicrosToLocalDateTimeUTC(dataReader.readLong()); + } + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromMillisMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromMillisMaterializer.java new file mode 100644 index 00000000000..5c2451b181d --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromMillisMaterializer.java @@ -0,0 +1,53 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LocalDateTimeFromMicrosMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import io.deephaven.parquet.base.ParquetTimeUtils; +import org.apache.parquet.column.values.ValuesReader; + +import java.time.LocalDateTime; + +public class LocalDateTimeFromMillisMaterializer { + + public static final PageMaterializerFactory Factory = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new LocalDateTimeFromMillisPageMaterializer(dataReader, (LocalDateTime) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new LocalDateTimeFromMillisPageMaterializer(dataReader, numValues); + } + }; + + private static final class LocalDateTimeFromMillisPageMaterializer extends LocalDateTimePageMaterializerBase + implements PageMaterializer { + + final ValuesReader dataReader; + + private LocalDateTimeFromMillisPageMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, null, numValues); + } + + private LocalDateTimeFromMillisPageMaterializer(ValuesReader dataReader, LocalDateTime nullValue, + int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = ParquetTimeUtils.epochMillisToLocalDateTimeUTC(dataReader.readLong()); + } + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromNanosMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromNanosMaterializer.java new file mode 100644 index 00000000000..ae2947dda66 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeFromNanosMaterializer.java @@ -0,0 +1,53 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit LocalDateTimeFromMicrosMaterializer and run "./gradlew replicatePageMaterializers" to regenerate +// +// @formatter:off +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; +import io.deephaven.parquet.base.PageMaterializerFactory; +import io.deephaven.parquet.base.ParquetTimeUtils; +import org.apache.parquet.column.values.ValuesReader; + +import java.time.LocalDateTime; + +public class LocalDateTimeFromNanosMaterializer { + + public static final PageMaterializerFactory Factory = new PageMaterializerFactory() { + @Override + public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { + return new LocalDateTimeFromNanosPageMaterializer(dataReader, (LocalDateTime) nullValue, numValues); + } + + @Override + public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { + return new LocalDateTimeFromNanosPageMaterializer(dataReader, numValues); + } + }; + + private static final class LocalDateTimeFromNanosPageMaterializer extends LocalDateTimePageMaterializerBase + implements PageMaterializer { + + final ValuesReader dataReader; + + private LocalDateTimeFromNanosPageMaterializer(ValuesReader dataReader, int numValues) { + this(dataReader, null, numValues); + } + + private LocalDateTimeFromNanosPageMaterializer(ValuesReader dataReader, LocalDateTime nullValue, + int numValues) { + super(nullValue, numValues); + this.dataReader = dataReader; + } + + @Override + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = ParquetTimeUtils.epochNanosToLocalDateTimeUTC(dataReader.readLong()); + } + } + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeMaterializer.java deleted file mode 100644 index 176d2cd98c7..00000000000 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimeMaterializer.java +++ /dev/null @@ -1,83 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.parquet.base.materializers; - -import io.deephaven.parquet.base.PageMaterializer; -import io.deephaven.parquet.base.PageMaterializerFactory; -import io.deephaven.parquet.base.ParquetTimeUtils; -import org.apache.parquet.column.values.ValuesReader; - -import java.time.LocalDateTime; -import java.util.Arrays; -import java.util.function.LongFunction; - -public class LocalDateTimeMaterializer { - - public static final PageMaterializerFactory FromMillisFactory = - create(ParquetTimeUtils::epochMillisToLocalDateTimeUTC); - public static final PageMaterializerFactory FromMicrosFactory = - create(ParquetTimeUtils::epochMicrosToLocalDateTimeUTC); - public static final PageMaterializerFactory FromNanosFactory = - create(ParquetTimeUtils::epochNanosToLocalDateTimeUTC); - - private static PageMaterializerFactory create(final LongFunction converter) { - return new PageMaterializerFactory() { - @Override - public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, - int numValues) { - return new LocalDateTimePageMaterializer(dataReader, (LocalDateTime) nullValue, numValues, converter); - } - - @Override - public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { - return new LocalDateTimePageMaterializer(dataReader, numValues, converter); - } - }; - } - - private static final class LocalDateTimePageMaterializer implements PageMaterializer { - - final ValuesReader dataReader; - - final LocalDateTime nullValue; - final LocalDateTime[] data; - final LongFunction converter; - - private LocalDateTimePageMaterializer(ValuesReader dataReader, int numValues, - LongFunction converter) { - this(dataReader, null, numValues, converter); - } - - private LocalDateTimePageMaterializer(ValuesReader dataReader, LocalDateTime nullValue, int numValues, - LongFunction converter) { - this.dataReader = dataReader; - this.nullValue = nullValue; - this.data = new LocalDateTime[numValues]; - this.converter = converter; - } - - @Override - public void fillNulls(int startIndex, int endIndex) { - Arrays.fill(data, startIndex, endIndex, nullValue); - } - - @Override - public void fillValues(int startIndex, int endIndex) { - for (int ii = startIndex; ii < endIndex; ii++) { - data[ii] = converter.apply(dataReader.readLong()); - } - } - - @Override - public Object fillAll() { - fillValues(0, data.length); - return data; - } - - @Override - public Object data() { - return data; - } - } -} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimePageMaterializerBase.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimePageMaterializerBase.java new file mode 100644 index 00000000000..689240a3287 --- /dev/null +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalDateTimePageMaterializerBase.java @@ -0,0 +1,36 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.parquet.base.materializers; + +import io.deephaven.parquet.base.PageMaterializer; + +import java.time.LocalDateTime; +import java.util.Arrays; + +abstract class LocalDateTimePageMaterializerBase implements PageMaterializer { + + final LocalDateTime nullValue; + final LocalDateTime[] data; + + LocalDateTimePageMaterializerBase(LocalDateTime nullValue, int numValues) { + this.nullValue = nullValue; + this.data = new LocalDateTime[numValues]; + } + + @Override + public void fillNulls(int startIndex, int endIndex) { + Arrays.fill(data, startIndex, endIndex, nullValue); + } + + @Override + public Object fillAll() { + fillValues(0, data.length); + return data; + } + + @Override + public Object data() { + return data; + } +} diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMicrosMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMicrosMaterializer.java index 00ada8f73cc..1dc143dd350 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMicrosMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMicrosMaterializer.java @@ -39,8 +39,10 @@ private LocalTimeFromMicrosPageMaterializer(ValuesReader dataReader, LocalTime n } @Override - LocalTime readNext() { - return DateTimeUtils.microsOfDayToLocalTime(dataReader.readLong()); + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = DateTimeUtils.microsOfDayToLocalTime(dataReader.readLong()); + } } } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMillisMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMillisMaterializer.java index 1be0fe187f6..a6fc7a5fbda 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMillisMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMillisMaterializer.java @@ -43,8 +43,10 @@ private LocalTimeFromMillisPageMaterializer(ValuesReader dataReader, LocalTime n } @Override - LocalTime readNext() { - return DateTimeUtils.millisOfDayToLocalTime(dataReader.readInteger()); + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = DateTimeUtils.millisOfDayToLocalTime(dataReader.readInteger()); + } } } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromNanosMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromNanosMaterializer.java index e707afc6d55..b0ebdb3d088 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromNanosMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromNanosMaterializer.java @@ -43,8 +43,10 @@ private LocalTimeFromNanosPageMaterializer(ValuesReader dataReader, LocalTime nu } @Override - LocalTime readNext() { - return DateTimeUtils.nanosOfDayToLocalTime(dataReader.readLong()); + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = DateTimeUtils.nanosOfDayToLocalTime(dataReader.readLong()); + } } } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimePageMaterializerBase.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimePageMaterializerBase.java index abc731319d8..7c248040bc8 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimePageMaterializerBase.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimePageMaterializerBase.java @@ -13,11 +13,6 @@ abstract class LocalTimePageMaterializerBase implements PageMaterializer { final LocalTime nullValue; final LocalTime[] data; - /** - * @return a {@link LocalTime} value read from the dataReader. - */ - abstract LocalTime readNext(); - LocalTimePageMaterializerBase(LocalTime nullValue, int numValues) { this.nullValue = nullValue; this.data = new LocalTime[numValues]; @@ -28,13 +23,6 @@ public final void fillNulls(int startIndex, int endIndex) { Arrays.fill(data, startIndex, endIndex, nullValue); } - @Override - public final void fillValues(int startIndex, int endIndex) { - for (int ii = startIndex; ii < endIndex; ii++) { - data[ii] = readNext(); - } - } - @Override public final Object fillAll() { fillValues(0, data.length); diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedIntMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedIntMaterializer.java index ec698091ed1..b6d68ace134 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedIntMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongFromUnsignedIntMaterializer.java @@ -36,8 +36,10 @@ private LongFromUnsignedIntPageMaterializer(ValuesReader dataReader, long nullVa } @Override - long readLong() { - return Integer.toUnsignedLong(dataReader.readInteger()); + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = Integer.toUnsignedLong(dataReader.readInteger()); + } } } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongMaterializer.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongMaterializer.java index 28d360ce7ef..b94a798b218 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongMaterializer.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongMaterializer.java @@ -35,8 +35,10 @@ private LongPageMaterializer(ValuesReader dataReader, long nullValue, int numVal } @Override - long readLong() { - return dataReader.readLong(); + public void fillValues(int startIndex, int endIndex) { + for (int ii = startIndex; ii < endIndex; ii++) { + data[ii] = dataReader.readLong(); + } } } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongPageMaterializerBase.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongPageMaterializerBase.java index b2aae12f67c..3d5e3f67d38 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongPageMaterializerBase.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LongPageMaterializerBase.java @@ -12,11 +12,6 @@ abstract class LongPageMaterializerBase implements PageMaterializer { final long nullValue; final long[] data; - /** - * @return a long value read from the dataReader. - */ - abstract long readLong(); - LongPageMaterializerBase(long nullValue, int numValues) { this.nullValue = nullValue; this.data = new long[numValues]; @@ -27,13 +22,6 @@ public void fillNulls(int startIndex, int endIndex) { Arrays.fill(data, startIndex, endIndex, nullValue); } - @Override - public void fillValues(int startIndex, int endIndex) { - for (int ii = startIndex; ii < endIndex; ii++) { - data[ii] = readLong(); - } - } - @Override public Object fillAll() { fillValues(0, data.length); diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicatePageMaterializers.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicatePageMaterializers.java index 8e7c4e0b034..e81a88b2d44 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicatePageMaterializers.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicatePageMaterializers.java @@ -16,15 +16,38 @@ public class ReplicatePageMaterializers { private static final String TASK = "replicatePageMaterializers"; private static final String[] NO_EXCEPTIONS = new String[0]; + private static final String MATERIALIZER_DIR = + "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/"; + + private static final String CHAR_MATERIALIZER_PATH = MATERIALIZER_DIR + "CharMaterializer.java"; + private static final String FLOAT_MATERIALIZER_PATH = MATERIALIZER_DIR + "FloatMaterializer.java"; + private static final String INT_MATERIALIZER_PATH = MATERIALIZER_DIR + "IntMaterializer.java"; + private static final String STRING_MATERIALIZER_PATH = MATERIALIZER_DIR + "StringMaterializer.java"; + + private static final String LOCAL_TIME_FROM_MILLIS_MATERIALIZER_PATH = + MATERIALIZER_DIR + "LocalTimeFromMillisMaterializer.java"; + private static final String LOCAL_TIME_FROM_MICROS_MATERIALIZER_PATH = + MATERIALIZER_DIR + "LocalTimeFromMicrosMaterializer.java"; + private static final String LOCAL_TIME_FROM_NANOS_MATERIALIZER_PATH = + MATERIALIZER_DIR + "LocalTimeFromNanosMaterializer.java"; + + private static final String LOCAL_DATE_TIME_FROM_MILLIS_MATERIALIZER_PATH = + MATERIALIZER_DIR + "LocalDateTimeFromMillisMaterializer.java"; + private static final String LOCAL_DATE_TIME_FROM_MICROS_MATERIALIZER_PATH = + MATERIALIZER_DIR + "LocalDateTimeFromMicrosMaterializer.java"; + private static final String LOCAL_DATE_TIME_FROM_NANOS_MATERIALIZER_PATH = + MATERIALIZER_DIR + "LocalDateTimeFromNanosMaterializer.java"; + + private static final String INSTANT_NANOS_FROM_MICROS_MATERIALIZER_PATH = + MATERIALIZER_DIR + "InstantNanosFromMicrosMaterializer.java"; + private static final String INSTANT_NANOS_FROM_MILLIS_MATERIALIZER_PATH = + MATERIALIZER_DIR + "InstantNanosFromMillisMaterializer.java"; + public static void main(String... args) throws IOException { - charToShortAndByte(TASK, - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/CharMaterializer.java", - NO_EXCEPTIONS); + charToShortAndByte(TASK, CHAR_MATERIALIZER_PATH, NO_EXCEPTIONS); // Float -> Double - floatToAllFloatingPoints(TASK, - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/FloatMaterializer.java", - NO_EXCEPTIONS); + floatToAllFloatingPoints(TASK, FLOAT_MATERIALIZER_PATH, NO_EXCEPTIONS); // Float -> Int String[][] pairs = new String[][] { @@ -32,10 +55,7 @@ public static void main(String... args) throws IOException { {"Float", "Int"}, {"float", "int"} }; - replaceAll(TASK, - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/FloatMaterializer.java", - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/IntMaterializer.java", - null, NO_EXCEPTIONS, pairs); + replaceAll(TASK, FLOAT_MATERIALIZER_PATH, INT_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); // Float -> String pairs = new String[][] { @@ -44,10 +64,7 @@ public static void main(String... args) throws IOException { {"float", "String"}, {"dataReader, 0, numValues", "dataReader, null, numValues"} }; - replaceAll(TASK, - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/FloatMaterializer.java", - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/StringMaterializer.java", - null, NO_EXCEPTIONS, pairs); + replaceAll(TASK, FLOAT_MATERIALIZER_PATH, STRING_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); // LocalTimeFromMicros -> LocalTimeFromNanos pairs = new String[][] { @@ -55,8 +72,8 @@ public static void main(String... args) throws IOException { {"micros", "nanos"}, }; replaceAll(TASK, - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMicrosMaterializer.java", - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromNanosMaterializer.java", + LOCAL_TIME_FROM_MICROS_MATERIALIZER_PATH, + LOCAL_TIME_FROM_NANOS_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); // LocalTimeFromMicros -> LocalTimeFromMillis @@ -66,18 +83,36 @@ public static void main(String... args) throws IOException { {"readLong", "readInteger"}, }; replaceAll(TASK, - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMicrosMaterializer.java", - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/LocalTimeFromMillisMaterializer.java", + LOCAL_TIME_FROM_MICROS_MATERIALIZER_PATH, + LOCAL_TIME_FROM_MILLIS_MATERIALIZER_PATH, + null, NO_EXCEPTIONS, pairs); + + // LocalDateTimeFromMicros -> LocalDateTimeFromNanos + pairs = new String[][] { + {"Micros", "Nanos"} + }; + replaceAll(TASK, + LOCAL_DATE_TIME_FROM_MICROS_MATERIALIZER_PATH, + LOCAL_DATE_TIME_FROM_NANOS_MATERIALIZER_PATH, + null, NO_EXCEPTIONS, pairs); + + // LocalDateTimeFromMicros -> LocalDateTimeFromMillis + pairs = new String[][] { + {"Micros", "Millis"} + }; + replaceAll(TASK, + LOCAL_DATE_TIME_FROM_MICROS_MATERIALIZER_PATH, + LOCAL_DATE_TIME_FROM_MILLIS_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); - // TimestampNanosFromMicros -> TimestampNanosFromMillis + // InstantNanosFromMicros -> InstantNanosFromMillis pairs = new String[][] { {"Micros", "Millis"}, {"micros", "millis"} }; replaceAll(TASK, - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/TimestampNanosFromMicrosMaterializer.java", - "extensions/parquet/base/src/main/java/io/deephaven/parquet/base/materializers/TimestampNanosFromMillisMaterializer.java", + INSTANT_NANOS_FROM_MICROS_MATERIALIZER_PATH, + INSTANT_NANOS_FROM_MILLIS_MATERIALIZER_PATH, null, NO_EXCEPTIONS, pairs); } }