Tablet.serialize() may throw an exception due to null values in the D…

…ate column (#329) * Tablet.serialize() may throw an exception due to null values in the Date column * fix DateDataPoint * modify AlignedChunkGroupWriterImpl
apache · Dec 27, 2024 · 854f7b7 · 854f7b7
1 parent 789c65b
commit 854f7b7
Show file tree

Hide file tree

Showing 6 changed files with 54 additions and 3 deletions.
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/DateUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/DateUtils.java
@@ -28,6 +28,7 @@
 
 public class DateUtils {
   private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+  public static final int EMPTY_DATE_INT = 10000101;
 
   public static String formatDate(int date) {
     return date / 10000

diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
@@ -171,7 +171,7 @@ public int write(long time, List<DataPoint> data) throws WriteProcessException,
           break;
         case INT32:
         case DATE:
-          valueChunkWriter.write(time, (int) point.getValue(), isNull);
+          valueChunkWriter.write(time, isNull ? 0 : (int) point.getValue(), isNull);
           break;
         case INT64:
         case TIMESTAMP:

diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
@@ -705,7 +705,11 @@ private void serializeColumn(
         case DATE:
           LocalDate[] dateValues = (LocalDate[]) column;
           for (int j = 0; j < rowSize; j++) {
-            ReadWriteIOUtils.write(DateUtils.parseDateExpressionToInt(dateValues[j]), stream);
+            ReadWriteIOUtils.write(
+                dateValues[j] == null
+                    ? DateUtils.EMPTY_DATE_INT
+                    : DateUtils.parseDateExpressionToInt(dateValues[j]),
+                stream);
           }
           break;
         case INT64:

diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DateDataPoint.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DateDataPoint.java
@@ -52,7 +52,7 @@ public void writeTo(long time, ChunkWriterImpl writer) throws IOException {
 
   @Override
   public Object getValue() {
-    return DateUtils.parseDateExpressionToInt(value);
+    return value == null ? null : DateUtils.parseDateExpressionToInt(value);
   }
 
   @Override

diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
@@ -43,6 +43,8 @@
 import org.apache.tsfile.write.chunk.ChunkWriterImpl;
 import org.apache.tsfile.write.record.TSRecord;
 import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.record.datapoint.DateDataPoint;
+import org.apache.tsfile.write.record.datapoint.StringDataPoint;
 import org.apache.tsfile.write.schema.IMeasurementSchema;
 import org.apache.tsfile.write.schema.MeasurementSchema;
 import org.apache.tsfile.write.writer.TsFileIOWriter;
@@ -549,6 +551,31 @@ public void writeAlignedWithTabletWithNullValue() {
     }
   }
 
+  @Test
+  public void writeRecordWithNullValue() {
+    setEnv(100, 30);
+    measurementSchemas.add(new MeasurementSchema("s1", TSDataType.TEXT, TSEncoding.PLAIN));
+    measurementSchemas.add(new MeasurementSchema("s2", TSDataType.STRING, TSEncoding.PLAIN));
+    measurementSchemas.add(new MeasurementSchema("s3", TSDataType.BLOB, TSEncoding.PLAIN));
+    measurementSchemas.add(new MeasurementSchema("s4", TSDataType.DATE, TSEncoding.PLAIN));
+    try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
+
+      // register aligned timeseries
+      tsFileWriter.registerAlignedTimeseries(new Path(deviceId), measurementSchemas);
+
+      TSRecord record = new TSRecord(deviceId, 0);
+      record.addTuple(new StringDataPoint("s1", null));
+      record.addTuple(new StringDataPoint("s2", null));
+      record.addTuple(new StringDataPoint("s3", null));
+      record.addTuple(new DateDataPoint("s4", null));
+
+      tsFileWriter.writeRecord(record);
+    } catch (Throwable e) {
+      e.printStackTrace();
+      Assert.fail("Meet errors in test: " + e.getMessage());
+    }
+  }
+
   @Test
   public void writeDataToTabletsWithNegativeTimestamps() {
     setEnv(100, 30);

diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
@@ -30,6 +30,7 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.time.LocalDate;
 import java.util.ArrayList;
@@ -278,4 +279,22 @@ private void addValueWithException(Tablet tablet, String column, int rowIndex, O
     }
     Assert.fail();
   }
+
+  @Test
+  public void testSerializeDateColumnWithNullValue() throws IOException {
+    final List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+    measurementSchemas.add(new MeasurementSchema("s1", TSDataType.DATE, TSEncoding.PLAIN));
+    measurementSchemas.add(new MeasurementSchema("s2", TSDataType.DATE, TSEncoding.PLAIN));
+    Tablet tablet = new Tablet("root.testsg.d1", measurementSchemas);
+    tablet.addTimestamp(0, 0);
+    tablet.addValue(0, 0, LocalDate.now());
+    tablet.addTimestamp(1, 1);
+    tablet.addValue(1, 1, LocalDate.now());
+    ByteBuffer serialized = tablet.serialize();
+    Tablet deserializeTablet = Tablet.deserialize(serialized);
+    Assert.assertEquals(tablet.getValue(0, 0), deserializeTablet.getValue(0, 0));
+    Assert.assertTrue(deserializeTablet.isNull(0, 1));
+    Assert.assertEquals(tablet.getValue(1, 1), deserializeTablet.getValue(1, 1));
+    Assert.assertTrue(deserializeTablet.isNull(1, 0));
+  }
 }