Skip to content

Commit

Permalink
Add statistic compatibility (#382)
Browse files Browse the repository at this point in the history 
* add statistic compatibility

(cherry picked from commit 03c229d)
(cherry picked from commit c97ec47)

* fix test

(cherry picked from commit 92a06d1)

* add column compatibility

(cherry picked from commit 8dd5f1c)

* Add exception
  • Loading branch information
@jt2594838
jt2594838 authored Jan 17, 2025
1 parent 871cb09 commit 268b323
Show file tree
Hide file tree
Showing 15 changed files with 419 additions and 145 deletions.
26 changes: 15 additions & 11 deletions java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BinaryStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,18 +125,22 @@ public long getSumLongValue() {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.TEXT, "long sum"));
}

@Override
protected void mergeStatisticsValue(Statistics<Binary> stats) {
BinaryStatistics stringStats = (BinaryStatistics) stats;
if (isEmpty) {
initializeStats(stringStats.getFirstValue(), stringStats.getLastValue());
isEmpty = false;
@SuppressWarnings("rawtypes")
@Override
protected void mergeStatisticsValue(Statistics stats) {
if (stats instanceof BinaryStatistics || stats instanceof StringStatistics) {
if (isEmpty) {
initializeStats(((Binary) stats.getFirstValue()), ((Binary) stats.getLastValue()));
isEmpty = false;
} else {
updateStats(
((Binary) stats.getFirstValue()),
((Binary) stats.getLastValue()),
stats.getStartTime(),
stats.getEndTime());
}
} else {
updateStats(
stringStats.getFirstValue(),
stringStats.getLastValue(),
stats.getStartTime(),
stats.getEndTime());
throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}

Expand Down
3 changes: 2 additions & 1 deletion java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BlobStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,9 @@ public long getSumLongValue() {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.BLOB, "sum"));
}

@SuppressWarnings("rawtypes")
@Override
protected void mergeStatisticsValue(Statistics<Binary> stats) {
protected void mergeStatisticsValue(Statistics stats) {
// do nothing
if (isEmpty) {
isEmpty = false;
Expand Down
29 changes: 16 additions & 13 deletions java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BooleanStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,7 @@ public Boolean getLastValue() {

@Override
public double getSumDoubleValue() {
throw new StatisticsClassException(
String.format(STATS_UNSUPPORTED_MSG, TSDataType.BOOLEAN, "double sum"));
return sumValue;
}

@Override
Expand All @@ -137,18 +136,22 @@ public long getSumLongValue() {
}

@Override
protected void mergeStatisticsValue(Statistics<Boolean> stats) {
BooleanStatistics boolStats = (BooleanStatistics) stats;
if (isEmpty) {
initializeStats(boolStats.getFirstValue(), boolStats.getLastValue(), boolStats.sumValue);
isEmpty = false;
protected void mergeStatisticsValue(Statistics stats) {
if (stats instanceof BooleanStatistics) {
BooleanStatistics boolStats = (BooleanStatistics) stats;
if (isEmpty) {
initializeStats(boolStats.getFirstValue(), boolStats.getLastValue(), boolStats.sumValue);
isEmpty = false;
} else {
updateStats(
boolStats.getFirstValue(),
boolStats.getLastValue(),
stats.getStartTime(),
stats.getEndTime(),
boolStats.sumValue);
}
} else {
updateStats(
boolStats.getFirstValue(),
boolStats.getLastValue(),
stats.getStartTime(),
stats.getEndTime(),
boolStats.sumValue);
throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}

Expand Down
43 changes: 25 additions & 18 deletions java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/DoubleStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,26 +159,33 @@ public long getSumLongValue() {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.DOUBLE, "long sum"));
}

@SuppressWarnings("rawtypes")
@Override
protected void mergeStatisticsValue(Statistics<Double> stats) {
DoubleStatistics doubleStats = (DoubleStatistics) stats;
if (this.isEmpty) {
initializeStats(
doubleStats.getMinValue(),
doubleStats.getMaxValue(),
doubleStats.getFirstValue(),
doubleStats.getLastValue(),
doubleStats.sumValue);
isEmpty = false;
protected void mergeStatisticsValue(Statistics stats) {
if (stats instanceof DoubleStatistics
|| stats instanceof FloatStatistics
|| stats instanceof IntegerStatistics
|| stats instanceof LongStatistics) {
if (this.isEmpty) {
initializeStats(
((Number) stats.getMinValue()).doubleValue(),
((Number) stats.getMaxValue()).doubleValue(),
((Number) stats.getFirstValue()).doubleValue(),
((Number) stats.getLastValue()).doubleValue(),
stats.getSumDoubleValue());
isEmpty = false;
} else {
updateStats(
((Number) stats.getMinValue()).doubleValue(),
((Number) stats.getMaxValue()).doubleValue(),
((Number) stats.getFirstValue()).doubleValue(),
((Number) stats.getLastValue()).doubleValue(),
stats.getSumDoubleValue(),
stats.getStartTime(),
stats.getEndTime());
}
} else {
updateStats(
doubleStats.getMinValue(),
doubleStats.getMaxValue(),
doubleStats.getFirstValue(),
doubleStats.getLastValue(),
doubleStats.sumValue,
stats.getStartTime(),
stats.getEndTime());
throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}

Expand Down
42 changes: 23 additions & 19 deletions java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/FloatStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,26 +150,30 @@ public long getSumLongValue() {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.FLOAT, "long sum"));
}

@Override
protected void mergeStatisticsValue(Statistics<Float> stats) {
FloatStatistics floatStats = (FloatStatistics) stats;
if (isEmpty) {
initializeStats(
floatStats.getMinValue(),
floatStats.getMaxValue(),
floatStats.getFirstValue(),
floatStats.getLastValue(),
floatStats.sumValue);
isEmpty = false;
@SuppressWarnings("rawtypes")
@Override
protected void mergeStatisticsValue(Statistics stats) {
if (stats instanceof FloatStatistics || stats instanceof IntegerStatistics) {
if (isEmpty) {
initializeStats(
((Number) stats.getMinValue()).floatValue(),
((Number) stats.getMaxValue()).floatValue(),
((Number) stats.getFirstValue()).floatValue(),
((Number) stats.getLastValue()).floatValue(),
stats.getSumDoubleValue());
isEmpty = false;
} else {
updateStats(
((Number) stats.getMinValue()).floatValue(),
((Number) stats.getMaxValue()).floatValue(),
((Number) stats.getFirstValue()).floatValue(),
((Number) stats.getLastValue()).floatValue(),
stats.getSumDoubleValue(),
stats.getStartTime(),
stats.getEndTime());
}
} else {
updateStats(
floatStats.getMinValue(),
floatStats.getMaxValue(),
floatStats.getFirstValue(),
floatStats.getLastValue(),
floatStats.sumValue,
stats.getStartTime(),
stats.getEndTime());
throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}

Expand Down
46 changes: 25 additions & 21 deletions java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/IntegerStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,35 +142,39 @@ public Integer getLastValue() {

@Override
public double getSumDoubleValue() {
throw new StatisticsClassException(
String.format(STATS_UNSUPPORTED_MSG, TSDataType.INT32, "double sum"));
return sumValue;
}

@Override
public long getSumLongValue() {
return sumValue;
}

@Override
protected void mergeStatisticsValue(Statistics<Integer> stats) {
IntegerStatistics intStats = (IntegerStatistics) stats;
if (isEmpty) {
initializeStats(
intStats.getMinValue(),
intStats.getMaxValue(),
intStats.getFirstValue(),
intStats.getLastValue(),
intStats.sumValue);
isEmpty = false;
@SuppressWarnings("rawtypes")
@Override
protected void mergeStatisticsValue(Statistics stats) {
if (stats instanceof IntegerStatistics) {
IntegerStatistics intStats = (IntegerStatistics) stats;
if (isEmpty) {
initializeStats(
intStats.getMinValue(),
intStats.getMaxValue(),
intStats.getFirstValue(),
intStats.getLastValue(),
intStats.sumValue);
isEmpty = false;
} else {
updateStats(
intStats.getMinValue(),
intStats.getMaxValue(),
intStats.getFirstValue(),
intStats.getLastValue(),
intStats.sumValue,
stats.getStartTime(),
stats.getEndTime());
}
} else {
updateStats(
intStats.getMinValue(),
intStats.getMaxValue(),
intStats.getFirstValue(),
intStats.getLastValue(),
intStats.sumValue,
stats.getStartTime(),
stats.getEndTime());
throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}

Expand Down
40 changes: 22 additions & 18 deletions java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/LongStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,26 +160,30 @@ private void updateStats(
}
}

@SuppressWarnings("rawtypes")
@Override
protected void mergeStatisticsValue(Statistics<Long> stats) {
LongStatistics longStats = (LongStatistics) stats;
if (isEmpty) {
initializeStats(
longStats.getMinValue(),
longStats.getMaxValue(),
longStats.getFirstValue(),
longStats.getLastValue(),
longStats.sumValue);
isEmpty = false;
protected void mergeStatisticsValue(Statistics stats) {
if (stats instanceof LongStatistics || stats instanceof IntegerStatistics) {
if (isEmpty) {
initializeStats(
((Number) stats.getMinValue()).longValue(),
((Number) stats.getMaxValue()).longValue(),
((Number) stats.getFirstValue()).longValue(),
((Number) stats.getLastValue()).longValue(),
stats.getSumDoubleValue());
isEmpty = false;
} else {
updateStats(
((Number) stats.getMinValue()).longValue(),
((Number) stats.getMaxValue()).longValue(),
((Number) stats.getFirstValue()).longValue(),
((Number) stats.getLastValue()).longValue(),
stats.getSumDoubleValue(),
stats.getStartTime(),
stats.getEndTime());
}
} else {
updateStats(
longStats.getMinValue(),
longStats.getMaxValue(),
longStats.getFirstValue(),
longStats.getLastValue(),
longStats.sumValue,
stats.getStartTime(),
stats.getEndTime());
throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}

Expand Down
16 changes: 12 additions & 4 deletions java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ public static Statistics<? extends Serializable> deserialize(
*/
@SuppressWarnings("unchecked")
public void mergeStatistics(Statistics<? extends Serializable> stats) {
if (this.getClass() == stats.getClass()) {
if (this.getClass() == stats.getClass() || canMerge(stats.getType(), this.getType())) {
if (!stats.isEmpty) {
if (stats.startTime < this.startTime) {
this.startTime = stats.startTime;
Expand All @@ -213,7 +213,7 @@ public void mergeStatistics(Statistics<? extends Serializable> stats) {
}
// must be sure no overlap between two statistics
this.count += stats.count;
mergeStatisticsValue((Statistics<T>) stats);
mergeStatisticsValue(stats);
isEmpty = false;
}
} else {
Expand All @@ -225,6 +225,13 @@ public void mergeStatistics(Statistics<? extends Serializable> stats) {
}
}

public static boolean canMerge(TSDataType from, TSDataType to) {
return to.isCompatible(from)
&&
// cannot alter from TEXT to STRING because we cannot add statistic to the existing chunks
!(from == TSDataType.TEXT && to == TSDataType.STRING);
}

public void update(long time, boolean value) {
update(time);
updateStats(value);
Expand Down Expand Up @@ -315,7 +322,8 @@ public void update(long[] time, int batchSize, int arrayOffset) {
count += batchSize;
}

protected abstract void mergeStatisticsValue(Statistics<T> stats);
@SuppressWarnings("rawtypes")
protected abstract void mergeStatisticsValue(Statistics stats);

public boolean isEmpty() {
return isEmpty;
Expand Down Expand Up @@ -393,7 +401,7 @@ public long getEndTime() {
return endTime;
}

public long getCount() {
public int getCount() {
return count;
}

Expand Down
38 changes: 21 additions & 17 deletions java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/StringStatistics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,24 +143,28 @@ public long getSumLongValue() {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.STRING, "long sum"));
}

@Override
protected void mergeStatisticsValue(Statistics<Binary> stats) {
StringStatistics stringStats = (StringStatistics) stats;
if (isEmpty) {
initializeStats(
stringStats.getFirstValue(),
stringStats.getLastValue(),
stringStats.getMinValue(),
stringStats.getMaxValue());
isEmpty = false;
@SuppressWarnings("rawtypes")
@Override
protected void mergeStatisticsValue(Statistics stats) {
if (stats instanceof StringStatistics) {
if (isEmpty) {
initializeStats(
((Binary) stats.getFirstValue()),
((Binary) stats.getLastValue()),
((Binary) stats.getMinValue()),
((Binary) stats.getMaxValue()));
isEmpty = false;
} else {
updateStats(
((Binary) stats.getFirstValue()),
((Binary) stats.getLastValue()),
((Binary) stats.getMinValue()),
((Binary) stats.getMaxValue()),
stats.getStartTime(),
stats.getEndTime());
}
} else {
updateStats(
stringStats.getFirstValue(),
stringStats.getLastValue(),
stringStats.getMinValue(),
stringStats.getMaxValue(),
stats.getStartTime(),
stats.getEndTime());
throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}

Expand Down
Loading

0 comments on commit 268b323

Please sign in to comment.