From c087c24435ca25e6a2708216c7ba29c23cea6459 Mon Sep 17 00:00:00 2001 From: Jiang Tian Date: Thu, 9 Jan 2025 16:24:25 +0800 Subject: [PATCH 01/13] Do not add the source path of a view when it can be matched by the deletion pattern (#14658) --- .../plan/analyze/AnalyzeVisitor.java | 12 +++++++++ .../apache/iotdb/commons/utils/FileUtils.java | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index aa6653da70f8..156ceb5a73b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -3521,6 +3521,18 @@ public Analysis visitDeleteData( logicalViewSchema = (LogicalViewSchema) measurementSchema; if (logicalViewSchema.isWritable()) { sourcePathOfAliasSeries = logicalViewSchema.getSourcePathIfWritable(); + // if the source path can be matched by any of the deletion pattern, do not add it + boolean pathMatched = false; + for (MeasurementPath deletionPattern : deleteDataStatement.getPathList()) { + if (deletionPattern.matchFullPath(sourcePathOfAliasSeries)) { + pathMatched = true; + break; + } + } + if (pathMatched) { + continue; + } + deletePatternSet.add(new MeasurementPath(sourcePathOfAliasSeries.getNodes())); deduplicatedDeviceIDs.add(sourcePathOfAliasSeries.getIDeviceID()); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java index f1ae48c99565..fbc0621caaae 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java @@ -29,6 +29,7 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileFilter; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -41,7 +42,10 @@ import java.nio.file.StandardCopyOption; import java.text.CharacterIterator; import java.text.StringCharacterIterator; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.Stack; public class FileUtils { private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); @@ -50,6 +54,29 @@ public class FileUtils { private FileUtils() {} + public static List listFilesRecursively(File dir, FileFilter fileFilter) { + List result = new ArrayList<>(); + Stack stack = new Stack<>(); + if (dir.exists()) { + stack.push(dir); + } + while (!stack.isEmpty()) { + File file = stack.pop(); + if (file.isDirectory()) { + File[] files = file.listFiles(); + if (files != null) { + for (File f : files) { + stack.push(f); + } + } + } + if (fileFilter.accept(file)) { + result.add(file); + } + } + return result; + } + public static boolean deleteFileIfExist(File file) { try { Files.deleteIfExists(file.toPath()); From c60520aa41a04ad9ba288995cc2499579e1a9e51 Mon Sep 17 00:00:00 2001 From: Itami Sho <42286868+MiniSho@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:03:09 +0800 Subject: [PATCH 02/13] Load: Add a new configuration to enable datatype conversion when converting into Tablets & Refactor LoadAnalyzeException. (#14646) --- .../LoadAnalyzeException.java} | 8 ++--- ...adAnalyzeTableColumnDisorderException.java | 30 +++++++++++++++++++ .../LoadAnalyzeTypeMismatchException.java} | 6 ++-- .../PipeConvertedInsertTabletStatement.java | 4 +++ .../plan/analyze/load/LoadTsFileAnalyzer.java | 15 +++++----- .../load/LoadTsFileTableSchemaCache.java | 14 ++++----- .../load/LoadTsFileToTableModelAnalyzer.java | 13 ++++---- .../load/LoadTsFileToTreeModelAnalyzer.java | 9 +++--- .../TreeSchemaAutoCreatorAndVerifier.java | 30 +++++++++---------- .../plan/relational/metadata/Metadata.java | 4 ++- .../metadata/TableMetadataImpl.java | 4 ++- .../fetcher/TableHeaderSchemaValidator.java | 8 +++-- .../LoadConvertedInsertTabletStatement.java | 18 ++++++++++- ...tementDataTypeConvertExecutionVisitor.java | 3 +- ...tementDataTypeConvertExecutionVisitor.java | 3 +- 15 files changed, 113 insertions(+), 56 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/{VerifyMetadataException.java => load/LoadAnalyzeException.java} (82%) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTableColumnDisorderException.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/{VerifyMetadataTypeMismatchException.java => load/LoadAnalyzeTypeMismatchException.java} (83%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeException.java similarity index 82% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeException.java index 78c6d8d4d5cc..a92fdbfce4f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeException.java @@ -17,18 +17,18 @@ * under the License. */ -package org.apache.iotdb.db.exception; +package org.apache.iotdb.db.exception.load; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.rpc.TSStatusCode; -public class VerifyMetadataException extends IoTDBException { +public class LoadAnalyzeException extends IoTDBException { - public VerifyMetadataException(String message) { + public LoadAnalyzeException(String message) { super(message, TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode()); } - public VerifyMetadataException(String message, int errorCode) { + public LoadAnalyzeException(String message, int errorCode) { super(message, errorCode); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTableColumnDisorderException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTableColumnDisorderException.java new file mode 100644 index 000000000000..a34ecd12c167 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTableColumnDisorderException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.exception.load; + +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.rpc.TSStatusCode; + +public class LoadAnalyzeTableColumnDisorderException extends SemanticException { + + public LoadAnalyzeTableColumnDisorderException(String message) { + super(message, TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTypeMismatchException.java similarity index 83% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTypeMismatchException.java index 209fb83bcdbe..b057069546b4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTypeMismatchException.java @@ -17,13 +17,13 @@ * under the License. */ -package org.apache.iotdb.db.exception; +package org.apache.iotdb.db.exception.load; import org.apache.iotdb.rpc.TSStatusCode; -public class VerifyMetadataTypeMismatchException extends VerifyMetadataException { +public class LoadAnalyzeTypeMismatchException extends LoadAnalyzeException { - public VerifyMetadataTypeMismatchException(String message) { + public LoadAnalyzeTypeMismatchException(String message) { super(message, TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java index 606309366cc3..adc19ae7e0fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java @@ -71,6 +71,10 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { return true; } + protected boolean originalCheckAndCastDataType(int columnIndex, TSDataType dataType) { + return super.checkAndCastDataType(columnIndex, dataType); + } + @TableModel @Override public boolean isForceTypeConversion() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 87d32c0f88cf..83d685c007cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -22,8 +22,8 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.db.exception.VerifyMetadataException; -import org.apache.iotdb.db.exception.VerifyMetadataTypeMismatchException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException; import org.apache.iotdb.db.exception.load.LoadReadOnlyException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -149,8 +149,8 @@ protected boolean doAnalyzeFileByFile(IAnalysis analysis) { } catch (AuthException e) { setFailAnalysisForAuthException(analysis, e); return false; - } catch (VerifyMetadataTypeMismatchException e) { - executeDataTypeConversionOnTypeMismatch(analysis, e); + } catch (LoadAnalyzeException e) { + executeTabletConversion(analysis, e); // just return false to STOP the analysis process, // the real result on the conversion will be set in the analysis. return false; @@ -176,15 +176,14 @@ protected boolean doAnalyzeFileByFile(IAnalysis analysis) { } protected abstract void analyzeSingleTsFile(final File tsFile) - throws IOException, AuthException, VerifyMetadataException; + throws IOException, AuthException, LoadAnalyzeException; - protected void executeDataTypeConversionOnTypeMismatch( - final IAnalysis analysis, final VerifyMetadataTypeMismatchException e) { + protected void executeTabletConversion(final IAnalysis analysis, final LoadAnalyzeException e) { final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter = new LoadTsFileDataTypeConverter(isGeneratedByPipe); final TSStatus status = - isConvertOnTypeMismatch + (!(e instanceof LoadAnalyzeTypeMismatchException) || isConvertOnTypeMismatch) ? (isTableModelStatement ? loadTsFileDataTypeConverter .convertForTableModel(loadTsFileTableStatement) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index a3edfabadb8a..776bc63d823c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -23,8 +23,8 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.VerifyMetadataException; -import org.apache.iotdb.db.exception.VerifyMetadataTypeMismatchException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException; import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -254,11 +254,11 @@ private static Object[] truncateNullSuffixesOfDeviceIdSegments(Object[] segments } public void createTable(TableSchema fileSchema, MPPQueryContext context, Metadata metadata) - throws VerifyMetadataException { + throws LoadAnalyzeException { final TableSchema realSchema = metadata.validateTableHeaderSchema(database, fileSchema, context, true, true).orElse(null); if (Objects.isNull(realSchema)) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Failed to validate schema for table {%s, %s}", fileSchema.getTableName(), fileSchema)); @@ -267,7 +267,7 @@ public void createTable(TableSchema fileSchema, MPPQueryContext context, Metadat } private void verifyTableDataTypeAndGenerateIdColumnMapper( - TableSchema fileSchema, TableSchema realSchema) throws VerifyMetadataException { + TableSchema fileSchema, TableSchema realSchema) throws LoadAnalyzeException { final int realIdColumnCount = realSchema.getIdColumns().size(); final Map idColumnMapping = tableIdColumnMapper @@ -282,7 +282,7 @@ private void verifyTableDataTypeAndGenerateIdColumnMapper( if (realIndex != -1) { idColumnMapping.put(idColumnIndex++, realIndex); } else { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Id column %s in TsFile is not found in IoTDB table %s", fileColumn.getName(), realSchema.getTableName())); @@ -291,7 +291,7 @@ private void verifyTableDataTypeAndGenerateIdColumnMapper( final ColumnSchema realColumn = realSchema.getColumn(fileColumn.getName(), fileColumn.getColumnCategory()); if (!fileColumn.getType().equals(realColumn.getType())) { - throw new VerifyMetadataTypeMismatchException( + throw new LoadAnalyzeTypeMismatchException( String.format( "Data type mismatch for column %s in table %s, type in TsFile: %s, type in IoTDB: %s", realColumn.getName(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java index d322f4ea8bb4..fd75d2b06232 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.analyze.load; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; -import org.apache.iotdb.db.exception.VerifyMetadataException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; import org.apache.iotdb.db.exception.load.LoadEmptyFileException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -98,7 +98,7 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { try { autoCreateDatabaseIfAbsent(database); - } catch (VerifyMetadataException e) { + } catch (LoadAnalyzeException e) { LOGGER.warn("Auto create database failed: {}", database, e); analysis.setFinishQueryAfterAnalyze(true); analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage())); @@ -118,8 +118,7 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { } @Override - protected void analyzeSingleTsFile(final File tsFile) - throws IOException, VerifyMetadataException { + protected void analyzeSingleTsFile(final File tsFile) throws IOException, LoadAnalyzeException { try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { // can be reused when constructing tsfile resource final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator = @@ -189,7 +188,7 @@ protected void analyzeSingleTsFile(final File tsFile) } } - private void autoCreateDatabaseIfAbsent(final String database) throws VerifyMetadataException { + private void autoCreateDatabaseIfAbsent(final String database) throws LoadAnalyzeException { validateDatabaseName(database); if (DataNodeTableCache.getInstance().isDatabaseExist(database)) { return; @@ -202,13 +201,13 @@ private void autoCreateDatabaseIfAbsent(final String database) throws VerifyMeta task.execute(ClusterConfigTaskExecutor.getInstance()); final ConfigTaskResult result = future.get(); if (result.getStatusCode().getStatusCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Auto create database failed: %s, status code: %s", database, result.getStatusCode())); } } catch (final ExecutionException | InterruptedException e) { - throw new VerifyMetadataException("Auto create database failed because: " + e.getMessage()); + throw new LoadAnalyzeException("Auto create database failed because: " + e.getMessage()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java index 058a6f753c8b..84d684733dcc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java @@ -21,7 +21,8 @@ import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.VerifyMetadataTypeMismatchException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException; import org.apache.iotdb.db.exception.load.LoadEmptyFileException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -86,8 +87,8 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { } catch (AuthException e) { setFailAnalysisForAuthException(analysis, e); return analysis; - } catch (VerifyMetadataTypeMismatchException e) { - executeDataTypeConversionOnTypeMismatch(analysis, e); + } catch (LoadAnalyzeException e) { + executeTabletConversion(analysis, e); return analysis; } catch (Exception e) { final String exceptionMessage = @@ -110,7 +111,7 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { @Override protected void analyzeSingleTsFile(final File tsFile) - throws IOException, AuthException, VerifyMetadataTypeMismatchException { + throws IOException, AuthException, LoadAnalyzeTypeMismatchException { try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { // can be reused when constructing tsfile resource final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java index ac389b2a6dbf..3348a7f83f35 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java @@ -34,8 +34,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.VerifyMetadataException; -import org.apache.iotdb.db.exception.VerifyMetadataTypeMismatchException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException; import org.apache.iotdb.db.exception.load.LoadFileException; import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException; import org.apache.iotdb.db.exception.sql.SemanticException; @@ -105,7 +105,7 @@ public void setCurrentModificationsAndTimeIndex( public void autoCreateAndVerify( TsFileSequenceReader reader, Map> device2TimeseriesMetadataList) - throws IOException, AuthException, VerifyMetadataTypeMismatchException { + throws IOException, AuthException, LoadAnalyzeTypeMismatchException { for (final Map.Entry> entry : device2TimeseriesMetadataList.entrySet()) { final IDeviceID device = entry.getKey(); @@ -206,14 +206,14 @@ public void flushAndClearDeviceIsAlignedCacheIfNecessary() throws SemanticExcept schemaCache.clearDeviceIsAlignedCacheIfNecessary(); } - public void flush() throws AuthException, VerifyMetadataTypeMismatchException { + public void flush() throws AuthException, LoadAnalyzeTypeMismatchException { doAutoCreateAndVerify(); schemaCache.clearTimeSeries(); } private void doAutoCreateAndVerify() - throws SemanticException, AuthException, VerifyMetadataTypeMismatchException { + throws SemanticException, AuthException, LoadAnalyzeTypeMismatchException { if (schemaCache.getDevice2TimeSeries().isEmpty()) { return; } @@ -236,7 +236,7 @@ private void doAutoCreateAndVerify() } } catch (AuthException e) { throw e; - } catch (VerifyMetadataTypeMismatchException e) { + } catch (LoadAnalyzeTypeMismatchException e) { if (loadTsFileAnalyzer.isConvertOnTypeMismatch()) { // throw exception to convert data type in the upper layer (LoadTsFileAnalyzer) throw e; @@ -256,7 +256,7 @@ private void handleException(Exception e, String statementString) throws Semanti statementString, e.getMessage())); } - private void makeSureNoDuplicatedMeasurementsInDevices() throws VerifyMetadataException { + private void makeSureNoDuplicatedMeasurementsInDevices() throws LoadAnalyzeException { for (final Map.Entry> entry : schemaCache.getDevice2TimeSeries().entrySet()) { final IDeviceID device = entry.getKey(); @@ -264,7 +264,7 @@ private void makeSureNoDuplicatedMeasurementsInDevices() throws VerifyMetadataEx for (final MeasurementSchema timeseriesSchema : entry.getValue()) { final String measurement = timeseriesSchema.getMeasurementName(); if (measurement2Schema.containsKey(measurement)) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format("Duplicated measurements %s in device %s.", measurement, device)); } measurement2Schema.put(measurement, timeseriesSchema); @@ -273,7 +273,7 @@ private void makeSureNoDuplicatedMeasurementsInDevices() throws VerifyMetadataEx } private void autoCreateDatabase() - throws VerifyMetadataException, LoadFileException, IllegalPathException, AuthException { + throws LoadAnalyzeException, LoadFileException, IllegalPathException, AuthException { final int databasePrefixNodesLength = loadTsFileAnalyzer.getDatabaseLevel() + 1; final Set databasesNeededToBeSet = new HashSet<>(); @@ -282,7 +282,7 @@ private void autoCreateDatabase() final String[] devicePrefixNodes = devicePath.getNodes(); if (devicePrefixNodes.length < databasePrefixNodesLength) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Database level %d is longer than device %s.", databasePrefixNodesLength, device)); } @@ -411,7 +411,7 @@ private ISchemaTree autoCreateSchema() throws IllegalPathException { } private void verifySchema(ISchemaTree schemaTree) - throws VerifyMetadataException, IllegalPathException { + throws LoadAnalyzeException, IllegalPathException { for (final Map.Entry> entry : schemaCache.getDevice2TimeSeries().entrySet()) { final IDeviceID device = entry.getKey(); @@ -424,7 +424,7 @@ private void verifySchema(ISchemaTree schemaTree) .collect(Collectors.toList())); if (iotdbDeviceSchemaInfo == null) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Device %s does not exist in IoTDB and can not be created. " + "Please check weather auto-create-schema is enabled.", @@ -435,7 +435,7 @@ private void verifySchema(ISchemaTree schemaTree) final boolean isAlignedInTsFile = schemaCache.getDeviceIsAligned(device); final boolean isAlignedInIoTDB = iotdbDeviceSchemaInfo.isAligned(); if (isAlignedInTsFile != isAlignedInIoTDB) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Device %s in TsFile is %s, but in IoTDB is %s.", device, @@ -450,7 +450,7 @@ private void verifySchema(ISchemaTree schemaTree) final IMeasurementSchema tsFileSchema = tsfileTimeseriesSchemas.get(i); final IMeasurementSchema iotdbSchema = iotdbTimeseriesSchemas.get(i); if (iotdbSchema == null) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Measurement %s does not exist in IoTDB and can not be created. " + "Please check weather auto-create-schema is enabled.", @@ -459,7 +459,7 @@ private void verifySchema(ISchemaTree schemaTree) // check datatype if (!tsFileSchema.getType().equals(iotdbSchema.getType())) { - throw new VerifyMetadataTypeMismatchException( + throw new LoadAnalyzeTypeMismatchException( String.format( "Measurement %s%s%s datatype not match, TsFile: %s, IoTDB: %s", device, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index 23c74b525e5a..1234c1cf1458 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; @@ -112,7 +113,8 @@ Optional validateTableHeaderSchema( final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictIdColumn); + final boolean isStrictIdColumn) + throws LoadAnalyzeTableColumnDisorderException; /** * This method is used for table device validation and should be invoked after column validation. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 7bee39bdfb1d..ab2ba5b9f331 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinScalarFunction; import org.apache.iotdb.commons.udf.utils.TableUDFUtils; import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; @@ -734,7 +735,8 @@ public Optional validateTableHeaderSchema( TableSchema tableSchema, MPPQueryContext context, boolean allowCreateTable, - boolean isStrictIdColumn) { + boolean isStrictIdColumn) + throws LoadAnalyzeTableColumnDisorderException { return TableHeaderSchemaValidator.getInstance() .validateTableHeaderSchema( database, tableSchema, context, allowCreateTable, isStrictIdColumn); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java index 05cb52e84f61..3e0e00541f3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException; import org.apache.iotdb.db.exception.sql.ColumnCreationFailException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -86,7 +87,8 @@ public Optional validateTableHeaderSchema( final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictIdColumn) { + final boolean isStrictIdColumn) + throws LoadAnalyzeTableColumnDisorderException { InformationSchemaUtils.checkDBNameInWrite(database); // The schema cache R/W and fetch operation must be locked together thus the cache clean @@ -136,7 +138,7 @@ public Optional validateTableHeaderSchema( final String idName = realIdColumns.get(indexReal).getColumnName(); final int indexIncoming = tableSchema.getIndexAmongIdColumns(idName); if (indexIncoming != indexReal) { - throw new SemanticException( + throw new LoadAnalyzeTableColumnDisorderException( String.format( "Can not create table because incoming table has no less id columns than existing table, " + "and the existing id columns are not the prefix of the incoming id columns. " @@ -151,7 +153,7 @@ public Optional validateTableHeaderSchema( final String idName = incomingIdColumns.get(indexIncoming).getName(); final int indexReal = table.getIdColumnOrdinal(idName); if (indexReal != indexIncoming) { - throw new SemanticException( + throw new LoadAnalyzeTableColumnDisorderException( String.format( "Can not create table because existing table has more id columns than incoming table, " + "and the incoming id columns are not the prefix of the existing id columns. " diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java index 1a8144424028..1366b0720c55 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.pipe.receiver.transform.statement.PipeConvertedInsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.enums.TSDataType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,12 +33,21 @@ public class LoadConvertedInsertTabletStatement extends PipeConvertedInsertTable private static final Logger LOGGER = LoggerFactory.getLogger(LoadConvertedInsertTabletStatement.class); - public LoadConvertedInsertTabletStatement(final InsertTabletStatement insertTabletStatement) { + private final boolean shouldConvertOnTypeMismatch; + + public LoadConvertedInsertTabletStatement( + final InsertTabletStatement insertTabletStatement, + final boolean shouldConvertOnTypeMismatch) { super(insertTabletStatement); + this.shouldConvertOnTypeMismatch = shouldConvertOnTypeMismatch; } @Override protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { + if (!shouldConvertOnTypeMismatch) { + return originalCheckAndCastDataType(columnIndex, dataType); + } + LOGGER.info( "Load: Inserting tablet to {}.{}. Casting type from {} to {}.", devicePath, @@ -49,4 +59,10 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { dataTypes[columnIndex] = dataType; return true; } + + @TableModel + @Override + public boolean isForceTypeConversion() { + return shouldConvertOnTypeMismatch; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java index 35a163b4df02..de9dd0908879 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java @@ -90,7 +90,8 @@ public Optional visitLoadTsFile( rawTabletInsertionEvent.convertToTablet(), rawTabletInsertionEvent.isAligned(), databaseName) - .constructStatement()); + .constructStatement(), + loadTsFileStatement.isConvertOnTypeMismatch()); TSStatus result; try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java index 5793b96502a1..6f7d732e6fb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java @@ -76,7 +76,8 @@ file, new IoTDBTreePattern(null), Long.MIN_VALUE, Long.MAX_VALUE, null, null)) { new LoadConvertedInsertTabletStatement( PipeTransferTabletRawReq.toTPipeTransferRawReq( tabletWithIsAligned.getLeft(), tabletWithIsAligned.getRight()) - .constructStatement()); + .constructStatement(), + loadTsFileStatement.isConvertOnTypeMismatch()); TSStatus result; try { From debdb527d897decf62a37254be27061e3127edb9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:04:01 +0800 Subject: [PATCH 03/13] Reduced the error log cause by batch activate template when some devices are already activated (#14661) --- .../schemaregion/SchemaExecutionVisitor.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index c10d5d7339a4..daab55117d19 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -445,6 +445,7 @@ public TSStatus visitActivateTemplate( public TSStatus visitBatchActivateTemplate( final BatchActivateTemplateNode node, final ISchemaRegion schemaRegion) { final List statusList = new ArrayList<>(); + final List alreadyActivatedDeviceList = new ArrayList<>(); for (final Map.Entry> entry : node.getTemplateActivationMap().entrySet()) { final Template template = @@ -455,10 +456,20 @@ public TSStatus visitBatchActivateTemplate( entry.getKey(), entry.getValue().right, entry.getValue().left), template); } catch (final MetadataException e) { - logger.error(e.getMessage(), e); - statusList.add(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); + if (e.getErrorCode() == TSStatusCode.TEMPLATE_IS_IN_USE.getStatusCode()) { + alreadyActivatedDeviceList.add(entry.getKey()); + } else { + logger.error(e.getMessage(), e); + statusList.add(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); + } } } + if (!alreadyActivatedDeviceList.isEmpty()) { + final TemplateIsInUseException e = + new TemplateIsInUseException(alreadyActivatedDeviceList.toString()); + logger.error(e.getMessage(), e); + statusList.add(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); + } return statusList.isEmpty() ? RpcUtils.SUCCESS_STATUS : RpcUtils.getStatus(statusList); } From a54573a2afbe295ef218bcd910bb7354415e2f2d Mon Sep 17 00:00:00 2001 From: Haonan Date: Thu, 9 Jan 2025 18:06:37 +0800 Subject: [PATCH 04/13] Use DataNodeDevicePathCache when insert by sql (#14662) --- .../iotdb/db/queryengine/plan/parser/ASTVisitor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 9b45b9b03d15..e21aaa3096b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.cq.TimeoutPolicy; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.cache.CacheClearOptions; @@ -60,6 +61,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.window.ainode.InferenceWindow; import org.apache.iotdb.db.queryengine.execution.operator.window.ainode.TailInferenceWindow; import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.expression.Expression; import org.apache.iotdb.db.queryengine.plan.expression.ExpressionType; import org.apache.iotdb.db.queryengine.plan.expression.binary.AdditionExpression; @@ -1912,7 +1914,12 @@ private ResultSetFormat parseAlignBy(IoTDBSqlParser.AlignByClauseContext ctx) { @Override public Statement visitInsertStatement(IoTDBSqlParser.InsertStatementContext ctx) { InsertStatement insertStatement = new InsertStatement(); - insertStatement.setDevice(parsePrefixPath(ctx.prefixPath())); + try { + insertStatement.setDevice( + DataNodeDevicePathCache.getInstance().getPartialPath(ctx.prefixPath().getText())); + } catch (IllegalPathException e) { + throw new SemanticException(e); + } int timeIndex = parseInsertColumnSpec(ctx.insertColumnsSpec(), insertStatement); parseInsertValuesSpec(ctx.insertValuesSpec(), insertStatement, timeIndex); insertStatement.setAligned(ctx.ALIGNED() != null); From 5f01571f4c64033c460545ff885f320433ff8bfc Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 9 Jan 2025 19:18:54 +0800 Subject: [PATCH 05/13] Pipe: tree/table model isolation for alter/start/stop/drop/show pipe operations & support 'mode.double-living' (#14386) --- .../pipe/it/autocreate/IoTDBPipeAlterIT.java | 3 +- .../autocreate/IoTDBPipeSwitchStatusIT.java | 23 +- .../pipe/it/tablemodel/IoTDBPipeAlterIT.java | 30 +- .../tablemodel/IoTDBPipeDoubleLivingIT.java | 329 +++++++++++++++ .../it/tablemodel/IoTDBPipeIsolationIT.java | 377 ++++++++++++++++++ .../tablemodel/IoTDBPipeSwitchStatusIT.java | 23 +- .../pipe/it/tablemodel/IoTDBPipeSyntaxIT.java | 33 +- .../pipe/it/tablemodel/TableModelUtils.java | 17 + .../response/pipe/task/PipeTableResp.java | 7 + .../confignode/manager/ConfigManager.java | 10 +- .../iotdb/confignode/manager/IManager.java | 23 +- .../coordinator/task/PipeTaskCoordinator.java | 74 +++- .../persistence/pipe/PipeTaskInfo.java | 21 +- .../impl/pipe/task/AlterPipeProcedureV2.java | 4 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 16 +- .../dataregion/IoTDBDataRegionExtractor.java | 20 +- .../db/protocol/client/ConfigNodeClient.java | 14 + .../executor/ClusterConfigTaskExecutor.java | 20 +- .../config/sys/pipe/AlterPipeTask.java | 8 +- .../config/sys/pipe/DropPipeTask.java | 7 +- .../config/sys/pipe/ShowPipeTask.java | 3 +- .../config/sys/pipe/StartPipeTask.java | 7 +- .../config/sys/pipe/StopPipeTask.java | 7 +- .../plan/relational/sql/ast/AlterPipe.java | 22 +- .../plan/relational/sql/ast/DropPipe.java | 10 +- .../plan/relational/sql/ast/ShowPipes.java | 10 +- .../plan/relational/sql/ast/StartPipe.java | 8 +- .../plan/relational/sql/ast/StopPipe.java | 8 +- .../metadata/pipe/AlterPipeStatement.java | 31 +- .../metadata/pipe/DropPipeStatement.java | 23 +- .../metadata/pipe/ShowPipesStatement.java | 18 +- .../metadata/pipe/StartPipeStatement.java | 17 +- .../metadata/pipe/StopPipeStatement.java | 17 +- .../pipe/agent/task/meta/PipeMeta.java | 40 ++ .../pipe/agent/task/meta/PipeMetaKeeper.java | 8 + .../constant/PipeExtractorConstant.java | 3 + .../datastructure/pattern/TablePattern.java | 21 +- .../datastructure/pattern/TreePattern.java | 22 +- .../pipe/extractor/IoTDBExtractor.java | 65 ++- .../src/main/thrift/confignode.thrift | 19 + 40 files changed, 1239 insertions(+), 179 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java index 66bae32c5b10..a2f5c65cb73e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java @@ -26,7 +26,6 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.Assert; import org.junit.Test; @@ -106,7 +105,7 @@ public void testBasicAlterPipe() throws Exception { } // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { statement.execute("alter pipe a2b modify source ('source.pattern'='root.test2')"); } catch (SQLException e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java index cdcc41ad53fe..03e52a982e61 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java @@ -266,11 +266,14 @@ public void testWrongPipeName() throws Exception { .setExtractorAttributes(extractorAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("").getCode()); Assert.assertEquals( - TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("*").getCode()); + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("*").getCode()); List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); @@ -281,10 +284,14 @@ public void testWrongPipeName() throws Exception { Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("*").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("*").getCode()); showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java index a81b02f7dd73..1665f9d2ce2b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java @@ -65,7 +65,8 @@ public void testBasicAlterPipe() throws Exception { long lastCreationTime; try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -96,7 +97,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -114,7 +116,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -149,7 +152,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -183,7 +187,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -225,7 +230,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -257,7 +263,8 @@ public void testBasicAlterPipe() throws Exception { // show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -289,7 +296,8 @@ public void testBasicAlterPipe() throws Exception { // show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -323,7 +331,8 @@ public void testBasicAlterPipe() throws Exception { // show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -354,7 +363,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java new file mode 100644 index 000000000000..f6573728cf29 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.tablemodel; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelTestIT { + + @Test + public void testDoubleLivingInvalidParameter() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.table'='false'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p2", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'forwarding-pipe-requests'='true'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p3", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + // combination of org.apache.iotdb.pipe.it.tablemodel.IoTDBPipeLifeCycleIT.testDoubleLiving and + // org.apache.iotdb.pipe.it.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving + @Test + public void testBasicDoubleLiving() { + boolean insertResult; + + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + }; + + // insertion on sender + for (int i = 0; i < 100; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'database-name'='test'," + + "'table-name'='test'," + + "'path'='root.db.d1.s1'," + + "'mode.double-living'='true')" + + " with sink (" + + "'batch.enable'='false'," + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // insertion on sender + for (int i = 100; i < 200; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + for (int i = 200; i < 300; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test", 200, 300, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'database-name'='test'," + + "'table-name'='test'," + + "'path'='root.db.d1.s1'," + + "'mode.double-living'='true')" + + " with sink (" + + "'batch.enable'='false'," + + "'node-urls'='%s')", + "p2", senderDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // insertion on receiver + for (int i = 300; i < 400; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 300, 400, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + // check result + final Set expectedResSet = new HashSet<>(); + for (int i = 0; i < 400; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TableModelUtils.assertData("test", "test", 0, 400, senderEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + + // restart cluster + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + // insertion on receiver + for (int i = 400; i < 500; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 400, 500, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + // check result + for (int i = 400; i < 500; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TableModelUtils.assertData("test", "test", 0, 500, senderEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 500, receiverEnv, handleFailure); + } + + @Test + public void testDoubleLivingIsolation() throws Exception { + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // Create table pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Drop pipe + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java new file mode 100644 index 000000000000..57a3be38e217 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.tablemodel; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeIsolationIT extends AbstractPipeTableModelTestIT { + + @Test + public void testWritePipeIsolation() throws Exception { + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create table pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Start pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(treePipeName).setIsTableModel(true)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(treePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(tablePipeName).setIsTableModel(true)) + .getCode()); + + // Stop pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client.stopPipeExtended(new TStopPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .stopPipeExtended(new TStopPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.stopPipeExtended(new TStopPipeReq(treePipeName).setIsTableModel(false)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.stopPipeExtended(new TStopPipeReq(tablePipeName).setIsTableModel(true)).getCode()); + + // Alter pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + treePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(true)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + tablePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + treePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + tablePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(true)) + .getCode()); + + // Drop pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(false)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(true)).getCode()); + } + } + + @Test + public void testReadPipeIsolation() { + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + } + + @Test + public void testCaptureTreeAndTableIsolation() throws Exception { + final String treePipeName = "tree_a2b"; + final String tablePipeName = "table_a2b"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 3. Drop pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } + } + + @Test + public void testCaptureCornerCases() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe but capture table data + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe but capture tree data + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='false')" + + " with sink (" + + "'node-urls'='%s')", + "p2", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 3. Create pipe with capture.tree and capture.table set to false + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'capture.table'='false')" + + " with sink (" + + "'node-urls'='%s')", + "p3", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java index 5c5cbcd7b950..4c80f0ae6731 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java @@ -303,11 +303,14 @@ public void testWrongPipeName() throws Exception { .setExtractorAttributes(extractorAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("").getCode()); Assert.assertEquals( - TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("*").getCode()); + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("*").getCode()); List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); @@ -318,10 +321,14 @@ public void testWrongPipeName() throws Exception { Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("*").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("*").getCode()); showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java index 24b1c92563b6..09658a583b08 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java @@ -89,7 +89,8 @@ public void testValidPipeName() throws Exception { } } - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; for (final String pipeName : expectedPipeNames) { Assert.assertTrue( showPipeResult.stream() @@ -106,7 +107,7 @@ public void testValidPipeName() throws Exception { } } - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + showPipeResult = client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(0, showPipeResult.size()); } } @@ -124,7 +125,7 @@ public void testRevertParameterOrder() { String.format( "create pipe p1" + " with source ( " - + "'capture.table'='test'," + + "'capture.table'='true'," + "'database-name'='test'," + "'table-name'='test'," + "'mode.streaming'='true'," @@ -177,7 +178,8 @@ public void testRevertStageOrder() throws Exception { } catch (SQLException ignored) { } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(0, showPipeResult.size()); } } @@ -249,7 +251,8 @@ public void testMissingStage() throws Exception { fail(e.getMessage()); } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(2, showPipeResult.size()); } } @@ -315,7 +318,8 @@ public void testInvalidParameter() throws Exception { } catch (SQLException ignored) { } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); } } @@ -522,7 +526,8 @@ public void testBrackets() throws Exception { } catch (Exception ignored) { } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(2, showPipeResult.size()); } } @@ -540,6 +545,7 @@ public void testShowPipeWithWrongPipeName() throws Exception { final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.database-name", "test"); extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); @@ -578,10 +584,12 @@ public void testShowPipeWithWrongPipeName() throws Exception { .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(3, showPipeResult.size()); - showPipeResult = client.showPipe(new TShowPipeReq().setPipeName("p1")).pipeInfoList; + showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true).setPipeName("p1")).pipeInfoList; Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); @@ -589,7 +597,9 @@ public void testShowPipeWithWrongPipeName() throws Exception { // Show all pipes whose connector is also used by p1. // p1 and p2 share the same connector parameters, so they have the same connector. showPipeResult = - client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; + client.showPipe( + new TShowPipeReq().setIsTableModel(true).setPipeName("p1").setWhereClause(true)) + .pipeInfoList; Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); @@ -678,7 +688,8 @@ public void testInclusionPattern() throws Exception { fail(e.getMessage()); } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java index c7da1382535d..a3c8c3757420 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java @@ -37,6 +37,8 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.text.SimpleDateFormat; import java.time.LocalDate; @@ -772,4 +774,19 @@ public static Tablet generateTabletDeviceIDAllIsNull( return tablet; } + + public static int showPipesCount(final BaseEnv baseEnv, final String sqlDialect) { + try (final Connection connection = baseEnv.getConnection(sqlDialect); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + return count; + } catch (final SQLException e) { + fail(e.getMessage()); + } + return 0; + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java index 020690f24651..20cd76e4a41d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java @@ -94,6 +94,13 @@ public PipeTableResp filter(final Boolean whereClause, final String pipeName) { } } + public PipeTableResp filter( + final Boolean whereClause, final String pipeName, final boolean isTableModel) { + final PipeTableResp resp = filter(whereClause, pipeName); + resp.allPipeMeta.removeIf(meta -> !meta.visibleUnder(isTableModel)); + return resp; + } + public TGetAllPipeInfoResp convertToTGetAllPipeInfoResp() throws IOException { final List pipeInformationByteBuffers = new ArrayList<>(); for (final PipeMeta pipeMeta : allPipeMeta) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 130f1dcd15c4..6f6d625768c4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -226,6 +226,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; @@ -2193,18 +2195,18 @@ public TSStatus alterPipe(TAlterPipeReq req) { } @Override - public TSStatus startPipe(String pipeName) { + public TSStatus startPipe(TStartPipeReq req) { TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - ? pipeManager.getPipeTaskCoordinator().startPipe(pipeName) + ? pipeManager.getPipeTaskCoordinator().startPipe(req) : status; } @Override - public TSStatus stopPipe(String pipeName) { + public TSStatus stopPipe(TStopPipeReq req) { TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - ? pipeManager.getPipeTaskCoordinator().stopPipe(pipeName) + ? pipeManager.getPipeTaskCoordinator().stopPipe(req) : status; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 63e527440ede..82dae052ef23 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -147,6 +147,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; @@ -694,34 +696,39 @@ TDataPartitionTableResp getOrCreateDataPartition( * Alter Pipe. * * @param req Info about Pipe - * @return TSStatus + * @return {@link TSStatusCode#SUCCESS_STATUS} if altered the pipe successfully, {@link + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ TSStatus alterPipe(TAlterPipeReq req); /** * Start Pipe. * - * @param pipeName name of Pipe + * @param req Info about Pipe * @return {@link TSStatusCode#SUCCESS_STATUS} if started the pipe successfully, {@link - * TSStatusCode#PIPE_ERROR} if encountered failure. + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ - TSStatus startPipe(String pipeName); + TSStatus startPipe(TStartPipeReq req); /** * Stop Pipe. * - * @param pipeName name of Pipe + * @param req Info about Pipe * @return {@link TSStatusCode#SUCCESS_STATUS} if stopped the pipe successfully, {@link - * TSStatusCode#PIPE_ERROR} if encountered failure. + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ - TSStatus stopPipe(String pipeName); + TSStatus stopPipe(TStopPipeReq req); /** * Drop Pipe. * * @param req Info about Pipe * @return {@link TSStatusCode#SUCCESS_STATUS} if dropped the pipe successfully, {@link - * TSStatusCode#PIPE_ERROR} if encountered failure. + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ TSStatus dropPipe(TDropPipeReq req); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index 7de39849e599..4aaf3ab46c3f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -31,6 +31,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -116,7 +118,7 @@ public boolean isLocked() { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus createPipe(TCreatePipeReq req) { - TSStatus status = null; + final TSStatus status; if (req.getPipeName().startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().createConsensusPipe(req); } else { @@ -130,6 +132,17 @@ public TSStatus createPipe(TCreatePipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus alterPipe(TAlterPipeReq req) { + final String pipeName = req.getPipeName(); + final boolean isSetIfExistsCondition = + req.isSetIfExistsCondition() && req.isIfExistsCondition(); + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + return isSetIfExistsCondition + ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) + : RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to alter pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } final TSStatus status = configManager.getProcedureManager().alterPipe(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn("Failed to alter pipe {}. Result status: {}.", req.getPipeName(), status); @@ -138,8 +151,8 @@ public TSStatus alterPipe(TAlterPipeReq req) { } /** Caller should ensure that the method is called in the lock {@link #lock()}. */ - public TSStatus startPipe(String pipeName) { - TSStatus status = null; + private TSStatus startPipe(String pipeName) { + final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().startConsensusPipe(pipeName); } else { @@ -152,9 +165,21 @@ public TSStatus startPipe(String pipeName) { } /** Caller should ensure that the method is called in the lock {@link #lock()}. */ - public TSStatus stopPipe(String pipeName) { + public TSStatus startPipe(TStartPipeReq req) { + final String pipeName = req.getPipeName(); + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + return RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to start pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } + return startPipe(pipeName); + } + + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ + private TSStatus stopPipe(String pipeName) { final boolean isStoppedByRuntimeException = pipeTaskInfo.isStoppedByRuntimeException(pipeName); - TSStatus status = null; + final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().stopConsensusPipe(pipeName); } else { @@ -176,11 +201,32 @@ public TSStatus stopPipe(String pipeName) { return status; } + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ + public TSStatus stopPipe(TStopPipeReq req) { + final String pipeName = req.getPipeName(); + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + return RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to stop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } + return stopPipe(pipeName); + } + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus dropPipe(TDropPipeReq req) { final String pipeName = req.getPipeName(); - final boolean isPipeExistedBeforeDrop = pipeTaskInfo.isPipeExisted(pipeName); - TSStatus status = null; + final boolean isSetIfExistsCondition = + req.isSetIfExistsCondition() && req.isIfExistsCondition(); + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + return isSetIfExistsCondition + ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) + : RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to drop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } + final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().dropConsensusPipe(pipeName); } else { @@ -189,23 +235,13 @@ public TSStatus dropPipe(TDropPipeReq req) { if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn("Failed to drop pipe {}. Result status: {}.", pipeName, status); } - - final boolean isSetIfExistsCondition = - req.isSetIfExistsCondition() && req.isIfExistsCondition(); - // If the `IF EXISTS` condition is not set and the pipe does not exist before the delete - // operation, return an error status indicating that the pipe does not exist. - return isPipeExistedBeforeDrop || isSetIfExistsCondition - ? status - : RpcUtils.getStatus( - TSStatusCode.PIPE_NOT_EXIST_ERROR, - String.format( - "Failed to drop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + return status; } public TShowPipeResp showPipes(final TShowPipeReq req) { try { return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2())) - .filter(req.whereClause, req.pipeName) + .filter(req.whereClause, req.pipeName, req.isTableModel) .convertToTShowPipeResp(); } catch (final ConsensusException e) { LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 372209bfd741..8fe6659dc0c8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -187,23 +187,19 @@ private boolean checkBeforeCreatePipeInternal(final TCreatePipeReq createPipeReq throw new PipeException(exceptionMessage); } - public boolean checkAndUpdateRequestBeforeAlterPipe(final TAlterPipeReq alterPipeRequest) + public void checkAndUpdateRequestBeforeAlterPipe(final TAlterPipeReq alterPipeRequest) throws PipeException { acquireReadLock(); try { - return checkAndUpdateRequestBeforeAlterPipeInternal(alterPipeRequest); + checkAndUpdateRequestBeforeAlterPipeInternal(alterPipeRequest); } finally { releaseReadLock(); } } - private boolean checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq alterPipeRequest) + private void checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq alterPipeRequest) throws PipeException { if (!isPipeExisted(alterPipeRequest.getPipeName())) { - if (alterPipeRequest.isSetIfExistsCondition() && alterPipeRequest.isIfExistsCondition()) { - return false; - } - final String exceptionMessage = String.format( "Failed to alter pipe %s, %s", alterPipeRequest.getPipeName(), PIPE_NOT_EXIST_MSG); @@ -267,8 +263,6 @@ private boolean checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq .getAttribute()); } } - - return true; } public void checkBeforeStartPipe(final String pipeName) throws PipeException { @@ -351,6 +345,15 @@ public boolean isPipeExisted(final String pipeName) { } } + public boolean isPipeExisted(final String pipeName, final boolean isTableModel) { + acquireReadLock(); + try { + return pipeMetaKeeper.containsPipeMeta(pipeName, isTableModel); + } finally { + releaseReadLock(); + } + } + private PipeStatus getPipeStatus(final String pipeName) { acquireReadLock(); try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java index 8b6aceb9c300..ba7d40abc53f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java @@ -96,9 +96,7 @@ public boolean executeFromValidateTask(ConfigNodeProcedureEnv env) throws PipeEx // We should execute checkBeforeAlterPipe before checking the pipe plugin. This method will // update the alterPipeRequest based on the alterPipeRequest and existing pipe metadata. - if (!pipeTaskInfo.get().checkAndUpdateRequestBeforeAlterPipe(alterPipeRequest)) { - return false; - } + pipeTaskInfo.get().checkAndUpdateRequestBeforeAlterPipe(alterPipeRequest); final PipeManager pipeManager = env.getConfigManager().getPipeManager(); pipeManager diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 2961a5f7c421..219b68813e6b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -200,6 +200,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp; import org.apache.iotdb.confignode.rpc.thrift.TTestOperation; @@ -1113,12 +1115,22 @@ public TSStatus alterPipe(TAlterPipeReq req) { @Override public TSStatus startPipe(String pipeName) { - return configManager.startPipe(pipeName); + return configManager.startPipe(new TStartPipeReq().setPipeName(pipeName)); + } + + @Override + public TSStatus startPipeExtended(TStartPipeReq req) { + return configManager.startPipe(req); } @Override public TSStatus stopPipe(String pipeName) { - return configManager.stopPipe(pipeName); + return configManager.stopPipe(new TStopPipeReq().setPipeName(pipeName)); + } + + @Override + public TSStatus stopPipeExtended(TStopPipeReq req) { + return configManager.stopPipe(req); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java index bf9a67061b43..0a033aff297b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java @@ -48,6 +48,7 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -134,7 +135,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { .getStringOrDefault( SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); - final boolean isTreeModelDataAllowedToBeCaptured = + final boolean isCaptureTree = validator .getParameters() .getBooleanOrDefault( @@ -142,7 +143,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), isTreeDialect); - final boolean isTableModelDataAllowedToBeCaptured = + final boolean isCaptureTable = validator .getParameters() .getBooleanOrDefault( @@ -150,6 +151,21 @@ public void validate(final PipeParameterValidator validator) throws Exception { PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), !isTreeDialect); + if (!isCaptureTree && !isCaptureTable) { + throw new PipeParameterNotValidException( + "capture.tree and capture.table can not both be specified as false"); + } + + final boolean isDoubleLiving = + validator + .getParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + final boolean isTreeModelDataAllowedToBeCaptured = isDoubleLiving || isCaptureTree; + final boolean isTableModelDataAllowedToBeCaptured = isDoubleLiving || isCaptureTable; if (!isTreeModelDataAllowedToBeCaptured && validator .getParameters() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 5fd09f22a9aa..f50fbfd63bdb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -163,6 +163,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp; import org.apache.iotdb.confignode.rpc.thrift.TTestOperation; @@ -1054,12 +1056,24 @@ public TSStatus startPipe(String pipeName) throws TException { () -> client.startPipe(pipeName), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus startPipeExtended(TStartPipeReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.startPipeExtended(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus stopPipe(String pipeName) throws TException { return executeRemoteCallWithRetry( () -> client.stopPipe(pipeName), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus stopPipeExtended(TStopPipeReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.stopPipeExtended(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus dropPipe(String pipeName) throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 4a14c71a3d37..3c41ea2ac2ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -133,6 +133,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -2044,6 +2046,7 @@ public SettableFuture alterPipe(final AlterPipeStatement alter req.setExtractorAttributes(alterPipeStatement.getExtractorAttributes()); req.setIsReplaceAllExtractorAttributes(alterPipeStatement.isReplaceAllExtractorAttributes()); req.setIfExistsCondition(alterPipeStatement.hasIfExistsCondition()); + req.setIsTableModel(alterPipeStatement.isTableModel()); final TSStatus tsStatus = configNodeClient.alterPipe(req); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); @@ -2073,7 +2076,11 @@ public SettableFuture startPipe(final StartPipeStatement start try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TSStatus tsStatus = configNodeClient.startPipe(startPipeStatement.getPipeName()); + final TSStatus tsStatus = + configNodeClient.startPipeExtended( + new TStartPipeReq() + .setPipeName(startPipeStatement.getPipeName()) + .setIsTableModel(startPipeStatement.isTableModel())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2106,7 +2113,8 @@ public SettableFuture dropPipe(final DropPipeStatement dropPip configNodeClient.dropPipeExtended( new TDropPipeReq() .setPipeName(dropPipeStatement.getPipeName()) - .setIfExistsCondition(dropPipeStatement.hasIfExistsCondition())); + .setIfExistsCondition(dropPipeStatement.hasIfExistsCondition()) + .setIsTableModel(dropPipeStatement.isTableModel())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2135,7 +2143,12 @@ public SettableFuture stopPipe(final StopPipeStatement stopPip try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TSStatus tsStatus = configNodeClient.stopPipe(stopPipeStatement.getPipeName()); + + final TSStatus tsStatus = + configNodeClient.stopPipeExtended( + new TStopPipeReq() + .setPipeName(stopPipeStatement.getPipeName()) + .setIsTableModel(stopPipeStatement.isTableModel())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2159,6 +2172,7 @@ public SettableFuture showPipes(final ShowPipesStatement showP if (showPipesStatement.getWhereClause()) { tShowPipeReq.setWhereClause(true); } + tShowPipeReq.setIsTableModel(showPipesStatement.isTableModel()); final List tShowPipeInfoList = configNodeClient.showPipe(tShowPipeReq).getPipeInfoList(); ShowPipeTask.buildTSBlock(tShowPipeInfoList, future); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java index f6d0ebab2cbf..159c6941628e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java @@ -37,13 +37,13 @@ public class AlterPipeTask implements IConfigTask { private final AlterPipeStatement alterPipeStatement; - public AlterPipeTask(AlterPipeStatement alterPipeStatement) { + public AlterPipeTask(final AlterPipeStatement alterPipeStatement) { // support now() function applyNowFunctionToExtractorAttributes(alterPipeStatement.getExtractorAttributes()); this.alterPipeStatement = alterPipeStatement; } - public AlterPipeTask(AlterPipe node) { + public AlterPipeTask(final AlterPipe node) { alterPipeStatement = new AlterPipeStatement(StatementType.ALTER_PIPE); alterPipeStatement.setPipeName(node.getPipeName()); alterPipeStatement.setIfExists(node.hasIfExistsCondition()); @@ -57,10 +57,12 @@ public AlterPipeTask(AlterPipe node) { alterPipeStatement.setReplaceAllExtractorAttributes(node.isReplaceAllExtractorAttributes()); alterPipeStatement.setReplaceAllProcessorAttributes(node.isReplaceAllProcessorAttributes()); alterPipeStatement.setReplaceAllConnectorAttributes(node.isReplaceAllConnectorAttributes()); + + alterPipeStatement.setTableModel(true); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.alterPipe(alterPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java index a28ca91f3c65..a10c6042241c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java @@ -32,18 +32,19 @@ public class DropPipeTask implements IConfigTask { private final DropPipeStatement dropPipeStatement; - public DropPipeTask(DropPipeStatement dropPipeStatement) { + public DropPipeTask(final DropPipeStatement dropPipeStatement) { this.dropPipeStatement = dropPipeStatement; } - public DropPipeTask(DropPipe node) { + public DropPipeTask(final DropPipe node) { dropPipeStatement = new DropPipeStatement(StatementType.DROP_PIPE); dropPipeStatement.setPipeName(node.getPipeName()); dropPipeStatement.setIfExists(node.hasIfExistsCondition()); + dropPipeStatement.setTableModel(true); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.dropPipe(dropPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java index 6481d0713ac2..c43a9953b584 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java @@ -52,10 +52,11 @@ public ShowPipeTask(final ShowPipesStatement showPipesStatement) { this.showPipesStatement = showPipesStatement; } - public ShowPipeTask(ShowPipes node) { + public ShowPipeTask(final ShowPipes node) { showPipesStatement = new ShowPipesStatement(); showPipesStatement.setPipeName(node.getPipeName()); showPipesStatement.setWhereClause(node.hasWhereClause()); + showPipesStatement.setTableModel(true); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java index 39ff162a43c7..c118f0a3a58a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java @@ -32,17 +32,18 @@ public class StartPipeTask implements IConfigTask { private final StartPipeStatement startPipeStatement; - public StartPipeTask(StartPipeStatement startPipeStatement) { + public StartPipeTask(final StartPipeStatement startPipeStatement) { this.startPipeStatement = startPipeStatement; } - public StartPipeTask(StartPipe node) { + public StartPipeTask(final StartPipe node) { startPipeStatement = new StartPipeStatement(StatementType.START_PIPE); startPipeStatement.setPipeName(node.getPipeName()); + startPipeStatement.setTableModel(true); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.startPipe(startPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java index 04806bef42f6..91764c43263d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java @@ -32,17 +32,18 @@ public class StopPipeTask implements IConfigTask { private final StopPipeStatement stopPipeStatement; - public StopPipeTask(StopPipeStatement stopPipeStatement) { + public StopPipeTask(final StopPipeStatement stopPipeStatement) { this.stopPipeStatement = stopPipeStatement; } - public StopPipeTask(StopPipe node) { + public StopPipeTask(final StopPipe node) { stopPipeStatement = new StopPipeStatement(StatementType.STOP_PIPE); stopPipeStatement.setPipeName(node.getPipeName()); + stopPipeStatement.setTableModel(true); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.stopPipe(stopPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java index 4d398b55fcc2..a40fab917a51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java @@ -88,7 +88,7 @@ public boolean isReplaceAllConnectorAttributes() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitAlterPipe(this, context); } @@ -106,25 +106,25 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - AlterPipe alterPipe = (AlterPipe) obj; - return Objects.equals(pipeName, alterPipe.pipeName) - && Objects.equals(ifExistsCondition, alterPipe.ifExistsCondition) - && Objects.equals(extractorAttributes, alterPipe.extractorAttributes) - && Objects.equals(processorAttributes, alterPipe.processorAttributes) - && Objects.equals(connectorAttributes, alterPipe.connectorAttributes) + final AlterPipe that = (AlterPipe) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.ifExistsCondition, that.ifExistsCondition) + && Objects.equals(this.extractorAttributes, that.extractorAttributes) + && Objects.equals(this.processorAttributes, that.processorAttributes) + && Objects.equals(this.connectorAttributes, that.connectorAttributes) && Objects.equals( - isReplaceAllExtractorAttributes, alterPipe.isReplaceAllExtractorAttributes) + this.isReplaceAllExtractorAttributes, that.isReplaceAllExtractorAttributes) && Objects.equals( - isReplaceAllProcessorAttributes, alterPipe.isReplaceAllProcessorAttributes) + this.isReplaceAllProcessorAttributes, that.isReplaceAllProcessorAttributes) && Objects.equals( - isReplaceAllConnectorAttributes, alterPipe.isReplaceAllConnectorAttributes); + this.isReplaceAllConnectorAttributes, that.isReplaceAllConnectorAttributes); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java index 6952c06d13d6..1b3837ba2147 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java @@ -43,7 +43,7 @@ public boolean hasIfExistsCondition() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitDropPipe(this, context); } @@ -53,16 +53,16 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - DropPipe other = (DropPipe) obj; - return Objects.equals(pipeName, other.pipeName) - && Objects.equals(ifExistsCondition, other.ifExistsCondition); + final DropPipe that = (DropPipe) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.ifExistsCondition, that.ifExistsCondition); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java index e65079610931..c0dfe9379de1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java @@ -44,7 +44,7 @@ public boolean hasWhereClause() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitShowPipes(this, context); } @@ -54,16 +54,16 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - ShowPipes other = (ShowPipes) obj; - return Objects.equals(pipeName, other.pipeName) - && Objects.equals(hasWhereClause, other.hasWhereClause); + final ShowPipes that = (ShowPipes) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.hasWhereClause, that.hasWhereClause); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java index 540d6fef9ec8..15eb5b2413dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java @@ -37,7 +37,7 @@ public String getPipeName() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitStartPipe(this, context); } @@ -47,15 +47,15 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - StartPipe other = (StartPipe) obj; - return Objects.equals(pipeName, other.pipeName); + final StartPipe that = (StartPipe) obj; + return Objects.equals(this.pipeName, that.pipeName); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java index daa9ab32e02b..6d1a629de882 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java @@ -37,7 +37,7 @@ public String getPipeName() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitStopPipe(this, context); } @@ -47,15 +47,15 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - StopPipe other = (StopPipe) obj; - return Objects.equals(pipeName, other.pipeName); + final StopPipe that = (StopPipe) obj; + return Objects.equals(this.pipeName, that.pipeName); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java index de5dc3c1d594..44065cf31850 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java @@ -44,8 +44,9 @@ public class AlterPipeStatement extends Statement implements IConfigStatement { private boolean isReplaceAllExtractorAttributes; private boolean isReplaceAllProcessorAttributes; private boolean isReplaceAllConnectorAttributes; + private boolean isTableModel; - public AlterPipeStatement(StatementType alterPipeStatement) { + public AlterPipeStatement(final StatementType alterPipeStatement) { this.statementType = alterPipeStatement; } @@ -81,38 +82,46 @@ public boolean isReplaceAllConnectorAttributes() { return isReplaceAllConnectorAttributes; } - public void setPipeName(String pipeName) { + public boolean isTableModel() { + return isTableModel; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setIfExists(boolean ifExistsCondition) { + public void setIfExists(final boolean ifExistsCondition) { this.ifExistsCondition = ifExistsCondition; } - public void setExtractorAttributes(Map extractorAttributes) { + public void setExtractorAttributes(final Map extractorAttributes) { this.extractorAttributes = extractorAttributes; } - public void setProcessorAttributes(Map processorAttributes) { + public void setProcessorAttributes(final Map processorAttributes) { this.processorAttributes = processorAttributes; } - public void setConnectorAttributes(Map connectorAttributes) { + public void setConnectorAttributes(final Map connectorAttributes) { this.connectorAttributes = connectorAttributes; } - public void setReplaceAllExtractorAttributes(boolean replaceAllExtractorAttributes) { + public void setReplaceAllExtractorAttributes(final boolean replaceAllExtractorAttributes) { isReplaceAllExtractorAttributes = replaceAllExtractorAttributes; } - public void setReplaceAllProcessorAttributes(boolean replaceAllProcessorAttributes) { + public void setReplaceAllProcessorAttributes(final boolean replaceAllProcessorAttributes) { isReplaceAllProcessorAttributes = replaceAllProcessorAttributes; } - public void setReplaceAllConnectorAttributes(boolean replaceAllConnectorAttributes) { + public void setReplaceAllConnectorAttributes(final boolean replaceAllConnectorAttributes) { isReplaceAllConnectorAttributes = replaceAllConnectorAttributes; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -124,7 +133,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -134,7 +143,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitAlterPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java index a3403e00e689..c61a15d777d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java @@ -37,27 +37,36 @@ public class DropPipeStatement extends Statement implements IConfigStatement { private String pipeName; private boolean ifExistsCondition; + private boolean isTableModel; - public DropPipeStatement(StatementType dropPipeStatement) { + public DropPipeStatement(final StatementType dropPipeStatement) { this.statementType = dropPipeStatement; } + public String getPipeName() { + return pipeName; + } + public boolean hasIfExistsCondition() { return ifExistsCondition; } - public String getPipeName() { - return pipeName; + public boolean isTableModel() { + return isTableModel; } - public void setPipeName(String pipeName) { + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setIfExists(boolean ifExistsCondition) { + public void setIfExists(final boolean ifExistsCondition) { this.ifExistsCondition = ifExistsCondition; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -69,7 +78,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -79,7 +88,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitDropPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java index 54fe1099f39f..658f36a0635f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java @@ -39,6 +39,8 @@ public ShowPipesStatement() { private boolean whereClause; + private boolean isTableModel; + public String getPipeName() { return pipeName; } @@ -47,21 +49,29 @@ public boolean getWhereClause() { return whereClause; } - public void setPipeName(String pipeName) { + public boolean isTableModel() { + return isTableModel; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setWhereClause(boolean whereClause) { + public void setWhereClause(final boolean whereClause) { this.whereClause = whereClause; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.READ; } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -71,7 +81,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitShowPipes(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java index 43b61a1fb445..9023d77c06ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java @@ -36,8 +36,9 @@ public class StartPipeStatement extends Statement implements IConfigStatement { private String pipeName; + private boolean isTableModel; - public StartPipeStatement(StatementType startPipeStatement) { + public StartPipeStatement(final StatementType startPipeStatement) { this.statementType = startPipeStatement; } @@ -45,10 +46,18 @@ public String getPipeName() { return pipeName; } - public void setPipeName(String pipeName) { + public boolean isTableModel() { + return isTableModel; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -60,7 +69,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -70,7 +79,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitStartPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java index d463fda199d3..ea14a3dbd1b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java @@ -36,8 +36,9 @@ public class StopPipeStatement extends Statement implements IConfigStatement { private String pipeName; + private boolean isTableModel; - public StopPipeStatement(StatementType stopPipeStatement) { + public StopPipeStatement(final StatementType stopPipeStatement) { this.statementType = stopPipeStatement; } @@ -45,10 +46,18 @@ public String getPipeName() { return pipeName; } - public void setPipeName(String pipeName) { + public boolean isTableModel() { + return isTableModel; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -60,7 +69,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -70,7 +79,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitStopPipe(this, context); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index 997278010e9f..033fe6505824 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -19,6 +19,10 @@ package org.apache.iotdb.commons.pipe.agent.task.meta; +import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + import org.apache.tsfile.utils.PublicBAOS; import java.io.DataOutputStream; @@ -26,6 +30,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Objects; public class PipeMeta { @@ -61,6 +66,41 @@ public PipeTemporaryMeta getTemporaryMeta() { return temporaryMeta; } + public boolean visibleUnder(final boolean isTableModel) { + final PipeParameters extractorParameters = getStaticMeta().getExtractorParameters(); + + // visible under all model when 'mode.double-living' is set to true + final boolean isDoubleLiving = + extractorParameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + if (isDoubleLiving) { + return true; + } + + final boolean isTreeDialect = + extractorParameters + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); + final Boolean _isCaptureTree = + extractorParameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY); + final boolean isCaptureTree = Objects.nonNull(_isCaptureTree) ? _isCaptureTree : isTreeDialect; + final Boolean _isCaptureTable = + extractorParameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY); + final boolean isCaptureTable = + Objects.nonNull(_isCaptureTable) ? _isCaptureTable : !isTreeDialect; + + // visible under specific tree or table model <-> actually capture tree or table data + return isTableModel ? isCaptureTable : isCaptureTree; + } + public ByteBuffer serialize() throws IOException { final PublicBAOS byteArrayOutputStream = new PublicBAOS(); final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java index f8a8c9e55ac9..4009288dfc70 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java @@ -85,6 +85,14 @@ public boolean containsPipeMeta(String pipeName) { return pipeNameToPipeMetaMap.containsKey(pipeName); } + public boolean containsPipeMeta(String pipeName, boolean isTableModel) { + final PipeMeta pipeMeta = pipeNameToPipeMetaMap.get(pipeName); + if (Objects.isNull(pipeMeta)) { + return false; + } + return pipeMeta.visibleUnder(isTableModel); + } + public Iterable getPipeMetaList() { return pipeNameToPipeMetaMap.values(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java index 1335b995a593..d5d34c269206 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java @@ -119,6 +119,9 @@ public class PipeExtractorConstant { public static final String EXTRACTOR_MODE_SNAPSHOT_KEY = "extractor.mode.snapshot"; public static final String SOURCE_MODE_SNAPSHOT_KEY = "source.mode.snapshot"; public static final boolean EXTRACTOR_MODE_SNAPSHOT_DEFAULT_VALUE = false; + public static final String EXTRACTOR_MODE_DOUBLE_LIVING_KEY = "extractor.mode.double-living"; + public static final String SOURCE_MODE_DOUBLE_LIVING_KEY = "source.mode.double-living"; + public static final boolean EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE = false; public static final String EXTRACTOR_START_TIME_KEY = "extractor.start-time"; public static final String SOURCE_START_TIME_KEY = "source.start-time"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java index 963595c62a62..00462cb5eb46 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java @@ -102,15 +102,22 @@ public String getTablePattern() { */ public static TablePattern parsePipePatternFromSourceParameters( final PipeParameters sourceParameters) { - final boolean isTableModelDataAllowedToBeCaptured = + final boolean isDoubleLiving = sourceParameters.getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), - !sourceParameters - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + final boolean isTableModelDataAllowedToBeCaptured = + isDoubleLiving + || sourceParameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), + !sourceParameters + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); final String databaseNamePattern = sourceParameters.getStringOrDefault( Arrays.asList( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java index 7ee8f6dd88db..f03f0a291868 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java @@ -70,15 +70,23 @@ public boolean isRoot() { */ public static TreePattern parsePipePatternFromSourceParameters( final PipeParameters sourceParameters) { - final boolean isTreeModelDataAllowedToBeCaptured = + final boolean isDoubleLiving = sourceParameters.getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), - sourceParameters - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + + final boolean isTreeModelDataAllowedToBeCaptured = + isDoubleLiving + || sourceParameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), + sourceParameters + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); final String path = sourceParameters.getStringByKeys(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java index 2d3a1615f00e..cecee61c7fae 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java @@ -26,8 +26,10 @@ import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import java.util.Arrays; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_EXCLUSION_DEFAULT_VALUE; @@ -85,6 +87,51 @@ public void validate(final PipeParameterValidator validator) throws Exception { .getStringOrDefault( Arrays.asList(EXTRACTOR_EXCLUSION_KEY, SOURCE_EXCLUSION_KEY), EXTRACTOR_EXCLUSION_DEFAULT_VALUE)); + + // Validate double living + validateDoubleLiving(validator.getParameters()); + } + + private void validateDoubleLiving(final PipeParameters parameters) { + final boolean isDoubleLiving = + parameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + if (!isDoubleLiving) { + return; + } + + // check 'capture.tree' + final Boolean isCaptureTree = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY); + if (Objects.nonNull(isCaptureTree) && !isCaptureTree) { + throw new PipeParameterNotValidException( + "capture.tree can not be specified to false when double living is enabled"); + } + + // check 'capture.table' + final Boolean isCaptureTable = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY); + if (Objects.nonNull(isCaptureTable) && !isCaptureTable) { + throw new PipeParameterNotValidException( + "capture.table can not be specified to false when double living is enabled"); + } + + // check 'forwarding-pipe-requests' + final Boolean isForwardingPipeRequests = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, + PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY); + if (Objects.nonNull(isForwardingPipeRequests) && isForwardingPipeRequests) { + throw new PipeParameterNotValidException( + "forwarding-pipe-requests can not be specified to true when double living is enabled"); + } } @Override @@ -99,12 +146,22 @@ public void customize( taskID = pipeName + "_" + regionId + "_" + creationTime; pipeTaskMeta = environment.getPipeTaskMeta(); - isForwardingPipeRequests = + final boolean isDoubleLiving = parameters.getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, - PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), - PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + if (isDoubleLiving) { + isForwardingPipeRequests = false; + } else { + isForwardingPipeRequests = + parameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, + PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); + } } @Override diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 26b1b4260821..ebe673995523 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -751,11 +751,23 @@ struct TAlterPipeReq { 6: optional map extractorAttributes 7: optional bool isReplaceAllExtractorAttributes 8: optional bool ifExistsCondition + 9: optional bool isTableModel +} + +struct TStartPipeReq { + 1: required string pipeName + 2: optional bool isTableModel +} + +struct TStopPipeReq { + 1: required string pipeName + 2: optional bool isTableModel } struct TDropPipeReq { 1: required string pipeName 2: optional bool ifExistsCondition + 3: optional bool isTableModel } // Deprecated, restored for compatibility @@ -768,6 +780,7 @@ struct TPipeSinkInfo { struct TShowPipeReq { 1: optional string pipeName 2: optional bool whereClause + 3: optional bool isTableModel } struct TShowPipeResp { @@ -1680,9 +1693,15 @@ service IConfigNodeRPCService { /** Start Pipe */ common.TSStatus startPipe(string pipeName) + /** Start Pipe */ + common.TSStatus startPipeExtended(TStartPipeReq req) + /** Stop Pipe */ common.TSStatus stopPipe(string pipeName) + /** Stop Pipe */ + common.TSStatus stopPipeExtended(TStopPipeReq req) + /** Drop Pipe */ common.TSStatus dropPipe(string pipeName) From 7dc5179def6fa4f11f596625ab8130ecbf812503 Mon Sep 17 00:00:00 2001 From: Weihao Li <60659567+Wei-hao-Li@users.noreply.github.com> Date: Thu, 9 Jan 2025 21:27:14 +0800 Subject: [PATCH 06/13] Fix select-distinct when there is no order by and select items contains expression --- .../it/query/recent/IoTDBTableAggregationIT.java | 8 ++++++++ .../queryengine/plan/relational/planner/QueryPlanner.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index 7566703ca48e..e2197b0750cc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -4159,6 +4159,14 @@ public void simpleTest() { expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,"}; + tableResultSetEqualTest( + "select distinct s1 < 0 from table1 where s1 is not null", + expectedHeader, + retArray, + DATABASE_NAME); } @Test diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 33c2207fc43a..747288a48c87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -256,7 +256,7 @@ public RelationPlan plan(QuerySpecification node) { } List orderBy = analysis.getOrderByExpressions(node); - if (!orderBy.isEmpty()) { + if (!orderBy.isEmpty() || node.getSelect().isDistinct()) { builder = builder.appendProjections( Iterables.concat(orderBy, outputs), symbolAllocator, queryContext); From e92be6f63ab83d44340cfd8570b5e6b7c4c63fd7 Mon Sep 17 00:00:00 2001 From: Liao Lanyu <1435078631@qq.com> Date: Fri, 10 Jan 2025 10:46:12 +0800 Subject: [PATCH 07/13] [Table Model Subquery] Support uncorrelated in predicate (#14438) --- .../query/recent/IoTDBTableAggregationIT.java | 4 - .../recent/subquery/SubqueryDataSetUtils.java | 12 +- ...oTDBUncorrelatedInPredicateSubqueryIT.java | 319 ++++++++++++++++++ .../IoTDBUncorrelatedScalarSubqueryIT.java | 4 +- .../process/EnforceSingleRowOperator.java | 5 +- .../comparator/JoinKeyComparatorFactory.java | 2 +- .../relational/MergeSortSemiJoinOperator.java | 228 +++++++++++++ .../plan/planner/TableOperatorGenerator.java | 73 +++- .../planner/plan/node/PlanGraphPrinter.java | 12 + .../plan/planner/plan/node/PlanNodeType.java | 4 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../analyzer/ExpressionAnalyzer.java | 31 +- .../relational/planner/IrTypeAnalyzer.java | 11 + .../relational/planner/SubqueryPlanner.java | 33 +- .../TableDistributedPlanGenerator.java | 15 + .../iterative/rule/PruneApplyColumns.java | 138 ++++++++ .../iterative/rule/PruneApplyCorrelation.java | 70 ++++ .../rule/PruneApplySourceColumns.java | 95 ++++++ .../RemoveUnreferencedScalarApplyNodes.java | 42 +++ .../RemoveUnreferencedScalarSubqueries.java | 70 ++++ ...TransformFilteringSemiJoinToInnerJoin.java | 150 ++++++++ ...rrelatedInPredicateSubqueryToSemiJoin.java | 95 ++++++ .../relational/planner/node/Patterns.java | 7 +- .../relational/planner/node/SemiJoinNode.java | 181 ++++++++++ .../optimizations/LogicalOptimizeFactory.java | 18 +- .../PushPredicateIntoTableScan.java | 204 +++++++++++ .../UnaliasSymbolReferences.java | 29 ++ .../relational/sql/parser/AstBuilder.java | 5 +- .../plan/relational/planner/SubqueryTest.java | 98 ++++++ .../planner/assertions/PlanMatchPattern.java | 11 + .../planner/assertions/SemiJoinMatcher.java | 79 +++++ 31 files changed, 2004 insertions(+), 46 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java rename integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/{ => uncorrelated}/IoTDBUncorrelatedScalarSubqueryIT.java (99%) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index e2197b0750cc..21c53f12d07c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -4051,10 +4051,6 @@ public void modeTest() { @Test public void exceptionTest() { - tableAssertTestFail( - "select s1 from table1 where s2 in (select s2 from table1)", - "Only TableSubquery is supported now", - DATABASE_NAME); tableAssertTestFail( "select avg() from table1", "701: Aggregate functions [avg] should only have one argument", diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java index 466b0535b52f..0e6e2124f328 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java @@ -20,9 +20,9 @@ package org.apache.iotdb.relational.it.query.recent.subquery; public class SubqueryDataSetUtils { - protected static final String DATABASE_NAME = "subqueryTest"; - protected static final String[] NUMERIC_MEASUREMENTS = new String[] {"s1", "s2", "s3", "s4"}; - protected static final String[] CREATE_SQLS = + public static final String DATABASE_NAME = "subqueryTest"; + public static final String[] NUMERIC_MEASUREMENTS = new String[] {"s1", "s2", "s3", "s4"}; + public static final String[] CREATE_SQLS = new String[] { "CREATE DATABASE " + DATABASE_NAME, "USE " + DATABASE_NAME, @@ -104,6 +104,12 @@ public class SubqueryDataSetUtils { + " values(4, 'd1', 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", + // table3 + "CREATE TABLE table3(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d01',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d01',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d_null',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table3(time,device_id,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d_null',40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", "FLUSH", "CLEAR ATTRIBUTE CACHE", }; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java new file mode 100644 index 000000000000..ad26fb555909 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedInPredicateSubqueryIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testInPredicateSubqueryInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s in (subquery) + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s not in (subquery) + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery returns empty set + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s not in (subquery), subquery returns empty set. Should return all rows + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains scalar subquery. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT min(%s) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains scalar subquery. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01' and (%s) > (select avg(%s) from table3 where device_id = 'd01'))"; + retArray = new String[] {"40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains expression. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s AS INT32) in (SELECT cast(((%s) + 30) AS INT32) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), s contains null value + sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 in (SELECT s1 from table3)"; + expectedHeader = new String[] {"s1"}; + retArray = new String[] {"30,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: where s not in (subquery), s contains null value, the resutl should be empty + sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 not in (SELECT s1 from table3)"; + expectedHeader = new String[] {"s1"}; + retArray = new String[] {}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testInPredicateSubqueryInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: having s in (subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s not in (subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 30 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01') and count(*) > 3"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s in (subquery), subquery returns empty set + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd010')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s not in (subquery), subquery returns empty set, should return all rows + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd11') and count(*) > 3"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testInPredicateSubqueryInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: select s in (subquery) + sql = + "SELECT %s in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s not in (subquery) + sql = + "SELECT %s not in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "false,", "true,", "true,", "true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s in (subquery), s contains null value. The result should also be null when + // s is null. + sql = "SELECT s1 in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: select s in (subquery), s contains null value. The result should also be null when + // s is null. + sql = "SELECT s1 not in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: select s in (subquery), s contains null value. 36 not in(30, 40, null) returns + // null + sql = "SELECT s1 not in (SELECT s1 from table3) from table1 where device_id = 'd02'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "false,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testInPredicateSubqueryLegalityCheck() { + // Legality check: Multiple parentheses around subquery, this behaves the same as Trino + tableAssertTestFail( + "select s1 from table1 where device_id = 'd01' and s1 not in ((select s1 from table3 where device_id = 'd01'))", + "701: Scalar sub-query has returned multiple rows.", + DATABASE_NAME); + + // Legality check: Join key type mismatch.(left key is int and right key is double) + tableAssertTestFail( + "select s1 from table1 where device_id = 'd01' and s1 in (select s1 + 30.0 from table3 where device_id = 'd01')", + "701: Join key type mismatch", + DATABASE_NAME); + + // Legality check: Row Type is not supported for now. + tableAssertTestFail( + "select s1, s2 in (select (s1, s2) from table1) from table1", + "701: Subquery must return only one column for now. Row Type is not supported for now.", + DATABASE_NAME); + + // Legality check: Row Type is not supported for now. + tableAssertTestFail( + "select (s1, s2) in (select (s1, s2) from table1) from table1", + "701: Subquery must return only one column for now. Row Type is not supported for now.", + DATABASE_NAME); + + // Legality check: subquery can not be parsed(without parentheses) + tableAssertTestFail( + "select s1 from table1 where s1 in select s1 from table1", + "mismatched input", + DATABASE_NAME); + + // Legality check: subquery can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 in (select s1 from)", "mismatched input", DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedScalarSubqueryIT.java rename to integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java index 1d93e36197e8..d2bce2e0d4ae 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.relational.it.query.recent.subquery; +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -262,7 +262,7 @@ public void testScalarSubqueryAfterComparisonLegalityCheck() { // Legality check: subquery returns multiple rows (should fail) tableAssertTestFail( "select s1 from table1 where s1 = (select s1 from table1)", - "301: Scalar sub-query has returned multiple rows.", + "701: Scalar sub-query has returned multiple rows.", DATABASE_NAME); // Legality check: subquery can not be parsed diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java index e9f6fba26f68..bbc9b2c155ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.process; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; @@ -59,7 +60,7 @@ public TsBlock next() throws Exception { return tsBlock; } if (tsBlock.getPositionCount() > 1 || finished) { - throw new IllegalStateException(MULTIPLE_ROWS_ERROR_MESSAGE); + throw new SemanticException(MULTIPLE_ROWS_ERROR_MESSAGE); } finished = true; return tsBlock; @@ -83,7 +84,7 @@ public boolean isFinished() throws Exception { if (childFinished && !finished) { // finished == false means the child has no result returned up to now, but we need at least // one result. - throw new IllegalStateException(NO_RESULT_ERROR_MESSAGE); + throw new SemanticException(NO_RESULT_ERROR_MESSAGE); } // Even if finished == true, we can not return true here, we need to call child.next() to check // if child has more data. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java index a64ff35257d0..33620690fed6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java @@ -36,7 +36,7 @@ public static List getComparators( return comparators; } - private static JoinKeyComparator getComparator(Type type, boolean isAscending) { + public static JoinKeyComparator getComparator(Type type, boolean isAscending) { switch (type.getTypeEnum()) { case INT32: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java new file mode 100644 index 000000000000..b21ed578965f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational; + +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator.JoinKeyComparator; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Collections; +import java.util.List; + +public class MergeSortSemiJoinOperator extends AbstractMergeSortJoinOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(MergeSortSemiJoinOperator.class); + + private final int outputColumnNum; + + private boolean rightHasNullValue = false; + + public MergeSortSemiJoinOperator( + OperatorContext operatorContext, + Operator leftChild, + int leftJoinKeyPosition, + int[] leftOutputSymbolIdx, + Operator rightChild, + int rightJoinKeyPosition, + JoinKeyComparator joinKeyComparator, + List dataTypes) { + super( + operatorContext, + leftChild, + new int[] {leftJoinKeyPosition}, + leftOutputSymbolIdx, + rightChild, + new int[] {rightJoinKeyPosition}, + null, + Collections.singletonList(joinKeyComparator), + dataTypes); + outputColumnNum = dataTypes.size(); + } + + @Override + public boolean hasNext() throws Exception { + if (retainedTsBlock != null) { + return true; + } + + return !leftFinished; + } + + @Override + protected boolean prepareInput() throws Exception { + gotCandidateBlocks(); + if (rightFinished) { + return leftBlockNotEmpty(); + } + return leftBlockNotEmpty() && rightBlockNotEmpty() && gotNextRightBlock(); + } + + @Override + protected boolean processFinished() { + if (rightFinished) { + appendAllLeftBlock(); + return true; + } + + // skip all NULL values in right, because NULL value will not match the left value + while (currentRightHasNullValue()) { + rightHasNullValue = true; + if (rightFinishedWithIncIndex()) { + return true; + } + } + // all the join keys in rightTsBlock are less than leftTsBlock, just skip right + if (allRightLessThanLeft()) { + resetRightBlockList(); + return true; + } + + // all the join Keys in leftTsBlock are less than rightTsBlock, just append the left value + if (allLeftLessThanRight()) { + appendAllLeftBlock(); + resetLeftBlock(); + return true; + } + + // continue right < left, until right >= left + while (lessThan( + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPositions, + rightIndex, + leftBlock, + leftJoinKeyPositions, + leftIndex)) { + if (rightFinishedWithIncIndex()) { + return true; + } + } + if (currentRoundNeedStop()) { + return true; + } + + // if current left is null, append null to result + while (currentLeftHasNullValue()) { + appendNullValueToResult(); + if (leftFinishedWithIncIndex()) { + return true; + } + } + + // continue left < right, until left >= right + while (lessThan( + leftBlock, + leftJoinKeyPositions, + leftIndex, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPositions, + rightIndex)) { + appendWhenNotMatch(); + leftIndex++; + if (leftIndex >= leftBlock.getPositionCount()) { + resetLeftBlock(); + return true; + } + } + if (currentRoundNeedStop()) { + return true; + } + + // has right value equals to current left, append to join result, inc leftIndex + return hasMatchedRightValueToProbeLeft() && leftFinishedWithIncIndex(); + } + + @Override + protected boolean hasMatchedRightValueToProbeLeft() { + boolean matches = + equalsTo( + leftBlock, + leftJoinKeyPositions, + leftIndex, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPositions, + rightIndex); + if (matches) { + appendValueToResult(true); + } else { + appendWhenNotMatch(); + } + + return matches; + } + + private void appendWhenNotMatch() { + // current left won't match any right, append left with false SemiJoin result + if (!rightHasNullValue) { + appendValueToResult(false); + } else { + // if right has null value, append null to result. This behaves like MySQL and Trino. + appendNullValueToResult(); + } + } + + private void appendValueToResult(boolean matches) { + appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(outputColumnNum - 1); + columnBuilder.writeBoolean(matches); + resultBuilder.declarePosition(); + } + + private void appendNullValueToResult() { + appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(outputColumnNum - 1); + columnBuilder.appendNull(); + resultBuilder.declarePosition(); + } + + private void appendAllLeftBlock() { + if (rightHasNullValue) { + while (leftBlockNotEmpty()) { + appendNullValueToResult(); + leftIndex++; + } + } else { + while (leftBlockNotEmpty()) { + appendValueToResult(false); + leftIndex++; + } + } + } + + @Override + protected void recordsWhenDataMatches() { + // do nothing + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(leftChild) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(rightChild) + + RamUsageEstimator.sizeOf(leftOutputSymbolIdx) + + RamUsageEstimator.sizeOf(rightOutputSymbolIdx) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + resultBuilder.getRetainedSizeInBytes(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index fbba891ffce8..df229a1598c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator.ITableTimeRangeIterator; import org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator.TableDateBinTimeRangeIterator; @@ -83,6 +84,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.source.relational.LastQueryAggTableScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.MergeSortFullOuterJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.MergeSortInnerJoinOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.MergeSortSemiJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TreeAlignedDeviceViewAggregationScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TreeAlignedDeviceViewScanOperator; @@ -143,6 +145,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; @@ -214,6 +217,7 @@ import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.FIELD; @@ -1419,10 +1423,9 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { Type leftJoinKeyType = context.getTypeProvider().getTableModelType(node.getCriteria().get(i).getLeft()); - checkArgument( - leftJoinKeyType - == context.getTypeProvider().getTableModelType(node.getCriteria().get(i).getRight()), - "Join key type mismatch."); + checkIfJoinKeyTypeMatches( + leftJoinKeyType, + context.getTypeProvider().getTableModelType(node.getCriteria().get(i).getRight())); joinKeyTypes.add(leftJoinKeyType); } @@ -1498,6 +1501,68 @@ private BiFunction buildUpdateLastRowFunction(Type join } } + @Override + public Operator visitSemiJoin(SemiJoinNode node, LocalExecutionPlanContext context) { + List dataTypes = getOutputColumnTypes(node, context.getTypeProvider()); + + Operator leftChild = node.getLeftChild().accept(this, context); + Operator rightChild = node.getRightChild().accept(this, context); + + ImmutableMap sourceColumnNamesMap = + makeLayoutFromOutputSymbols(node.getSource().getOutputSymbols()); + List sourceOutputSymbols = node.getSource().getOutputSymbols(); + int[] sourceOutputSymbolIdx = new int[node.getSource().getOutputSymbols().size()]; + for (int i = 0; i < sourceOutputSymbolIdx.length; i++) { + Integer index = sourceColumnNamesMap.get(sourceOutputSymbols.get(i)); + checkNotNull(index, "Source of SemiJoinNode doesn't contain sourceOutputSymbol."); + sourceOutputSymbolIdx[i] = index; + } + + ImmutableMap filteringSourceColumnNamesMap = + makeLayoutFromOutputSymbols(node.getRightChild().getOutputSymbols()); + + Integer sourceJoinKeyPosition = sourceColumnNamesMap.get(node.getSourceJoinSymbol()); + checkNotNull(sourceJoinKeyPosition, "Source of SemiJoinNode doesn't contain sourceJoinSymbol."); + + Integer filteringSourceJoinKeyPosition = + filteringSourceColumnNamesMap.get(node.getFilteringSourceJoinSymbol()); + checkNotNull( + filteringSourceJoinKeyPosition, + "FilteringSource of SemiJoinNode doesn't contain filteringSourceJoinSymbol."); + + Type sourceJoinKeyType = + context.getTypeProvider().getTableModelType(node.getSourceJoinSymbol()); + checkIfJoinKeyTypeMatches( + sourceJoinKeyType, + context.getTypeProvider().getTableModelType(node.getFilteringSourceJoinSymbol())); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + MergeSortSemiJoinOperator.class.getSimpleName()); + return new MergeSortSemiJoinOperator( + operatorContext, + leftChild, + sourceJoinKeyPosition, + sourceOutputSymbolIdx, + rightChild, + filteringSourceJoinKeyPosition, + JoinKeyComparatorFactory.getComparator(sourceJoinKeyType, true), + dataTypes); + } + + private void checkIfJoinKeyTypeMatches(Type leftJoinKeyType, Type rightJoinKeyType) { + if (leftJoinKeyType != rightJoinKeyType) { + throw new SemanticException( + "Join key type mismatch. Left join key type: " + + leftJoinKeyType + + ", right join key type: " + + rightJoinKeyType); + } + } + @Override public Operator visitEnforceSingleRow( EnforceSingleRowNode node, LocalExecutionPlanContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index 1811d7f46ae8..442d59f8be33 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -72,6 +72,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; @@ -935,6 +936,17 @@ public List visitJoin( return render(node, boxValue, context); } + @Override + public List visitSemiJoin(SemiJoinNode node, GraphContext context) { + List boxValue = new ArrayList<>(); + boxValue.add(String.format("SemiJoin-%s", node.getPlanNodeId().getId())); + boxValue.add(String.format("OutputSymbols: %s", node.getOutputSymbols())); + boxValue.add(String.format("SourceJoinSymbol: %s", node.getSourceJoinSymbol())); + boxValue.add( + String.format("FilteringSourceJoinSymbol: %s", node.getFilteringSourceJoinSymbol())); + return render(node, boxValue, context); + } + private String printRegion(TRegionReplicaSet regionReplicaSet) { return String.format( "Partition: %s", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 9a8e5ed31e64..3e86d189e1ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -122,6 +122,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; @@ -288,6 +289,7 @@ public enum PlanNodeType { AGGREGATION_TREE_DEVICE_VIEW_SCAN_NODE((short) 1022), TREE_ALIGNED_DEVICE_VIEW_SCAN_NODE((short) 1023), TREE_NONALIGNED_DEVICE_VIEW_SCAN_NODE((short) 1024), + TABLE_SEMI_JOIN_NODE((short) 1025), RELATIONAL_INSERT_TABLET((short) 2000), RELATIONAL_INSERT_ROW((short) 2001), @@ -653,6 +655,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return TreeAlignedDeviceViewScanNode.deserialize(buffer); case 1024: return TreeNonAlignedDeviceViewScanNode.deserialize(buffer); + case 1025: + return SemiJoinNode.deserialize(buffer); case 2000: return RelationalInsertTabletNode.deserialize(buffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 348104f95f97..d04c57e77fbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -127,6 +127,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; @@ -754,6 +755,10 @@ public R visitJoin( return visitTwoChildProcess(node, context); } + public R visitSemiJoin(SemiJoinNode node, C context) { + return visitTwoChildProcess(node, context); + } + public R visitGroupReference(GroupReference node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 404c88b2e1e8..e551638b69c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -106,6 +106,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Iterators.getOnlyElement; import static java.lang.String.format; import static java.util.Collections.unmodifiableMap; import static java.util.Collections.unmodifiableSet; @@ -164,6 +165,9 @@ public class ExpressionAnalyzer { private final Map, LabelPrefixedReference> labelDereferences = new LinkedHashMap<>(); + private static final String SUBQUERY_COLUMN_NUM_CHECK = + "Subquery must return only one column for now. Row Type is not supported for now."; + private ExpressionAnalyzer( Metadata metadata, MPPQueryContext context, @@ -928,6 +932,10 @@ public Type visitCast(Cast node, StackableAstVisitorContext context) { @Override protected Type visitInPredicate(InPredicate node, StackableAstVisitorContext context) { Expression value = node.getValue(); + // Attention: remove this check after supporting RowType + if (value instanceof Row) { + throw new SemanticException(SUBQUERY_COLUMN_NUM_CHECK); + } Expression valueList = node.getValueList(); // When an IN-predicate containing a subquery: `x IN (SELECT ...)` is planned, both `value` @@ -996,21 +1004,15 @@ private Type analyzePredicateWithSubquery( Type declaredValueType, SubqueryExpression subquery, StackableAstVisitorContext context) { + // For now, we only support one column in subqueries, we have checked this before. Type valueRowType = declaredValueType; - if (!(declaredValueType instanceof RowType) && !(declaredValueType instanceof UnknownType)) { + /*if (!(declaredValueType instanceof RowType) && !(declaredValueType instanceof UnknownType)) { valueRowType = RowType.anonymous(ImmutableList.of(declaredValueType)); - } + }*/ Type subqueryType = analyzeSubquery(subquery, context); setExpressionType(subquery, subqueryType); - if (subqueryType.equals(valueRowType)) { - throw new SemanticException( - String.format( - "Value expression and result of subquery must be of the same type: %s vs %s", - valueRowType, subqueryType)); - } - Optional valueCoercion = Optional.empty(); // if (!valueRowType.equals(commonType.get())) { // valueCoercion = commonType; @@ -1047,8 +1049,17 @@ private Type analyzeSubquery( } } + List fieldList = fields.build(); + + // Attention: remove this check after supporting RowType + if (fieldList.size() != 1 || fieldList.get(0).getType() instanceof RowType) { + throw new SemanticException(SUBQUERY_COLUMN_NUM_CHECK); + } + sourceFields.addAll(queryScope.getRelationType().getVisibleFields()); - return RowType.from(fields.build()); + // return RowType.from(fields.build()); + // For now, we only support one column in subqueries, we have checked this before. + return getOnlyElement(fields.build().stream().iterator()).getType(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java index cdf01015f40e..1a19f40132f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java @@ -54,6 +54,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; @@ -63,6 +64,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.DateType; +import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.TimestampType; import org.apache.tsfile.read.common.type.Type; @@ -75,6 +77,7 @@ import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; @@ -416,6 +419,14 @@ protected Type visitInPredicate(InPredicate node, Context context) { return setExpressionType(node, BOOLEAN); } + @Override + protected Type visitRow(Row node, Context context) { + List types = + node.getItems().stream().map(child -> process(child, context)).collect(toImmutableList()); + + return setExpressionType(node, RowType.anonymous(types)); + } + @Override protected Type visitLikePredicate(LikePredicate node, Context context) { process(node.getValue(), context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 184fc0739436..17422f80247b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -45,7 +45,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import com.google.common.collect.ImmutableList; @@ -72,7 +71,6 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.ScopeAware.scopeAwareKey; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression.Quantifier.ALL; -import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toSqlType; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; class SubqueryPlanner { @@ -259,6 +257,11 @@ private PlanBuilder planScalarSubquery(PlanBuilder subPlan, Cluster fieldMappings = relationPlan.getFieldMappings(); Symbol column; + // Attention: remove this check after supporting RowType + checkArgument( + descriptor.getVisibleFieldCount() <= 1, + "For now, only single column subqueries are supported"); + /* if (descriptor.getVisibleFieldCount() > 1) { column = symbolAllocator.newSymbol("row", type); @@ -274,8 +277,9 @@ private PlanBuilder planScalarSubquery(PlanBuilder subPlan, Cluster fieldsList = fields.build(); + // Attention: remove this check after supporting RowType + checkArgument(fieldsList.size() == 1, "For now, only single column subqueries are supported."); + /*subqueryPlan = + subqueryPlan.withNewRoot( + new ProjectNode( + idAllocator.genPlanNodeId(), + relationPlan.getRoot(), + Assignments.of(column, new Cast(new Row(fields.build()), toSqlType(type)))));*/ + subqueryPlan = subqueryPlan.withNewRoot( new ProjectNode( idAllocator.genPlanNodeId(), relationPlan.getRoot(), - Assignments.of( - column, - new Cast( - new Row(fields.build()), - new GenericDataType( - new Identifier(type.toString()), ImmutableList.of()))))); + Assignments.of(column, fieldsList.get(0)))); return coerceIfNecessary(subqueryPlan, column, subquery, coercion); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 53d6516e2e3f..d1fcd56799fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -57,6 +57,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; @@ -464,6 +465,20 @@ public List visitJoin(JoinNode node, PlanContext context) { } @Override + public List visitSemiJoin(SemiJoinNode node, PlanContext context) { + List leftChildrenNodes = node.getLeftChild().accept(this, context); + List rightChildrenNodes = node.getRightChild().accept(this, context); + checkArgument( + leftChildrenNodes.size() == 1, + "The size of left children node of SemiJoinNode should be 1"); + checkArgument( + rightChildrenNodes.size() == 1, + "The size of right children node of SemiJoinNode should be 1"); + node.setLeftChild(leftChildrenNodes.get(0)); + node.setRightChild(rightChildrenNodes.get(0)); + return Collections.singletonList(node); + } + public List visitDeviceTableScan( final DeviceTableScanNode node, final PlanContext context) { final Map tableScanNodeMap = new HashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java new file mode 100644 index 000000000000..11dcaefba17a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.Sets.intersection; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +/** + * This rule restricts the outputs of ApplyNode's input and subquery based on which ApplyNode's + * output symbols are referenced. + * + *

A symbol from input source can be pruned, when - it is not a referenced output symbol - it is + * not a correlation symbol - it is not referenced in subqueryAssignments + * + *

A symbol from subquery source can be pruned, when it is not referenced in subqueryAssignments. + * + *

A subquery assignment can be removed, when its key is not a referenced output symbol. + * + *

Note: this rule does not remove any symbols from the correlation list. This is responsibility + * of PruneApplyCorrelation rule. + * + *

Transforms: + * + * 

+ * - Project (i1, r1)
+ *      - Apply
+ *          correlation: [corr]
+ *          assignments:
+ *              r1 -> a in s1,
+ *              r2 -> b in s2,
+ *          - Input (a, b, corr)
+ *          - Subquery (s1, s2)
+ *
+ * + * Into: + * + * 
+ * - Project (i1, r1)
+ *      - Apply
+ *          correlation: [corr]
+ *          assignments:
+ *              r1 -> a in s1,
+ *          - Project (a, corr)
+ *              - Input (a, b, corr)
+ *          - Project (s1)
+ *              - Subquery (s1, s2)
+ *
+ */ +public class PruneApplyColumns extends ProjectOffPushDownRule { + public PruneApplyColumns() { + super(applyNode()); + } + + @Override + protected Optional pushDownProjectOff( + Context context, ApplyNode applyNode, Set referencedOutputs) { + // remove unused apply node + if (intersection(applyNode.getSubqueryAssignments().keySet(), referencedOutputs).isEmpty()) { + return Optional.of(applyNode.getInput()); + } + + // extract referenced assignments + ImmutableSet.Builder requiredAssignmentsSymbols = ImmutableSet.builder(); + ImmutableMap.Builder newSubqueryAssignments = + ImmutableMap.builder(); + for (Map.Entry entry : + applyNode.getSubqueryAssignments().entrySet()) { + if (referencedOutputs.contains(entry.getKey())) { + requiredAssignmentsSymbols.addAll(entry.getValue().inputs()); + newSubqueryAssignments.put(entry); + } + } + + // prune subquery symbols + Optional newSubquery = + restrictOutputs( + context.getIdAllocator(), applyNode.getSubquery(), requiredAssignmentsSymbols.build()); + + // prune input symbols + Set requiredInputSymbols = + ImmutableSet.builder() + .addAll(referencedOutputs) + .addAll(applyNode.getCorrelation()) + .addAll(requiredAssignmentsSymbols.build()) + .build(); + + Optional newInput = + restrictOutputs(context.getIdAllocator(), applyNode.getInput(), requiredInputSymbols); + + boolean pruned = + newSubquery.isPresent() + || newInput.isPresent() + || newSubqueryAssignments.buildOrThrow().size() + < applyNode.getSubqueryAssignments().size(); + + if (pruned) { + return Optional.of( + new ApplyNode( + applyNode.getPlanNodeId(), + newInput.orElse(applyNode.getInput()), + newSubquery.orElse(applyNode.getSubquery()), + newSubqueryAssignments.buildOrThrow(), + applyNode.getCorrelation(), + applyNode.getOriginSubquery())); + } + + return Optional.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java new file mode 100644 index 000000000000..942c5ba6e639 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import java.util.List; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +/** + * This rule updates ApplyNode's correlation list. A symbol can be removed from the correlation list + * if it is not referenced by the subquery node. Note: This rule does not restrict ApplyNode's + * children outputs. It requires additional information about context (symbols required by the outer + * plan) and is done in PruneApplyColumns rule. + */ +public class PruneApplyCorrelation implements Rule { + private static final Pattern PATTERN = applyNode(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + Set subquerySymbols = extractUnique(applyNode.getSubquery(), context.getLookup()); + List newCorrelation = + applyNode.getCorrelation().stream() + .filter(subquerySymbols::contains) + .collect(toImmutableList()); + + if (newCorrelation.size() < applyNode.getCorrelation().size()) { + return Result.ofPlanNode( + new ApplyNode( + applyNode.getPlanNodeId(), + applyNode.getInput(), + applyNode.getSubquery(), + applyNode.getSubqueryAssignments(), + newCorrelation, + applyNode.getOriginSubquery())); + } + + return Result.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java new file mode 100644 index 000000000000..746866102d2c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import com.google.common.collect.ImmutableList; + +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +/** + * This rule restricts outputs of ApplyNode's subquery to include only the symbols needed for + * subqueryAssignments. Symbols from the subquery are not produced at ApplyNode's output. They are + * only used for the assignments. Transforms: + * + * 
+ * - Apply
+ *      correlation: [corr_symbol]
+ *      assignments:
+ *          result_1 -> a in subquery_symbol_1,
+ *          result_2 -> b > ALL subquery_symbol_2
+ *    - Input (a, b, corr_symbol)
+ *    - Subquery (subquery_symbol_1, subquery_symbol_2, subquery_symbol_3)
+ *
+ * + * Into: + * + * 
+ * - Apply
+ *      correlation: [corr_symbol]
+ *      assignments:
+ *          result_1 -> a in subquery_symbol_1,
+ *          result_2 -> b > ALL subquery_symbol_2
+ *    - Input (a, b, corr_symbol)
+ *    - Project
+ *          subquery_symbol_1 -> subquery_symbol_1
+ *          subquery_symbol_2 -> subquery_symbol_2
+ *        - Subquery (subquery_symbol_1, subquery_symbol_2, subquery_symbol_3)
+ *
+ * + * Note: ApplyNode's input symbols are produced on ApplyNode's output. They cannot be pruned without + * outer context. + */ +public class PruneApplySourceColumns implements Rule { + private static final Pattern PATTERN = applyNode(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + Set subqueryAssignmentsSymbols = + applyNode.getSubqueryAssignments().values().stream() + .flatMap(expression -> expression.inputs().stream()) + .collect(toImmutableSet()); + + Optional prunedSubquery = + restrictOutputs( + context.getIdAllocator(), applyNode.getSubquery(), subqueryAssignmentsSymbols); + return prunedSubquery + .map( + subquery -> applyNode.replaceChildren(ImmutableList.of(applyNode.getInput(), subquery))) + .map(Result::ofPlanNode) + .orElse(Result.empty()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java new file mode 100644 index 000000000000..fcecc2c3b821 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +public class RemoveUnreferencedScalarApplyNodes implements Rule { + private static final Pattern PATTERN = + applyNode().matching(applyNode -> applyNode.getSubqueryAssignments().isEmpty()); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + return Result.ofPlanNode(applyNode.getInput()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java new file mode 100644 index 000000000000..badc7dff3c49 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.INNER; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.RIGHT; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.CorrelatedJoin.filter; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.correlatedJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isAtLeastScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; + +public class RemoveUnreferencedScalarSubqueries implements Rule { + private static final Pattern PATTERN = + correlatedJoin().with(filter().equalTo(TRUE_LITERAL)); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(CorrelatedJoinNode correlatedJoinNode, Captures captures, Context context) { + PlanNode input = correlatedJoinNode.getInput(); + PlanNode subquery = correlatedJoinNode.getSubquery(); + + if (isUnreferencedScalar(input, context.getLookup()) + && correlatedJoinNode.getCorrelation().isEmpty()) { + if (correlatedJoinNode.getJoinType() == INNER + || correlatedJoinNode.getJoinType() == RIGHT + || isAtLeastScalar(subquery, context.getLookup())) { + return Result.ofPlanNode(subquery); + } + } + + if (isUnreferencedScalar(subquery, context.getLookup())) { + return Result.ofPlanNode(input); + } + + return Result.empty(); + } + + private boolean isUnreferencedScalar(PlanNode planNode, Lookup lookup) { + return planNode.getOutputSymbols().isEmpty() && isScalar(planNode, lookup); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java new file mode 100644 index 000000000000..67f814af6616 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ExpressionSymbolInliner.inlineSymbols; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils.and; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils.extractConjuncts; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.singleAggregation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.singleGroupingSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.INNER; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.filter; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.semiJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.source; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture.newCapture; + +/** + * Rewrite filtering semi-join to inner join. + * + *

Transforms: + * + * 

+ * - Filter (semiJoinSymbol AND predicate)
+ *    - SemiJoin (semiJoinSymbol <- (a IN b))
+ *        source: plan A producing symbol a
+ *        filtering source: plan B producing symbol b
+ *
+ * + *

Into: + * + * 

+ * - Project (semiJoinSymbol <- TRUE)
+ *    - Join INNER on (a = b), joinFilter (predicate with semiJoinSymbol replaced with TRUE)
+ *       - source
+ *       - Aggregation distinct(b)
+ *          - filtering source
+ *
+ */ +public class TransformFilteringSemiJoinToInnerJoin implements Rule { + private static final Capture SEMI_JOIN = newCapture(); + + private static final Pattern PATTERN = + filter().with(source().matching(semiJoin().capturedAs(SEMI_JOIN))); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(FilterNode filterNode, Captures captures, Context context) { + SemiJoinNode semiJoin = captures.get(SEMI_JOIN); + + Symbol semiJoinSymbol = semiJoin.getSemiJoinOutput(); + Predicate isSemiJoinSymbol = + expression -> expression.equals(semiJoinSymbol.toSymbolReference()); + + List conjuncts = extractConjuncts(filterNode.getPredicate()); + if (conjuncts.stream().noneMatch(isSemiJoinSymbol)) { + return Result.empty(); + } + Expression filteredPredicate = + and(conjuncts.stream().filter(isSemiJoinSymbol.negate()).collect(toImmutableList())); + + Expression simplifiedPredicate = + inlineSymbols( + symbol -> { + if (symbol.equals(semiJoinSymbol)) { + return TRUE_LITERAL; + } + return symbol.toSymbolReference(); + }, + filteredPredicate); + + Optional joinFilter = + simplifiedPredicate.equals(TRUE_LITERAL) + ? Optional.empty() + : Optional.of(simplifiedPredicate); + + PlanNode filteringSourceDistinct = + singleAggregation( + context.getIdAllocator().genPlanNodeId(), + semiJoin.getFilteringSource(), + ImmutableMap.of(), + singleGroupingSet(ImmutableList.of(semiJoin.getFilteringSourceJoinSymbol()))); + + JoinNode innerJoin = + new JoinNode( + semiJoin.getPlanNodeId(), + INNER, + semiJoin.getSource(), + filteringSourceDistinct, + ImmutableList.of( + new JoinNode.EquiJoinClause( + semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), + semiJoin.getSource().getOutputSymbols(), + ImmutableList.of(), + joinFilter, + Optional.empty()); + + ProjectNode project = + new ProjectNode( + context.getIdAllocator().genPlanNodeId(), + innerJoin, + Assignments.builder() + .putIdentities(innerJoin.getOutputSymbols()) + .put(semiJoinSymbol, TRUE_LITERAL) + .build()); + + return Result.ofPlanNode(project); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java new file mode 100644 index 000000000000..80fe79c7e5f4 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.Apply.correlation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern.empty; + +/** + * This optimizers looks for InPredicate expressions in ApplyNodes and replaces the nodes with + * SemiJoin nodes. + * + *

Plan before optimizer: + * + * 

+ * Filter(a IN b):
+ *   Apply
+ *     - correlation: []  // empty
+ *     - input: some plan A producing symbol a
+ *     - subquery: some plan B producing symbol b
+ *
+ * + *

Plan after optimizer: + * + * 

+ * Filter(semijoinresult):
+ *   SemiJoin
+ *     - source: plan A
+ *     - filteringSource: B
+ *     - sourceJoinSymbol: symbol a
+ *     - filteringSourceJoinSymbol: symbol b
+ *     - semiJoinOutput: semijoinresult
+ *
+ */ +public class TransformUncorrelatedInPredicateSubqueryToSemiJoin implements Rule { + private static final Pattern PATTERN = applyNode().with(empty(correlation())); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + if (applyNode.getSubqueryAssignments().size() != 1) { + return Result.empty(); + } + + ApplyNode.SetExpression expression = + getOnlyElement(applyNode.getSubqueryAssignments().values()); + if (!(expression instanceof ApplyNode.In)) { + return Result.empty(); + } + + ApplyNode.In inPredicate = (ApplyNode.In) expression; + + Symbol semiJoinSymbol = getOnlyElement(applyNode.getSubqueryAssignments().keySet()); + + SemiJoinNode replacement = + new SemiJoinNode( + context.getIdAllocator().genPlanNodeId(), + applyNode.getInput(), + applyNode.getSubquery(), + inPredicate.getValue(), + inPredicate.getReference(), + semiJoinSymbol); + + return Result.ofPlanNode(replacement); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java index f0056e024971..604e2b06a3a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java @@ -143,10 +143,9 @@ public static Pattern project() { return typeOf(SampleNode.class); }*/ - // public static Pattern semiJoin() - // { - // return typeOf(SemiJoinNode.class); - // } + public static Pattern semiJoin() { + return typeOf(SemiJoinNode.class); + } public static Pattern gapFill() { return typeOf(GapFillNode.class); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java new file mode 100644 index 000000000000..bb8b85752d59 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TwoChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; + +import com.google.common.collect.ImmutableList; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class SemiJoinNode extends TwoChildProcessNode { + private final Symbol sourceJoinSymbol; + private final Symbol filteringSourceJoinSymbol; + private final Symbol semiJoinOutput; + + public SemiJoinNode( + PlanNodeId id, + PlanNode source, + PlanNode filteringSource, + Symbol sourceJoinSymbol, + Symbol filteringSourceJoinSymbol, + Symbol semiJoinOutput) { + super(id, source, filteringSource); + this.sourceJoinSymbol = requireNonNull(sourceJoinSymbol, "sourceJoinSymbol is null"); + this.filteringSourceJoinSymbol = + requireNonNull(filteringSourceJoinSymbol, "filteringSourceJoinSymbol is null"); + this.semiJoinOutput = requireNonNull(semiJoinOutput, "semiJoinOutput is null"); + + if (source != null) { + checkArgument( + source.getOutputSymbols().contains(sourceJoinSymbol), + "Source does not contain join symbol"); + } + + if (filteringSource != null) { + checkArgument( + filteringSource.getOutputSymbols().contains(filteringSourceJoinSymbol), + "Filtering source does not contain filtering join symbol"); + } + } + + public PlanNode getSource() { + return leftChild; + } + + public PlanNode getFilteringSource() { + return rightChild; + } + + public Symbol getSourceJoinSymbol() { + return sourceJoinSymbol; + } + + public Symbol getFilteringSourceJoinSymbol() { + return filteringSourceJoinSymbol; + } + + public Symbol getSemiJoinOutput() { + return semiJoinOutput; + } + + @Override + public List getOutputSymbols() { + return ImmutableList.builder() + .addAll(leftChild.getOutputSymbols()) + .add(semiJoinOutput) + .build(); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitSemiJoin(this, context); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); + return new SemiJoinNode( + getPlanNodeId(), + newChildren.get(0), + newChildren.get(1), + sourceJoinSymbol, + filteringSourceJoinSymbol, + semiJoinOutput); + } + + @Override + public PlanNode clone() { + // clone without children + return new SemiJoinNode( + getPlanNodeId(), null, null, sourceJoinSymbol, filteringSourceJoinSymbol, semiJoinOutput); + } + + @Override + public List getOutputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || !this.getClass().equals(obj.getClass())) { + return false; + } + + if (!super.equals(obj)) { + return false; + } + + SemiJoinNode other = (SemiJoinNode) obj; + + return Objects.equals(this.sourceJoinSymbol, other.sourceJoinSymbol) + && Objects.equals(this.filteringSourceJoinSymbol, other.filteringSourceJoinSymbol) + && Objects.equals(this.semiJoinOutput, other.semiJoinOutput); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), sourceJoinSymbol, filteringSourceJoinSymbol, semiJoinOutput); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_SEMI_JOIN_NODE.serialize(byteBuffer); + + Symbol.serialize(sourceJoinSymbol, byteBuffer); + Symbol.serialize(filteringSourceJoinSymbol, byteBuffer); + Symbol.serialize(semiJoinOutput, byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_SEMI_JOIN_NODE.serialize(stream); + + Symbol.serialize(sourceJoinSymbol, stream); + Symbol.serialize(filteringSourceJoinSymbol, stream); + Symbol.serialize(semiJoinOutput, stream); + } + + public static SemiJoinNode deserialize(ByteBuffer byteBuffer) { + Symbol sourceJoinSymbol = Symbol.deserialize(byteBuffer); + Symbol filteringSourceJoinSymbol = Symbol.deserialize(byteBuffer); + Symbol semiJoinOutput = Symbol.deserialize(byteBuffer); + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new SemiJoinNode( + planNodeId, null, null, sourceJoinSymbol, filteringSourceJoinSymbol, semiJoinOutput); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index d52c23cad31c..62bdd1bf1b9d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -32,6 +32,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.MergeLimits; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationSourceColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneApplyColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneApplyCorrelation; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneApplySourceColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinCorrelation; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneDistinctAggregation; @@ -54,7 +57,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantEnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantIdentityProjections; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveTrivialFilters; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveUnreferencedScalarSubqueries; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.SimplifyExpressions; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedSubqueryToJoin; import com.google.common.collect.ImmutableList; @@ -78,6 +83,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { // TODO After ValuesNode introduced // new RemoveEmptyGlobalAggregation(), new PruneAggregationSourceColumns(), + new PruneApplyColumns(), + new PruneApplyCorrelation(), + new PruneApplySourceColumns(), new PruneCorrelatedJoinColumns(), new PruneCorrelatedJoinCorrelation(), new PruneEnforceSingleRowColumns(), @@ -204,13 +212,19 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { plannerContext, ruleStats, ImmutableSet.of( - new RemoveRedundantEnforceSingleRowNode(), - new TransformUncorrelatedSubqueryToJoin())), + new RemoveRedundantEnforceSingleRowNode(), new RemoveUnreferencedScalarSubqueries(), + new TransformUncorrelatedSubqueryToJoin(), + new TransformUncorrelatedInPredicateSubqueryToSemiJoin())), new CheckSubqueryNodesAreRewritten(), new IterativeOptimizer( plannerContext, ruleStats, ImmutableSet.of(new PruneDistinctAggregation())), simplifyOptimizer, new PushPredicateIntoTableScan(), + // Currently, Distinct is not supported, so we cant use this rule for now. + // new IterativeOptimizer( + // plannerContext, + // ruleStats, + // ImmutableSet.of(new TransformFilteringSemiJoinToInnerJoin())), // redo columnPrune and inlineProjections after pushPredicateIntoTableScan columnPruningOptimizer, inlineProjectionLimitFiltersOptimizer, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 79da48a8cce6..7cf45ff3cbd3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -51,6 +51,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; @@ -61,6 +62,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; @@ -73,6 +75,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -87,6 +90,7 @@ import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.SCHEMA_FETCHER; import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.TABLE_TYPE; import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.getTimePartitionSlotList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_FIRST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_LAST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.DeterminismEvaluator.isDeterministic; @@ -102,6 +106,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.processInnerJoin; import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.extractCardinality; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.EQUAL; /** * Optimization phase: Logical plan planning. @@ -751,6 +756,205 @@ private void appendSortNodeForMergeSortJoin(JoinNode joinNode) { joinNode.setRightChild(rightSortNode); } + @Override + public PlanNode visitSemiJoin(SemiJoinNode node, RewriteContext context) { + Expression inheritedPredicate = + context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; + if (!extractConjuncts(inheritedPredicate) + .contains(node.getSemiJoinOutput().toSymbolReference())) { + return visitNonFilteringSemiJoin(node, context); + } + return visitFilteringSemiJoin(node, context); + } + + private PlanNode visitNonFilteringSemiJoin(SemiJoinNode node, RewriteContext context) { + Expression inheritedPredicate = + context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; + + List sourceConjuncts = new ArrayList<>(); + List postJoinConjuncts = new ArrayList<>(); + + // TODO: see if there are predicates that can be inferred from the semi join output + PlanNode rewrittenFilteringSource = + node.getFilteringSource().accept(this, new RewriteContext()); + + // Push inheritedPredicates down to the source if they don't involve the semi join output + ImmutableSet sourceScope = ImmutableSet.copyOf(node.getSource().getOutputSymbols()); + EqualityInference inheritedInference = new EqualityInference(metadata, inheritedPredicate); + EqualityInference.nonInferrableConjuncts(metadata, inheritedPredicate) + .forEach( + conjunct -> { + Expression rewrittenConjunct = inheritedInference.rewrite(conjunct, sourceScope); + // Since each source row is reflected exactly once in the output, ok to push + // non-deterministic predicates down + if (rewrittenConjunct != null) { + sourceConjuncts.add(rewrittenConjunct); + } else { + postJoinConjuncts.add(conjunct); + } + }); + + // Add the inherited equality predicates back in + EqualityInference.EqualityPartition equalityPartition = + inheritedInference.generateEqualitiesPartitionedBy(sourceScope); + sourceConjuncts.addAll(equalityPartition.getScopeEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeComplementEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeStraddlingEqualities()); + + PlanNode rewrittenSource = + node.getSource().accept(this, new RewriteContext(combineConjuncts(sourceConjuncts))); + + PlanNode output = appendSortNodeForSemiJoin(node, rewrittenSource, rewrittenFilteringSource); + + if (!postJoinConjuncts.isEmpty()) { + output = + new FilterNode(queryId.genPlanNodeId(), output, combineConjuncts(postJoinConjuncts)); + } + return output; + } + + private SemiJoinNode appendSortNodeForSemiJoin( + SemiJoinNode node, PlanNode rewrittenSource, PlanNode rewrittenFilteringSource) { + OrderingScheme sourceOrderingScheme = + new OrderingScheme( + ImmutableList.of(node.getSourceJoinSymbol()), + ImmutableMap.of(node.getSourceJoinSymbol(), ASC_NULLS_LAST)); + OrderingScheme filteringSourceOrderingScheme = + new OrderingScheme( + ImmutableList.of(node.getFilteringSourceJoinSymbol()), + // NULL first is used to make sure that we can know if there's null value in the + // result set of right table. + // For x in (subquery), if subquery returns null and some value a,b, and x is not + // in(a,b), the result of SemiJoinOutput should be NULL. + ImmutableMap.of(node.getFilteringSourceJoinSymbol(), ASC_NULLS_FIRST)); + SortNode sourceSortNode = + new SortNode( + queryId.genPlanNodeId(), rewrittenSource, sourceOrderingScheme, false, false); + SortNode filteringSourceSortNode = + new SortNode( + queryId.genPlanNodeId(), + rewrittenFilteringSource, + filteringSourceOrderingScheme, + false, + false); + return new SemiJoinNode( + node.getPlanNodeId(), + sourceSortNode, + filteringSourceSortNode, + node.getSourceJoinSymbol(), + node.getFilteringSourceJoinSymbol(), + node.getSemiJoinOutput()); + } + + private PlanNode visitFilteringSemiJoin(SemiJoinNode node, RewriteContext context) { + Expression inheritedPredicate = + context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; + Expression deterministicInheritedPredicate = filterDeterministicConjuncts(inheritedPredicate); + Expression sourceEffectivePredicate = TRUE_LITERAL; + Expression filteringSourceEffectivePredicate = TRUE_LITERAL; + // Expression sourceEffectivePredicate = + // filterDeterministicConjuncts(effectivePredicateExtractor.extract(session, node.getSource(), + // types, typeAnalyzer)); + // Expression filteringSourceEffectivePredicate = filterDeterministicConjuncts(metadata, + // effectivePredicateExtractor.extract(session, node.getFilteringSource(), types, + // typeAnalyzer)); + Expression joinExpression = + new ComparisonExpression( + EQUAL, + node.getSourceJoinSymbol().toSymbolReference(), + node.getFilteringSourceJoinSymbol().toSymbolReference()); + + List sourceSymbols = node.getSource().getOutputSymbols(); + List filteringSourceSymbols = node.getFilteringSource().getOutputSymbols(); + + List sourceConjuncts = new ArrayList<>(); + List filteringSourceConjuncts = new ArrayList<>(); + List postJoinConjuncts = new ArrayList<>(); + + // Generate equality inferences + EqualityInference allInference = + new EqualityInference( + metadata, + deterministicInheritedPredicate, + sourceEffectivePredicate, + filteringSourceEffectivePredicate, + joinExpression); + EqualityInference allInferenceWithoutSourceInferred = + new EqualityInference( + metadata, + deterministicInheritedPredicate, + filteringSourceEffectivePredicate, + joinExpression); + EqualityInference allInferenceWithoutFilteringSourceInferred = + new EqualityInference( + metadata, deterministicInheritedPredicate, sourceEffectivePredicate, joinExpression); + + // Push inheritedPredicates down to the source if they don't involve the semi join output + Set sourceScope = ImmutableSet.copyOf(sourceSymbols); + EqualityInference.nonInferrableConjuncts(metadata, inheritedPredicate) + .forEach( + conjunct -> { + Expression rewrittenConjunct = allInference.rewrite(conjunct, sourceScope); + // Since each source row is reflected exactly once in the output, ok to push + // non-deterministic predicates down + if (rewrittenConjunct != null) { + sourceConjuncts.add(rewrittenConjunct); + } else { + postJoinConjuncts.add(conjunct); + } + }); + + // Push inheritedPredicates down to the filtering source if possible + Set filterScope = ImmutableSet.copyOf(filteringSourceSymbols); + EqualityInference.nonInferrableConjuncts(metadata, deterministicInheritedPredicate) + .forEach( + conjunct -> { + Expression rewrittenConjunct = allInference.rewrite(conjunct, filterScope); + // We cannot push non-deterministic predicates to filtering side. Each filtering + // side row have to be + // logically reevaluated for each source row. + if (rewrittenConjunct != null) { + filteringSourceConjuncts.add(rewrittenConjunct); + } + }); + + // move effective predicate conjuncts source <-> filter + // See if we can push the filtering source effective predicate to the source side + EqualityInference.nonInferrableConjuncts(metadata, filteringSourceEffectivePredicate) + .map(conjunct -> allInference.rewrite(conjunct, sourceScope)) + .filter(Objects::nonNull) + .forEach(sourceConjuncts::add); + + // See if we can push the source effective predicate to the filtering source side + EqualityInference.nonInferrableConjuncts(metadata, sourceEffectivePredicate) + .map(conjunct -> allInference.rewrite(conjunct, filterScope)) + .filter(Objects::nonNull) + .forEach(filteringSourceConjuncts::add); + + // Add equalities from the inference back in + sourceConjuncts.addAll( + allInferenceWithoutSourceInferred + .generateEqualitiesPartitionedBy(sourceScope) + .getScopeEqualities()); + filteringSourceConjuncts.addAll( + allInferenceWithoutFilteringSourceInferred + .generateEqualitiesPartitionedBy(filterScope) + .getScopeEqualities()); + + PlanNode rewrittenSource = + node.getSource().accept(this, new RewriteContext(combineConjuncts(sourceConjuncts))); + PlanNode rewrittenFilteringSource = + node.getFilteringSource() + .accept(this, new RewriteContext(combineConjuncts(filteringSourceConjuncts))); + + PlanNode output = appendSortNodeForSemiJoin(node, rewrittenSource, rewrittenFilteringSource); + if (!postJoinConjuncts.isEmpty()) { + output = + new FilterNode(queryId.genPlanNodeId(), output, combineConjuncts(postJoinConjuncts)); + } + return output; + } + @Override public PlanNode visitInsertTablet(InsertTabletNode node, RewriteContext context) { return node; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index dda33ea1c2f6..2b4144389837 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; @@ -731,6 +732,34 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { node.isSpillable()), outputMapping); } + + @Override + public PlanAndMappings visitSemiJoin(SemiJoinNode node, UnaliasContext context) { + // it is assumed that symbols are distinct between SemiJoin source and filtering source. Only + // symbols from outer correlation might be the exception + PlanAndMappings rewrittenSource = node.getSource().accept(this, context); + PlanAndMappings rewrittenFilteringSource = node.getFilteringSource().accept(this, context); + + Map outputMapping = new HashMap<>(); + outputMapping.putAll(rewrittenSource.getMappings()); + outputMapping.putAll(rewrittenFilteringSource.getMappings()); + + SymbolMapper mapper = symbolMapper(outputMapping); + + Symbol newSourceJoinSymbol = mapper.map(node.getSourceJoinSymbol()); + Symbol newFilteringSourceJoinSymbol = mapper.map(node.getFilteringSourceJoinSymbol()); + Symbol newSemiJoinOutput = mapper.map(node.getSemiJoinOutput()); + + return new PlanAndMappings( + new SemiJoinNode( + node.getPlanNodeId(), + rewrittenSource.getRoot(), + rewrittenFilteringSource.getRoot(), + newSourceJoinSymbol, + newFilteringSourceJoinSymbol, + newSemiJoinOutput), + outputMapping); + } } private static class UnaliasContext { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 17412241cc64..361956dd49b4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -1871,8 +1871,7 @@ public Node visitInList(RelationalSqlParser.InListContext ctx) { @Override public Node visitInSubquery(RelationalSqlParser.InSubqueryContext ctx) { - throw new SemanticException("Only TableSubquery is supported now"); - /*Expression result = + Expression result = new InPredicate( getLocation(ctx), (Expression) visit(ctx.value), @@ -1882,7 +1881,7 @@ public Node visitInSubquery(RelationalSqlParser.InSubqueryContext ctx) { result = new NotExpression(getLocation(ctx), result); } - return result;*/ + return result; } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java index 0e1c0ad8f999..52321265280b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import com.google.common.collect.ImmutableList; @@ -45,9 +46,12 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.filter; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.join; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.mergeSort; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.semiJoin; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.singleGroupingSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.sort; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.FINAL; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.INTERMEDIATE; @@ -224,4 +228,98 @@ public void testUncorrelatedScalarSubqueryInWhereClauseWithEnforceSingleRowNode( JoinNode.JoinType.INNER, builder -> builder.left(tableScan1).right(enforceSingleRow(any()))))))); } + + @Test + public void testUncorrelatedInPredicateSubquery() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 in (select s1 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + Expression filterPredicate = new SymbolReference("expr"); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + PlanMatchPattern tableScan2 = tableScan("testdb.table1", ImmutableMap.of("s1_6", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──SemiJoinNode + * |──SortNode + * | └──TableScanNode + * ├──SortNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + filter( + filterPredicate, + semiJoin("s1", "s1_6", "expr", sort(tableScan1), sort(tableScan2)))))); + + // Verify DistributionPlan + assertPlan( + planTester.getFragmentPlan(0), + output( + project( + filter( + filterPredicate, + semiJoin( + "s1", + "s1_6", + "expr", + mergeSort(exchange(), sort(tableScan1), exchange()), + mergeSort(exchange(), sort(tableScan2), exchange())))))); + + assertPlan(planTester.getFragmentPlan(1), sort(tableScan1)); + + assertPlan(planTester.getFragmentPlan(2), sort(tableScan1)); + + assertPlan(planTester.getFragmentPlan(3), sort(tableScan2)); + + assertPlan(planTester.getFragmentPlan(4), sort(tableScan2)); + } + + @Test + public void testUncorrelatedNotInPredicateSubquery() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 not in (select s1 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + Expression filterPredicate = new NotExpression(new SymbolReference("expr")); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + PlanMatchPattern tableScan2 = tableScan("testdb.table1", ImmutableMap.of("s1_6", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──SemiJoinNode + * |──SortNode + * | └──TableScanNode + * ├──SortNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + filter( + filterPredicate, + semiJoin("s1", "s1_6", "expr", sort(tableScan1), sort(tableScan2)))))); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 40e6abba5184..5c2ce8e2f806 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; @@ -461,6 +462,16 @@ public static PlanMatchPattern join( return builder.build(); } + public static PlanMatchPattern semiJoin( + String sourceSymbolAlias, + String filteringSymbolAlias, + String outputAlias, + PlanMatchPattern source, + PlanMatchPattern filtering) { + return node(SemiJoinNode.class, source, filtering) + .with(new SemiJoinMatcher(sourceSymbolAlias, filteringSymbolAlias, outputAlias)); + } + public static PlanMatchPattern streamSort(PlanMatchPattern source) { return node(StreamSortNode.class, source); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java new file mode 100644 index 000000000000..7ee9192254e7 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.assertions; + +import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.MatchResult.NO_MATCH; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.MatchResult.match; + +final class SemiJoinMatcher implements Matcher { + private final String sourceSymbolAlias; + private final String filteringSymbolAlias; + private final String outputAlias; + + SemiJoinMatcher(String sourceSymbolAlias, String filteringSymbolAlias, String outputAlias) { + this.sourceSymbolAlias = requireNonNull(sourceSymbolAlias, "sourceSymbolAlias is null"); + this.filteringSymbolAlias = + requireNonNull(filteringSymbolAlias, "filteringSymbolAlias is null"); + this.outputAlias = requireNonNull(outputAlias, "outputAlias is null"); + } + + @Override + public boolean shapeMatches(PlanNode node) { + return node instanceof SemiJoinNode; + } + + @Override + public MatchResult detailMatches( + PlanNode node, SessionInfo sessionInfo, Metadata metadata, SymbolAliases symbolAliases) { + checkState( + shapeMatches(node), + "Plan testing framework error: shapeMatches returned false in detailMatches in %s", + this.getClass().getName()); + + SemiJoinNode semiJoinNode = (SemiJoinNode) node; + if (!(symbolAliases + .get(sourceSymbolAlias) + .equals(semiJoinNode.getSourceJoinSymbol().toSymbolReference()) + && symbolAliases + .get(filteringSymbolAlias) + .equals(semiJoinNode.getFilteringSourceJoinSymbol().toSymbolReference()))) { + return NO_MATCH; + } + + return match(outputAlias, semiJoinNode.getSemiJoinOutput().toSymbolReference()); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("filteringSymbolAlias", filteringSymbolAlias) + .add("sourceSymbolAlias", sourceSymbolAlias) + .add("outputAlias", outputAlias) + .toString(); + } +} From bdf36ba1907ac663f744221d54132ab079632799 Mon Sep 17 00:00:00 2001 From: Liao Lanyu <1435078631@qq.com> Date: Fri, 10 Jan 2025 11:17:20 +0800 Subject: [PATCH 08/13] Skip subquery plan if subqueryExpression List is empty for efficiency (#14647) --- .../relational/planner/SubqueryPlanner.java | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 17422f80247b..ebc32b6a247e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -115,32 +115,39 @@ public PlanBuilder handleSubqueries( public PlanBuilder handleSubqueries( PlanBuilder builder, Expression expression, Analysis.SubqueryAnalysis subqueries) { - for (Cluster cluster : - cluster( - builder.getScope(), - selectSubqueries(builder, expression, subqueries.getInPredicatesSubqueries()))) { - builder = planInPredicate(builder, cluster, subqueries); - } - for (Cluster cluster : - cluster( - builder.getScope(), - selectSubqueries(builder, expression, subqueries.getSubqueries()))) { - builder = planScalarSubquery(builder, cluster); + List inPredicates = subqueries.getInPredicatesSubqueries(); + if (!inPredicates.isEmpty()) { + for (Cluster cluster : + cluster(builder.getScope(), selectSubqueries(builder, expression, inPredicates))) { + builder = planInPredicate(builder, cluster, subqueries); + } } - for (Cluster cluster : - cluster( - builder.getScope(), - selectSubqueries(builder, expression, subqueries.getExistsSubqueries()))) { - builder = planExists(builder, cluster); + + List scalarSubqueries = subqueries.getSubqueries(); + if (!scalarSubqueries.isEmpty()) { + for (Cluster cluster : + cluster(builder.getScope(), selectSubqueries(builder, expression, scalarSubqueries))) { + builder = planScalarSubquery(builder, cluster); + } } - for (Cluster cluster : - cluster( - builder.getScope(), - selectSubqueries( - builder, expression, subqueries.getQuantifiedComparisonSubqueries()))) { - builder = planQuantifiedComparison(builder, cluster, subqueries); + + List existsPredicates = subqueries.getExistsSubqueries(); + if (!existsPredicates.isEmpty()) { + for (Cluster cluster : + cluster(builder.getScope(), selectSubqueries(builder, expression, existsPredicates))) { + builder = planExists(builder, cluster); + } } + List quantifiedComparisons = + subqueries.getQuantifiedComparisonSubqueries(); + if (!quantifiedComparisons.isEmpty()) { + for (Cluster cluster : + cluster( + builder.getScope(), selectSubqueries(builder, expression, quantifiedComparisons))) { + builder = planQuantifiedComparison(builder, cluster, subqueries); + } + } return builder; } From 2d436e0689704cd7efad14ac0cb7aafeff31f0d6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 10 Jan 2025 12:13:40 +0800 Subject: [PATCH 09/13] Optimized the error code when table/column does not exist & Reduced some unnecessary IT execution --- .../it/db/it/IoTDBDeletionTableIT.java | 2 +- .../it/db/it/IoTDBInsertTableIT.java | 2 +- ...IoTDBAlignByDeviceWithTemplateTableIT.java | 4 +-- ...oSelectExpressionAfterAnalyzedTableIT.java | 6 ++-- .../old/query/IoTDBResultSetTableIT.java | 2 +- .../relational/it/schema/IoTDBDeviceIT.java | 10 +++---- .../relational/it/schema/IoTDBTableIT.java | 4 +-- .../it/session/IoTDBSessionRelationalIT.java | 29 +++---------------- .../persistence/schema/ConfigMTree.java | 6 ++-- .../schema/source/TableDeviceQuerySource.java | 4 +-- .../analyzer/ExpressionAnalyzer.java | 6 ++-- .../plan/relational/analyzer/Scope.java | 8 ++++- .../analyzer/StatementAnalyzer.java | 13 ++++----- .../metadata/TableMetadataImpl.java | 14 +++++++++ .../fetcher/TableDeviceSchemaFetcher.java | 4 +-- .../fetcher/TableHeaderSchemaValidator.java | 3 +- .../table/InformationSchemaUtils.java | 2 +- .../table/ColumnNotExistsException.java | 2 +- .../table/TableAlreadyExistsException.java | 2 +- .../table/TableNotExistsException.java | 2 +- 20 files changed, 62 insertions(+), 63 deletions(-) rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/exception/metadata => node-commons/src/main/java/org/apache/iotdb/commons/exception}/table/ColumnNotExistsException.java (95%) rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/exception/metadata => node-commons/src/main/java/org/apache/iotdb/commons/exception}/table/TableAlreadyExistsException.java (95%) rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/exception/metadata => node-commons/src/main/java/org/apache/iotdb/commons/exception}/table/TableNotExistsException.java (95%) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index b69a557b41aa..99dfe05db99b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -593,7 +593,7 @@ public void testDeleteTable() throws SQLException { try (ResultSet ignored = statement.executeQuery("SELECT * FROM vehicle" + testNum)) { fail("Exception expected"); } catch (SQLException e) { - assertEquals("701: Table 'test.vehicle12' does not exist", e.getMessage()); + assertEquals("550: Table 'test.vehicle12' does not exist.", e.getMessage()); } statement.execute( diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java index 71b1eb4a8c0f..e751ab6423f2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java @@ -464,7 +464,7 @@ public void testInsertMultiRowWithNull() throws SQLException { st1.execute("insert into wt14(time, s1, s2) values(100, null, 1), (101, null, 2)"); fail(); } catch (SQLException e) { - assertEquals("507: Table wt14 does not exist", e.getMessage()); + assertEquals("550: Table 'test.wt14' does not exist.", e.getMessage()); } try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT)) { try (Statement st2 = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java index c221dc401670..bf9e7f89db0c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java @@ -187,7 +187,7 @@ public void selectMeasurementNoFilterTest() { statement.executeQuery("SELECT s3,s1,s_null FROM table1 order by device_id")) { fail("should throw exception to indicate that s_null doesn't exist"); } catch (SQLException e) { - assertEquals("701: Column 's_null' cannot be resolved", e.getMessage()); + assertEquals("616: Column 's_null' cannot be resolved", e.getMessage()); } } } catch (SQLException e) { @@ -277,7 +277,7 @@ public void selectWildcardWithFilterOrderByTimeTest() { statement.executeQuery("SELECT * FROM table1 WHERE s_null > 1 order by device_id")) { fail("should throw exception to indicate that s_null doesn't exist"); } catch (SQLException e) { - assertEquals("701: Column 's_null' cannot be resolved", e.getMessage()); + assertEquals("616: Column 's_null' cannot be resolved", e.getMessage()); } } } catch (SQLException e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java index de0dbea13d9b..27d14c0e1fcc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java @@ -64,7 +64,7 @@ public void testAlignByDevice() { String[] retArray = new String[] {}; tableAssertTestFail( "select s2 from sg where s1>0 order by device", - "701: Column 's2' cannot be resolved", + "616: Column 's2' cannot be resolved", DATABASE_NAME); // TODO After Aggregation supported @@ -81,14 +81,14 @@ public void testAlignByDevice() { tableAssertTestFail( "select s1, s2 from sg where s1>0 order by device", - "701: Column 's2' cannot be resolved", + "616: Column 's2' cannot be resolved", DATABASE_NAME); } @Test public void testAlignByTime() { tableAssertTestFail( - "select s2 from sg where s1>0", "701: Column 's2' cannot be resolved", DATABASE_NAME); + "select s2 from sg where s1>0", "616: Column 's2' cannot be resolved", DATABASE_NAME); /*tableResultSetEqualTest("select count(s2) from sg where s1>0", expectedHeader, retArray,DATABASE_NAME);*/ } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java index 31a8cb21aa21..a7ad4a07be95 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java @@ -144,7 +144,7 @@ public void columnTypeTest() { @Test public void emptyQueryTest1() { - tableAssertTestFail("select * from sg1", "701: Table 'test.sg1' does not exist", DATABASE_NAME); + tableAssertTestFail("select * from sg1", "550: Table 'test.sg1' does not exist", DATABASE_NAME); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java index a7c6f5f3dd94..d2479f68f351 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java @@ -153,14 +153,14 @@ public void testDevice() throws SQLException { statement.executeQuery("show devices from table2"); fail("Show devices shall fail for non-exist table"); } catch (final Exception e) { - assertEquals("701: Table 'test.table2' does not exist.", e.getMessage()); + assertEquals("550: Table 'test.table2' does not exist.", e.getMessage()); } try { statement.executeQuery("count devices from table2"); fail("Count devices shall fail for non-exist table"); } catch (final Exception e) { - assertEquals("701: Table 'test.table2' does not exist.", e.getMessage()); + assertEquals("550: Table 'test.table2' does not exist.", e.getMessage()); } try { @@ -176,7 +176,7 @@ public void testDevice() throws SQLException { statement.executeQuery("count devices from table0 where a = 1"); fail("Count devices shall fail for non-exist column"); } catch (final Exception e) { - assertEquals("701: Column 'a' cannot be resolved", e.getMessage()); + assertEquals("616: Column 'a' cannot be resolved", e.getMessage()); } // Test fully qualified name @@ -193,7 +193,7 @@ public void testDevice() throws SQLException { statement.execute("update table2 set model = '1'"); fail("Update shall fail for non-exist table"); } catch (final Exception e) { - assertEquals("701: Table 'test.table2' does not exist.", e.getMessage()); + assertEquals("550: Table 'test.table2' does not exist.", e.getMessage()); } try { @@ -214,7 +214,7 @@ public void testDevice() throws SQLException { statement.execute("update table0 set col = '1'"); fail("Update shall fail for non-exist column"); } catch (final Exception e) { - assertEquals("701: Column 'col' cannot be resolved", e.getMessage()); + assertEquals("616: Column 'col' cannot be resolved", e.getMessage()); } try { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index 2def42f0dcc6..1bfd554dde26 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -424,14 +424,14 @@ public void testManageTable() { statement.executeQuery("select color from table2"); fail(); } catch (final SQLException e) { - assertEquals("701: Column 'color' cannot be resolved", e.getMessage()); + assertEquals("616: Column 'color' cannot be resolved", e.getMessage()); } try { statement.executeQuery("select speed from table2"); fail(); } catch (final SQLException e) { - assertEquals("701: Column 'speed' cannot be resolved", e.getMessage()); + assertEquals("616: Column 'speed' cannot be resolved", e.getMessage()); } try { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index b2d45d554668..1f70341892b3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -23,8 +23,6 @@ import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.ClusterIT; -import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -222,7 +220,6 @@ private static void insertRelationalTabletPerformanceTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalSqlTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -334,7 +331,6 @@ public void insertRelationalSqlTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void partialInsertSQLTest() throws IoTDBConnectionException, StatementExecutionException { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { // disable auto-creation only for this test @@ -365,7 +361,6 @@ public void partialInsertSQLTest() throws IoTDBConnectionException, StatementExe } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void partialInsertRelationalTabletTest() throws IoTDBConnectionException, StatementExecutionException { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -510,7 +505,6 @@ public void partialInsertRelationalTabletTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalTabletTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -587,7 +581,6 @@ public void insertRelationalTabletTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalTabletWithCacheLeaderTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -669,7 +662,6 @@ public void insertRelationalTabletWithCacheLeaderTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoCreateNontagColumnTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -744,7 +736,6 @@ public void autoCreateNontagColumnTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoCreateTableTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("USE \"db1\""); @@ -803,7 +794,6 @@ public void autoCreateTableTest() throws IoTDBConnectionException, StatementExec } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoCreateTagColumnTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -882,7 +872,6 @@ public void autoCreateTagColumnTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoAdjustTagTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("USE \"db1\""); @@ -950,7 +939,6 @@ public void autoAdjustTagTest() throws IoTDBConnectionException, StatementExecut } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalSqlWithoutDBTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -1008,7 +996,6 @@ public void insertRelationalSqlWithoutDBTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalSqlAnotherDBTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -1057,7 +1044,6 @@ public void insertRelationalSqlAnotherDBTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertNonExistTableTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -1070,7 +1056,7 @@ public void insertNonExistTableTest() 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); fail("Exception expected"); } catch (StatementExecutionException e) { - assertEquals("507: Table table13 does not exist", e.getMessage()); + assertEquals("550: Table 'db1.table13' does not exist.", e.getMessage()); } try { @@ -1080,13 +1066,12 @@ public void insertNonExistTableTest() 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); fail("Exception expected"); } catch (StatementExecutionException e) { - assertEquals("507: Table table13 does not exist", e.getMessage()); + assertEquals("550: Table 'db2.table13' does not exist.", e.getMessage()); } } } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertNonExistDBTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("USE \"db1\""); @@ -1098,13 +1083,12 @@ public void insertNonExistDBTest() throws IoTDBConnectionException, StatementExe 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); fail("Exception expected"); } catch (StatementExecutionException e) { - assertEquals("507: Table table13 does not exist", e.getMessage()); + assertEquals("550: Table 'db3.table13' does not exist.", e.getMessage()); } } } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertWithoutMeasurementTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -1356,7 +1340,6 @@ private Object genValue(TSDataType dataType, int i) { } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalTabletWithAutoCastTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 14; @@ -1380,7 +1363,6 @@ public void insertRelationalTabletWithAutoCastTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void deleteTableAndWriteDifferentTypeTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 15; @@ -1410,7 +1392,6 @@ public void deleteTableAndWriteDifferentTypeTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void dropTableOfTheSameNameTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 16; @@ -1441,13 +1422,12 @@ public void dropTableOfTheSameNameTest() session.executeQueryStatement("select * from db2.table" + testNum + " order by time"); fail("expected exception"); } catch (StatementExecutionException e) { - assertEquals("701: Table 'db2.table16' does not exist", e.getMessage()); + assertEquals("550: Table 'db2.table16' does not exist.", e.getMessage()); } } } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalRowWithAutoCastTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 17; @@ -1471,7 +1451,6 @@ public void insertRelationalRowWithAutoCastTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoCreateTagColumnTest2() throws IoTDBConnectionException, StatementExecutionException { int testNum = 18; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index 45e6fda8fcde..0b4bf3f840c5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -23,6 +23,9 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.exception.table.ColumnNotExistsException; +import org.apache.iotdb.commons.exception.table.TableAlreadyExistsException; +import org.apache.iotdb.commons.exception.table.TableNotExistsException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.node.role.IDatabaseMNode; @@ -42,9 +45,6 @@ import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; -import org.apache.iotdb.db.exception.metadata.table.ColumnNotExistsException; -import org.apache.iotdb.db.exception.metadata.table.TableAlreadyExistsException; -import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.DatabaseCollector; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MNodeAboveDBCollector; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java index dfa0cf4baf1f..fd18ff5ed83d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; @@ -189,8 +190,7 @@ public static List getDevicePatternList( final String tableName, final List> idDeterminedPredicateList) { if (Objects.isNull(DataNodeTableCache.getInstance().getTable(database, tableName))) { - throw new SchemaExecutionException( - String.format("Table '%s.%s' does not exist.", database, tableName)); + TableMetadataImpl.throwTableNotExistsException(database, tableName); } return DeviceFilterUtil.convertToDevicePattern( database, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index e551638b69c8..1798a0ed0b7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.OperatorNotFoundException; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticBinaryExpression; @@ -419,8 +420,7 @@ protected Type visitDereferenceExpression( return handleResolvedField(node, resolvedField.get(), context); } if (!scope.isColumnReference(qualifiedName)) { - throw new SemanticException( - String.format("Column '%s' cannot be resolved", qualifiedName)); + TableMetadataImpl.throwColumnNotExistsException(qualifiedName); } } @@ -450,7 +450,7 @@ protected Type visitDereferenceExpression( } if (rowFieldType == null) { - throw new SemanticException(String.format("Column '%s' cannot be resolved", qualifiedName)); + TableMetadataImpl.throwColumnNotExistsException(qualifiedName); } return setExpressionType(node, rowFieldType); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java index dfcc21e4e9c4..0e7588a59f49 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java @@ -19,11 +19,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; +import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WithQuery; +import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.collect.ImmutableMap; @@ -228,7 +230,11 @@ public boolean isLocalScope(Scope other) { public ResolvedField resolveField(Expression expression, QualifiedName name) { return tryResolveField(expression, name) .orElseThrow( - () -> new SemanticException(String.format("Column '%s' cannot be resolved", name))); + () -> + new SemanticException( + new IoTDBException( + String.format("Column '%s' cannot be resolved", name), + TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode()))); } public Optional tryResolveField(Expression expression) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index fc7e33303973..93ec5bef3a50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.relational.planner.PlannerContext; import org.apache.iotdb.db.queryengine.plan.relational.planner.ScopeAware; @@ -462,9 +463,7 @@ protected Scope visitDeleteDevice(final DeleteDevice node, final Optional final TsTable table = DataNodeTableCache.getInstance().getTable(node.getDatabase(), node.getTableName()); if (Objects.isNull(table)) { - throw new SemanticException( - String.format( - "Table '%s.%s' does not exist.", node.getDatabase(), node.getTableName())); + TableMetadataImpl.throwTableNotExistsException(node.getDatabase(), node.getTableName()); } node.parseModEntries(table); analyzeTraverseDevice(node, context, node.getWhere().isPresent()); @@ -1765,7 +1764,8 @@ protected Scope visitTable(Table table, Optional scope) { Optional tableSchema = metadata.getTableSchema(sessionContext, name); // This can only be a table if (!tableSchema.isPresent()) { - throw new SemanticException(String.format("Table '%s' does not exist", name)); + TableMetadataImpl.throwTableNotExistsException( + name.getDatabaseName(), name.getObjectName()); } analysis.addEmptyColumnReferencesForTable(accessControl, sessionContext.getIdentity(), name); @@ -2961,8 +2961,7 @@ private TranslationMap analyzeTraverseDevice( } if (!metadata.tableExists(new QualifiedObjectName(database, tableName))) { - throw new SemanticException( - String.format("Table '%s.%s' does not exist.", database, tableName)); + TableMetadataImpl.throwTableNotExistsException(database, tableName); } node.setColumnHeaderList(); @@ -2972,7 +2971,7 @@ private TranslationMap analyzeTraverseDevice( final Optional tableSchema = metadata.getTableSchema(sessionContext, name); // This can only be a table if (!tableSchema.isPresent()) { - throw new SemanticException(String.format("Table '%s' does not exist", name)); + TableMetadataImpl.throwTableNotExistsException(database, tableName); } final TableSchema originalSchema = tableSchema.get(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index ab2ba5b9f331..fb437d3d5966 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.table.TableNotExistsException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; @@ -50,6 +52,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignature; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.utils.constant.SqlConstant; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.udf.api.customizer.analysis.AggregateFunctionAnalysis; import org.apache.iotdb.udf.api.customizer.analysis.ScalarFunctionAnalysis; import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; @@ -900,4 +903,15 @@ public static boolean isTwoTypeCalculable(List argumentTypes) { } return isArithmeticType(left) && isArithmeticType(right); } + + public static void throwTableNotExistsException(final String database, final String tableName) { + throw new SemanticException(new TableNotExistsException(database, tableName)); + } + + public static void throwColumnNotExistsException(final Object columnName) { + throw new SemanticException( + new IoTDBException( + String.format("Column '%s' cannot be resolved.", columnName), + TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index ae5362cd08f4..b26563c521be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -27,7 +27,6 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; -import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.Coordinator; @@ -36,6 +35,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.schema.ConvertSchemaPredicateToFilterVisitor; import org.apache.iotdb.db.queryengine.plan.relational.metadata.AlignedDeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AbstractTraverseDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -172,7 +172,7 @@ public List fetchDeviceSchemaForDataQuery( final ShowDevice statement = new ShowDevice(database, table); final TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); if (tableInstance == null) { - throw new SemanticException(String.format("Table '%s.%s' does not exist", database, table)); + TableMetadataImpl.throwTableNotExistsException(database, table); } if (parseFilter4TraverseDevice( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java index 3e0e00541f3e..41932c8944d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java @@ -38,6 +38,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateTableTask; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -123,7 +124,7 @@ public Optional validateTableHeaderSchema( "auto create table succeed, but cannot get table schema in current node's DataNodeTableCache, may be caused by concurrently auto creating table"); } } else { - throw new SemanticException("Table " + tableSchema.getTableName() + " does not exist"); + TableMetadataImpl.throwTableNotExistsException(database, tableSchema.getTableName()); } } else { // If table with this name already exists and isStrictIdColumn is true, make sure the existing diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java index 2b3875114bdc..54f5769aead0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java @@ -21,11 +21,11 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.table.TableNotExistsException; import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; -import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/ColumnNotExistsException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/ColumnNotExistsException.java similarity index 95% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/ColumnNotExistsException.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/ColumnNotExistsException.java index 9fcda36998e8..0a9fa784e9f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/ColumnNotExistsException.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/ColumnNotExistsException.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.exception.metadata.table; +package org.apache.iotdb.commons.exception.table; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableAlreadyExistsException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableAlreadyExistsException.java similarity index 95% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableAlreadyExistsException.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableAlreadyExistsException.java index d28c04e9e7ec..d0baa216b84f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableAlreadyExistsException.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableAlreadyExistsException.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.exception.metadata.table; +package org.apache.iotdb.commons.exception.table; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableNotExistsException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableNotExistsException.java similarity index 95% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableNotExistsException.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableNotExistsException.java index 616a20d49bc3..c0caabe711a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableNotExistsException.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableNotExistsException.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.exception.metadata.table; +package org.apache.iotdb.commons.exception.table; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.rpc.TSStatusCode; From 04df3e279a2d51083567a6098b2d488566d6e118 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:09:59 +0800 Subject: [PATCH 10/13] Fixed the un-renamed ID/MEASUREMENTs in some non-runnable ITs --- .../scalar/IoTDBFormatFunctionTableIT.java | 21 +- .../old/query/IoTDBNullValueFillTableIT.java | 327 ------------------ .../old/query/IoTDBSelectSchemaTableIT.java | 22 +- 3 files changed, 20 insertions(+), 350 deletions(-) delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullValueFillTableIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java index 35e20f7edec1..f9018b65d98b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java @@ -15,11 +15,17 @@ package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; import java.sql.Connection; import java.sql.Statement; @@ -27,6 +33,9 @@ import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +@Ignore +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) public class IoTDBFormatFunctionTableIT { private static final String DATABASE_NAME = "db"; @@ -36,12 +45,12 @@ public class IoTDBFormatFunctionTableIT { // normal cases "CREATE DATABASE " + DATABASE_NAME, "USE " + DATABASE_NAME, - "CREATE TABLE number_table(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT)", - "CREATE TABLE string_table(device_id STRING ID, s1 STRING MEASUREMENT, s2 TEXT MEASUREMENT)", - "CREATE TABLE boolean_table(device_id STRING ID, s1 BOOLEAN MEASUREMENT)", - "CREATE TABLE timestamp_table(device_id STRING ID, s1 TIMESTAMP MEASUREMENT)", - "CREATE TABLE date_table(device_id STRING ID, s1 DATE MEASUREMENT)", - "CREATE TABLE null_table(device_id STRING ID, s1 INT32 MEASUREMENT)", + "CREATE TABLE number_table(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD)", + "CREATE TABLE string_table(device_id STRING TAG, s1 STRING FIELD, s2 TEXT FIELD)", + "CREATE TABLE boolean_table(device_id STRING TAG, s1 BOOLEAN FIELD)", + "CREATE TABLE timestamp_table(device_id STRING TAG, s1 TIMESTAMP FIELD)", + "CREATE TABLE date_table(device_id STRING TAG, s1 DATE FIELD)", + "CREATE TABLE null_table(device_id STRING TAG, s1 INT32 FIELD)", // data for number series "INSERT INTO number_table(time, device_id, s1, s2, s3, s4) VALUES (10, 'd1', 1000000, 1000000, 3.1415926, 3.1415926)", "INSERT INTO number_table(time, device_id, s1, s2, s3, s4) VALUES (20, 'd1', 1, 1, 1234567.25, 1234567.25)", diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullValueFillTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullValueFillTableIT.java deleted file mode 100644 index a7ba0315d52b..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullValueFillTableIT.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.relational.it.query.old.query; - -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.TableClusterIT; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; -import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; -import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualWithDescOrderTest; - -@Ignore -@RunWith(IoTDBTestRunner.class) -@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBNullValueFillTableIT { - private static final String DATABASE_NAME = "test"; - - /** - * [root.sg1.d1 (aligned)] - * - *

Time, s1[INT32], s2[INT64], s3[FLOAT], s4[DOUBLE], s5[BOOLEAN], s6[TEXT]
- * 1, null, 1, null, 1.0, null, t1
- * 2, 2, 2, 2.0, 2.0, true, t2
- * 3, 3, null, 3.0, null, false, null
- * 4, null, 4, null, 4.0, null, t4
- * 5, 5, 5, 5.0, 5.0, false, t5
- * 6, null, 6, null, 6.0, false, null
- * 7, null, null, null, null, null, null
- * 8, 8, 8, 8.0, 8.0, true, t8
- * 9, 9, null, 9.0, null, true, null - * - *

[root.sg1.d2 (non-aligned)] - * - *

Time, s1[INT32], s2[INT64], s3[FLOAT], s4[DOUBLE], s5[BOOLEAN], s6[TEXT]
- * 1, 1, null, 1.0, null, true, null
- * 2, 2, 2, 2.0, 2.0, true, t2
- * 3, null, 3, null, 3.0, null, t3
- * 4, 4, null, 4.0, null, false, null
- * 5, 5, 5, 5.0, 5.0, false, t5
- * 6, 6, null, 6.0, null, false, null
- * 7, null, null, null, null, null, null
- * 8, 8, 8, 8.0, 8.0, true, t8
- * 9, null, 9, null, 9.0, null, t9 - */ - private static final String[] sqls = - new String[] { - "CREATE DATABASE " + DATABASE_NAME, - "USE " + DATABASE_NAME, - "CREATE TABLE testNullFilter(device STRING ID, s1 INT32 MEASUREMENT, s2 BOOLEAN MEASUREMENT, s3 DOUBLE MEASUREMENT)", - "create aligned timeseries root.sg1.d1(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT)", - "insert into root.sg1.d1(time, s2, s4, s6) aligned values(1, 1, 1.0, 't1')", - "insert into root.sg1.d1(time, s1, s2, s3, s4, s5, s6) aligned values(2, 2, 2, 2.0, 2.0, true, 't2')", - "insert into root.sg1.d1(time, s1, s3, s5) aligned values(3, 3, 3.0, false)", - "insert into root.sg1.d1(time, s2, s4, s6) aligned values(4, 4, 4.0, 't4')", - "insert into root.sg1.d1(time, s1, s2, s3, s4, s5, s6) aligned values(5, 5, 5, 5.0, 5.0, false, 't5')", - "insert into root.sg1.d1(time, s2, s4, s6) aligned values(6, 6, 6.0, 't6')", - "insert into root.sg1.d1(time, s1, s2, s3, s4, s5, s6) aligned values(8, 8, 8, 8.0, 8.0, true, 't8')", - "insert into root.sg1.d1(time, s1, s3, s5) aligned values(9, 9, 9.0, true)", - "create timeseries root.sg1.d2.s1 INT32", - "create timeseries root.sg1.d2.s2 INT64", - "create timeseries root.sg1.d2.s3 FLOAT", - "create timeseries root.sg1.d2.s4 DOUBLE", - "create timeseries root.sg1.d2.s5 BOOLEAN", - "create timeseries root.sg1.d2.s6 TEXT", - "insert into root.sg1.d2(time, s1, s3, s5) values(1, 1, 1.0, true)", - "insert into root.sg1.d2(time, s1, s2, s3, s4, s5, s6) values(2, 2, 2, 2.0, 2.0, true, 't2')", - "insert into root.sg1.d2(time, s2, s4, s6) values(3, 3, 3.0, 't3')", - "insert into root.sg1.d2(time, s1, s3, s5) values(4, 4, 4.0, false)", - "insert into root.sg1.d2(time, s1, s2, s3, s4, s5, s6) values(5, 5, 5, 5.0, 5.0, false, 't5')", - "insert into root.sg1.d2(time, s1, s3, s5) values(6, 6, 6.0, false)", - "insert into root.sg1.d2(time, s1, s2, s3, s4, s5, s6) values(8, 8, 8, 8.0, 8.0, true, 't8')", - "insert into root.sg1.d2(time, s2, s4, s6) values(9, 9, 9.0, 't9')" - }; - - private final String[] expectedHeader = - new String[] { - "Time", - "root.sg1.d1.s1", - "root.sg1.d1.s2", - "root.sg1.d1.s3", - "root.sg1.d1.s4", - "root.sg1.d1.s5", - "root.sg1.d1.s6" - }; - - private final String[] expectedAlignByDeviceHeader = - new String[] {"Time", "Device", "s1", "s2", "s3", "s4", "s5", "s6"}; - - @BeforeClass - public static void setUp() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); - prepareData(sqls); - } - - @AfterClass - public static void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void previousFillTest() { - String[] retArray = - new String[] { - "1,null,1,null,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,2,3.0,2.0,false,t2,", - "4,3,4,3.0,4.0,false,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,5,6,5.0,6.0,false,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,8,9.0,8.0,true,t8," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(previous)", expectedHeader, retArray); - } - - @Test - public void previousDescFillTest() { - String[] retArray = - new String[] { - "9,9,null,9.0,null,true,null,", - "8,8,8,8.0,8.0,true,t8,", - "6,8,6,8.0,6.0,true,t6,", - "5,5,5,5.0,5.0,false,t5,", - "4,5,4,5.0,4.0,false,t4,", - "3,3,4,3.0,4.0,false,t4,", - "2,2,2,2.0,2.0,true,t2,", - "1,2,1,2.0,1.0,true,t1," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(previous) order by time desc", - expectedHeader, - retArray); - } - - @Test - public void previousFillAlignByDeviceTest() { - String[] retArray = - new String[] { - "1,root.sg1.d1,null,1,null,1.0,null,t1,", - "2,root.sg1.d1,2,2,2.0,2.0,true,t2,", - "3,root.sg1.d1,3,2,3.0,2.0,false,t2,", - "4,root.sg1.d1,3,4,3.0,4.0,false,t4,", - "5,root.sg1.d1,5,5,5.0,5.0,false,t5,", - "6,root.sg1.d1,5,6,5.0,6.0,false,t6,", - "8,root.sg1.d1,8,8,8.0,8.0,true,t8,", - "9,root.sg1.d1,9,8,9.0,8.0,true,t8,", - "1,root.sg1.d2,1,8,1.0,8.0,true,t8,", - "2,root.sg1.d2,2,2,2.0,2.0,true,t2,", - "3,root.sg1.d2,2,3,2.0,3.0,true,t3,", - "4,root.sg1.d2,4,3,4.0,3.0,false,t3,", - "5,root.sg1.d2,5,5,5.0,5.0,false,t5,", - "6,root.sg1.d2,6,5,6.0,5.0,false,t5,", - "8,root.sg1.d2,8,8,8.0,8.0,true,t8,", - "9,root.sg1.d2,8,9,8.0,9.0,true,t9," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.* fill(previous) align by device", - expectedAlignByDeviceHeader, - retArray); - } - - @Test - public void previousDescFillAlignByDeviceTest() { - String[] retArray = - new String[] { - "9,root.sg1.d1,9,null,9.0,null,true,null,", - "8,root.sg1.d1,8,8,8.0,8.0,true,t8,", - "6,root.sg1.d1,8,6,8.0,6.0,true,t6,", - "5,root.sg1.d1,5,5,5.0,5.0,false,t5,", - "4,root.sg1.d1,5,4,5.0,4.0,false,t4,", - "3,root.sg1.d1,3,4,3.0,4.0,false,t4,", - "2,root.sg1.d1,2,2,2.0,2.0,true,t2,", - "1,root.sg1.d1,2,1,2.0,1.0,true,t1,", - "9,root.sg1.d2,2,9,2.0,9.0,true,t9,", - "8,root.sg1.d2,8,8,8.0,8.0,true,t8,", - "6,root.sg1.d2,6,8,6.0,8.0,false,t8,", - "5,root.sg1.d2,5,5,5.0,5.0,false,t5,", - "4,root.sg1.d2,4,5,4.0,5.0,false,t5,", - "3,root.sg1.d2,4,3,4.0,3.0,false,t3,", - "2,root.sg1.d2,2,2,2.0,2.0,true,t2,", - "1,root.sg1.d2,1,2,1.0,2.0,true,t2," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.* fill(previous) order by device,time desc align by device", - expectedAlignByDeviceHeader, - retArray); - } - - @Test - public void linearFillTest() { - String[] retArray = - new String[] { - "1,null,1,null,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,3,3.0,3.0,false,null,", - "4,4,4,4.0,4.0,null,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,6,6,6.0,6.0,null,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,null,9.0,null,true,null," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(linear)", expectedHeader, retArray); - } - - @Test - public void linearFillAlignByDeviceTest() { - String[] retArray = - new String[] { - "1,root.sg1.d1,null,1,null,1.0,null,t1,", - "2,root.sg1.d1,2,2,2.0,2.0,true,t2,", - "3,root.sg1.d1,3,3,3.0,3.0,false,null,", - "4,root.sg1.d1,4,4,4.0,4.0,null,t4,", - "5,root.sg1.d1,5,5,5.0,5.0,false,t5,", - "6,root.sg1.d1,6,6,6.0,6.0,null,t6,", - "8,root.sg1.d1,8,8,8.0,8.0,true,t8,", - "9,root.sg1.d1,9,9,9.0,9.0,true,null,", - "1,root.sg1.d2,1,1,1.0,1.0,true,null,", - "2,root.sg1.d2,2,2,2.0,2.0,true,t2,", - "3,root.sg1.d2,3,3,3.0,3.0,null,t3,", - "4,root.sg1.d2,4,4,4.0,4.0,false,null,", - "5,root.sg1.d2,5,5,5.0,5.0,false,t5,", - "6,root.sg1.d2,6,6,6.0,6.0,false,null,", - "8,root.sg1.d2,8,8,8.0,8.0,true,t8,", - "9,root.sg1.d2,null,9,null,9.0,null,t9," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.* fill(linear) align by device", - expectedAlignByDeviceHeader, - retArray); - } - - @Test - public void intFillTest() { - String[] retArray = - new String[] { - "1,1000,1,1000.0,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,1000,3.0,1000.0,false,1000,", - "4,1000,4,1000.0,4.0,null,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,1000,6,1000.0,6.0,null,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,1000,9.0,1000.0,true,1000," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(1000)", expectedHeader, retArray); - } - - @Test - public void floatFillTest() { - String[] retArray = - new String[] { - "1,null,1,3.14,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,null,3.0,3.14,false,3.14,", - "4,null,4,3.14,4.0,null,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,null,6,3.14,6.0,null,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,null,9.0,3.14,true,3.14," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(3.14)", expectedHeader, retArray); - } - - @Test - public void booleanFillTest() { - String[] retArray = - new String[] { - "1,null,1,null,1.0,true,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,null,3.0,null,false,true,", - "4,null,4,null,4.0,true,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,null,6,null,6.0,true,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,null,9.0,null,true,true," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(true)", expectedHeader, retArray); - } - - @Test - public void textFillTest() { - String[] retArray = - new String[] { - "1,null,1,null,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,null,3.0,null,false,t0,", - "4,null,4,null,4.0,null,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,null,6,null,6.0,null,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,null,9.0,null,true,t0," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill('t0')", expectedHeader, retArray); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java index aee30ec1de2b..e17f873e116b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java @@ -27,7 +27,6 @@ import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -49,7 +48,7 @@ public class IoTDBSelectSchemaTableIT { new String[] { "CREATE DATABASE " + DATABASE_NAME, "USE " + DATABASE_NAME, - "CREATE TABLE sg(device STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 DOUBLE MEASUREMENT)", + "CREATE TABLE sg(device STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 DOUBLE FIELD)", "insert into sg(time, device, s1, s2, s3) values (1, 'd1', 1, 2, 3.0)" }; @@ -64,7 +63,6 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } - @Ignore // TODO After sin supported @Test public void testSchemaExpression() { String[] expressions = { @@ -80,18 +78,9 @@ public void testSchemaExpression() { "sin(s1)+s1", "((s1+1)*2-1)%2+1.5+s2" }; - String[] completeExpressions = { - "s1+s2", - "-s1+s2", - "-(s1+s3)", - "not(s1>s2)", - "-(-s1)", - "(s1+s2)*s3", - "-2+s1", - "not true or s1>0", - "-(-1)+s1", - "sin(s1)+s1", - "((s1+1)*2-1)%2+1.5+s2", + String[] columnNames = { + "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", "_col10", + "_col11", }; try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { @@ -115,8 +104,7 @@ public void testSchemaExpression() { assertEquals(1 + expressions.length, columnCount); for (int i = 0; i < expressions.length; ++i) { - assertEquals( - completeExpressions[i], resultSet.getMetaData().getColumnName(i + 2).replace(" ", "")); + assertEquals(columnNames[i], resultSet.getMetaData().getColumnName(i + 2).replace(" ", "")); } } catch (SQLException throwable) { fail(throwable.getMessage()); From 7f0ac972839fa919e9cdc7c3eff814c0d249377f Mon Sep 17 00:00:00 2001 From: Zhenyu Luo Date: Fri, 10 Jan 2025 15:57:08 +0800 Subject: [PATCH 11/13] Pipe: Fix NPE caused by forced type conversion and fix IoTDBipipeTypeConversionISessionIT of table model (#14667) --- .../IoTDBPipeTypeConversionISessionIT.java | 93 +++++++++++++------ .../PipeConvertedInsertRowStatement.java | 30 +++++- .../PipeConvertedInsertTabletStatement.java | 48 +++++++++- ...tementDataTypeConvertExecutionVisitor.java | 3 +- ...tementDataTypeConvertExecutionVisitor.java | 3 +- .../statement/crud/InsertBaseStatement.java | 6 +- .../statement/crud/InsertRowStatement.java | 18 +++- 7 files changed, 159 insertions(+), 42 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java index 4145fc9a5d2a..b3e40bb8a10d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java @@ -56,12 +56,13 @@ import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2TableModel.class}) public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeTableModelTestIT { - private static final int generateDataSize = 100; + private static final int generateDataSize = 1000; @Test public void insertTablet() { @@ -90,8 +91,8 @@ private SessionDataSet query( param.append(schema.getMeasurementName()); param.append(','); } - sql = sql + param.substring(0, param.length() - 1); - sql = sql + " from " + tableName + " ORDER BY time ASC"; + + sql = sql + param + "time from " + tableName + " ORDER BY time ASC"; session.executeNonQueryStatement("use test"); return session.executeQueryStatement(sql); } @@ -106,7 +107,8 @@ private void prepareTypeConversionTest( createDatabaseAndTable(measurementSchemas, false, tablet.getColumnTypes(), receiverEnv); try (ITableSession senderSession = senderEnv.getTableSessionConnection(); ITableSession receiverSession = receiverEnv.getTableSessionConnection()) { - + senderSession.executeNonQueryStatement("use test"); + receiverSession.executeNonQueryStatement("use test"); if (isTsFile) { // Send TsFile data to receiver executeDataWriteOperation.accept(senderSession, receiverSession, tablet); @@ -121,7 +123,7 @@ private void prepareTypeConversionTest( } // Verify receiver data - long timeoutSeconds = 30; + long timeoutSeconds = 600; List> expectedValues = generateTabletResultSetForTable(tablet, measurementSchemas); await() @@ -140,10 +142,10 @@ private void prepareTypeConversionTest( fail(e.getMessage()); } }); - tablet.reset(); } catch (Exception e) { fail(e.getMessage()); } + tablet.reset(); } private void createDatabaseAndTable( @@ -174,16 +176,15 @@ private void createDataPipe(boolean isTSFile) { String sql = String.format( "create pipe test" - + " with source ('source'='iotdb-source','realtime.mode'='%s','realtime.enable'='%s','history.enable'='%s')" + + " with source ('source'='iotdb-source','realtime.mode'='%s')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')", isTSFile ? "file" : "forced-log", - !isTSFile, - isTSFile, receiverEnv.getIP(), receiverEnv.getPort(), isTSFile ? "tsfile" : "tablet"); - TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, Collections.singletonList(sql)); + TestUtils.tryExecuteNonQueriesWithRetry( + null, BaseEnv.TABLE_SQL_DIALECT, senderEnv, Collections.singletonList(sql)); } private void validateResultSet( @@ -193,11 +194,13 @@ private void validateResultSet( while (dataSet.hasNext()) { RowRecord record = dataSet.next(); List fields = record.getFields(); - - assertEquals(record.getTimestamp(), timestamps[index]); List rowValues = values.get(index++); for (int i = 0; i < fields.size(); i++) { Field field = fields.get(i); + if (field.getDataType() == null) { + assertNull(rowValues.get(i)); + continue; + } switch (field.getDataType()) { case INT64: case TIMESTAMP: @@ -207,11 +210,9 @@ private void validateResultSet( assertEquals(field.getDateV(), rowValues.get(i)); break; case BLOB: - assertEquals(field.getBinaryV(), rowValues.get(i)); - break; case TEXT: case STRING: - assertEquals(field.getStringValue(), rowValues.get(i)); + assertEquals(field.getBinaryV(), rowValues.get(i)); break; case INT32: assertEquals(field.getIntV(), (int) rowValues.get(i)); @@ -283,13 +284,11 @@ private void createTestDataForTimestamp(Tablet tablet, int j) { } } - private long[] createTestDataForTimestamp() { + private void createTestDataForTimeColumn(Tablet tablet) { long time = new Date().getTime(); - long[] data = new long[generateDataSize]; - for (int i = 0; i < data.length; i++) { - data[i] = time++; + for (int i = 0; i < generateDataSize; i++) { + tablet.addTimestamp(i, time++); } - return data; } private void createTestDataForDate(Tablet tablet, int j) { @@ -346,19 +345,35 @@ private List> generateTabletResultSetForTable( switch (sourceType) { case INT64: case TIMESTAMP: - value = ValueConverter.convert(sourceType, targetType, ((long[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((long[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case INT32: - value = ValueConverter.convert(sourceType, targetType, ((int[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((int[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case DOUBLE: - value = ValueConverter.convert(sourceType, targetType, ((double[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((double[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case FLOAT: - value = ValueConverter.convert(sourceType, targetType, ((float[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((float[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case DATE: @@ -366,24 +381,39 @@ private List> generateTabletResultSetForTable( ValueConverter.convert( sourceType, targetType, - DateUtils.parseDateExpressionToInt(((LocalDate[]) values[j])[i])); + tablet.bitMaps[j].isMarked(i) + ? null + : DateUtils.parseDateExpressionToInt(((LocalDate[]) values[j])[i])); insertRecord.add(convert(value, targetType)); break; case TEXT: case STRING: - value = ValueConverter.convert(sourceType, targetType, ((Binary[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((Binary[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case BLOB: - value = ValueConverter.convert(sourceType, targetType, ((Binary[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((Binary[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case BOOLEAN: - value = ValueConverter.convert(sourceType, targetType, ((boolean[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((boolean[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; } } + insertRecord.add(tablet.timestamps[i]); insertRecords.add(insertRecord); } @@ -391,12 +421,15 @@ private List> generateTabletResultSetForTable( } private Object convert(Object value, TSDataType targetType) { + if (value == null) { + return null; + } switch (targetType) { case DATE: return DateUtils.parseIntToLocalDate((Integer) value); case TEXT: case STRING: - return new String(((Binary) value).getValues(), TSFileConfig.STRING_CHARSET); + return value; } return value; } @@ -417,7 +450,7 @@ private Tablet generateTabletAndMeasurementSchema( columnTypes, generateDataSize); tablet.initBitMaps(); - tablet.timestamps = createTestDataForTimestamp(); + createTestDataForTimeColumn(tablet); for (int i = 0; i < pairs.size(); i++) { MeasurementSchema schema = pairs.get(i).left; switch (schema.getType()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java index f249a7734552..edf8b636ecf3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java @@ -28,10 +28,14 @@ import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.ZoneId; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; public class PipeConvertedInsertRowStatement extends InsertRowStatement { @@ -43,7 +47,6 @@ public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStateme // Statement isDebug = insertRowStatement.isDebug(); // InsertBaseStatement - insertRowStatement.removeAllFailedMeasurementMarks(); devicePath = insertRowStatement.getDevicePath(); isAligned = insertRowStatement.isAligned(); measurementSchemas = insertRowStatement.getMeasurementSchemas(); @@ -59,6 +62,31 @@ public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStateme values = insertRowStatement.getValues(); isNeedInferType = insertRowStatement.isNeedInferType(); deviceID = insertRowStatement.getRawTableDeviceID(); + + // To ensure that the measurement remains unchanged during the WAL writing process, the array + // needs to be copied before the failed Measurement mark can be deleted. + final MeasurementSchema[] measurementSchemas = insertRowStatement.getMeasurementSchemas(); + if (measurementSchemas != null) { + this.measurementSchemas = Arrays.copyOf(measurementSchemas, measurementSchemas.length); + } + + final String[] measurements = insertRowStatement.getMeasurements(); + if (measurements != null) { + this.measurements = Arrays.copyOf(measurements, measurements.length); + } + + final TSDataType[] dataTypes = insertRowStatement.getDataTypes(); + if (dataTypes != null) { + this.dataTypes = Arrays.copyOf(dataTypes, dataTypes.length); + } + + final Map failedMeasurementIndex2Info = + insertRowStatement.getFailedMeasurementInfoMap(); + if (failedMeasurementIndex2Info != null) { + this.failedMeasurementIndex2Info = new HashMap<>(failedMeasurementIndex2Info); + } + + removeAllFailedMeasurementMarks(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java index adc19ae7e0fe..eee47a5b0b18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java @@ -24,25 +24,27 @@ import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + public class PipeConvertedInsertTabletStatement extends InsertTabletStatement { private static final Logger LOGGER = LoggerFactory.getLogger(PipeConvertedInsertTabletStatement.class); - public PipeConvertedInsertTabletStatement(final InsertTabletStatement insertTabletStatement) { + public PipeConvertedInsertTabletStatement( + final InsertTabletStatement insertTabletStatement, boolean isCopyMeasurement) { super(); // Statement isDebug = insertTabletStatement.isDebug(); // InsertBaseStatement - insertTabletStatement.removeAllFailedMeasurementMarks(); devicePath = insertTabletStatement.getDevicePath(); isAligned = insertTabletStatement.isAligned(); - measurementSchemas = insertTabletStatement.getMeasurementSchemas(); - measurements = insertTabletStatement.getMeasurements(); - dataTypes = insertTabletStatement.getDataTypes(); columnCategories = insertTabletStatement.getColumnCategories(); idColumnIndices = insertTabletStatement.getIdColumnIndices(); attrColumnIndices = insertTabletStatement.getAttrColumnIndices(); @@ -55,6 +57,42 @@ public PipeConvertedInsertTabletStatement(final InsertTabletStatement insertTabl deviceIDs = insertTabletStatement.getRawTableDeviceIDs(); singleDevice = insertTabletStatement.isSingleDevice(); rowCount = insertTabletStatement.getRowCount(); + + // To ensure that the measurement remains unchanged during the WAL writing process, the array + // needs to be copied before the failed Measurement mark can be deleted. + if (isCopyMeasurement) { + final MeasurementSchema[] measurementSchemas = insertTabletStatement.getMeasurementSchemas(); + if (measurementSchemas != null) { + this.measurementSchemas = Arrays.copyOf(measurementSchemas, measurementSchemas.length); + } + + final String[] measurements = insertTabletStatement.getMeasurements(); + if (measurements != null) { + this.measurements = Arrays.copyOf(measurements, measurements.length); + } + + final TSDataType[] dataTypes = insertTabletStatement.getDataTypes(); + if (dataTypes != null) { + this.dataTypes = Arrays.copyOf(dataTypes, dataTypes.length); + } + + final Map failedMeasurementIndex2Info = + insertTabletStatement.getFailedMeasurementInfoMap(); + if (failedMeasurementIndex2Info != null) { + this.failedMeasurementIndex2Info = new HashMap<>(failedMeasurementIndex2Info); + } + } else { + this.measurementSchemas = insertTabletStatement.getMeasurementSchemas(); + this.measurements = insertTabletStatement.getMeasurements(); + this.dataTypes = insertTabletStatement.getDataTypes(); + this.failedMeasurementIndex2Info = insertTabletStatement.getFailedMeasurementInfoMap(); + } + + removeAllFailedMeasurementMarks(); + } + + public PipeConvertedInsertTabletStatement(final InsertTabletStatement insertTabletStatement) { + this(insertTabletStatement, true); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java index 11952814da09..11858853a2da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java @@ -141,7 +141,8 @@ public Optional visitLoadFile( rawTabletInsertionEvent.convertToTablet(), rawTabletInsertionEvent.isAligned(), databaseName) - .constructStatement()); + .constructStatement(), + false); TSStatus result; try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java index eef40c714dc8..ed47e519b00e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java @@ -106,7 +106,8 @@ file, new IoTDBTreePattern(null), Long.MIN_VALUE, Long.MAX_VALUE, null, null)) { new PipeConvertedInsertTabletStatement( PipeTransferTabletRawReq.toTPipeTransferRawReq( tabletWithIsAligned.getLeft(), tabletWithIsAligned.getRight()) - .constructStatement()); + .constructStatement(), + false); TSStatus result; try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java index eb6da0bc2ffa..7ae7b0232005 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java @@ -341,6 +341,10 @@ public List getFailedMeasurements() { .collect(Collectors.toList()); } + public Map getFailedMeasurementInfoMap() { + return failedMeasurementIndex2Info; + } + public List getFailedExceptions() { return failedMeasurementIndex2Info == null ? Collections.emptyList() @@ -364,7 +368,7 @@ public List getFailedMessages() { .collect(Collectors.toList()); } - protected static class FailedMeasurementInfo { + public static class FailedMeasurementInfo { protected String measurement; protected TSDataType dataType; protected Object value; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index 85f82ff447db..d33a83b0c326 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -276,13 +276,25 @@ public void removeAllFailedMeasurementMarks() { } failedMeasurementIndex2Info.forEach( (index, info) -> { - measurements[index] = info.getMeasurement(); - dataTypes[index] = info.getDataType(); - values[index] = info.getValue(); + if (measurements != null) { + measurements[index] = info.getMeasurement(); + } + + if (dataTypes != null) { + dataTypes[index] = info.getDataType(); + } + + if (values != null) { + values[index] = info.getValue(); + } }); failedMeasurementIndex2Info.clear(); } + public Map getFailedMeasurementInfoMap() { + return failedMeasurementIndex2Info; + } + @Override public void semanticCheck() { super.semanticCheck(); From a5e13fd8f7c1624d20f1628405bcca92c9900ed5 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 10 Jan 2025 17:31:26 +0800 Subject: [PATCH 12/13] Pipe: report linked tsfile size & Subscription: decrease reference count for other enriched events & add logging to observe possible stuck situations (#14668) --- .../tsfile/PipeTsFileResourceManager.java | 5 ++++- .../broker/SubscriptionPrefetchingQueue.java | 2 ++ .../subscription/event/SubscriptionEvent.java | 21 +++++++++++------- .../SubscriptionPipeTabletEventBatch.java | 22 +++++++++++++++++++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java index 9607c41a94b7..2bdfda64ef41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java @@ -98,7 +98,10 @@ private void ttlCheck() throws InterruptedException { } else { logBuilder.append( String.format( - "<%s , %d times> ", entry.getKey(), entry.getValue().getReferenceCount())); + "<%s , %d times, %d bytes> ", + entry.getKey(), + entry.getValue().getReferenceCount(), + entry.getValue().getFileSize())); } } catch (final IOException e) { LOGGER.warn("failed to close PipeTsFileResource when checking TTL: ", e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index 20a0d0983f05..5afbf2128293 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -357,6 +357,8 @@ private void tryPrefetch() { "Subscription: SubscriptionPrefetchingQueue {} ignore EnrichedEvent {} when prefetching.", this, event); + ((EnrichedEvent) event) + .decreaseReferenceCount(SubscriptionPrefetchingQueue.class.getName(), false); if (onEvent()) { return; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java index 97fa621faefc..2db5bc7a4b02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java @@ -65,6 +65,9 @@ public class SubscriptionEvent { // record file name for file payload private String fileName; + private static final long NACK_COUNT_REPORT_THRESHOLD = 3; + private final AtomicLong nackCount = new AtomicLong(); + /** * Constructs a {@link SubscriptionEvent} with the response type of {@link * SubscriptionEventSingleResponse}. @@ -143,11 +146,11 @@ public boolean isCommittable() { } public void ack(final Consumer onCommittedHook) { - // ack response - response.ack(onCommittedHook); - - // ack pipe events + // NOTE: we should ack pipe events before ack response since multiple events may reuse the same + // batch (as pipe events) + // TODO: consider more elegant design for this method pipeEvents.ack(); + response.ack(onCommittedHook); } /** @@ -156,11 +159,8 @@ public void ack(final Consumer onCommittedHook) { * SubscriptionPrefetchingQueue} or {@link SubscriptionPrefetchingQueue#cleanUp}. */ public void cleanUp() { - // reset serialized responses - response.cleanUp(); - - // clean up pipe events pipeEvents.cleanUp(); + response.cleanUp(); // TODO: clean more fields } @@ -216,6 +216,11 @@ public void nack() { // reset lastPolledTimestamp makes this event pollable lastPolledTimestamp.set(INVALID_TIMESTAMP); + + // record nack count + if (nackCount.getAndIncrement() > NACK_COUNT_REPORT_THRESHOLD) { + LOGGER.warn("{} has been nacked {} times", this, nackCount); + } } public void recordLastPolledConsumerId(final String consumerId) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java index 59528125f497..1031e1e22796 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; public class SubscriptionPipeTabletEventBatch extends SubscriptionPipeEventBatch implements Iterator> { @@ -62,6 +63,10 @@ public class SubscriptionPipeTabletEventBatch extends SubscriptionPipeEventBatch private final List iteratedEnrichedEvents = new ArrayList<>(); + private static final long ITERATED_COUNT_REPORT_FREQ = + 30000; // based on the full parse of a 128MB tsfile estimate + private final AtomicLong iteratedCount = new AtomicLong(); + public SubscriptionPipeTabletEventBatch( final int regionId, final SubscriptionPrefetchingTabletQueue prefetchingQueue, @@ -230,6 +235,23 @@ public boolean hasNext() { @Override public List next() { + final List tablets = nextInternal(); + if (Objects.isNull(tablets)) { + return null; + } + if (iteratedCount.incrementAndGet() % ITERATED_COUNT_REPORT_FREQ == 0) { + LOGGER.info( + "{} has been iterated {} times, current TsFileInsertionEvent {}", + this, + iteratedCount, + Objects.isNull(currentTsFileInsertionEvent) + ? "" + : ((EnrichedEvent) currentTsFileInsertionEvent).coreReportMessage()); + } + return tablets; + } + + private List nextInternal() { if (Objects.nonNull(currentTabletInsertionEventsIterator)) { if (currentTabletInsertionEventsIterator.hasNext()) { final TabletInsertionEvent tabletInsertionEvent = From c8d7c5dfea9afeb8883302fa0786034169a404cf Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 10 Jan 2025 17:36:15 +0800 Subject: [PATCH 13/13] Reopen drop column and delete devices operations --- .../relational/it/schema/IoTDBDeviceIT.java | 37 +++--- .../relational/it/schema/IoTDBTableIT.java | 122 +++++++++--------- .../persistence/schema/ConfigMTree.java | 2 +- .../plan/relational/sql/ast/DeleteDevice.java | 2 - .../plan/relational/sql/ast/DropColumn.java | 3 - pom.xml | 2 +- 6 files changed, 77 insertions(+), 91 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java index d2479f68f351..578d807a6dde 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java @@ -249,29 +249,26 @@ public void testDevice() throws SQLException { TestUtils.assertResultSetSize( statement.executeQuery("show devices from table0 offset 1 limit 1"), 1); - // TODO: Reopen - if (false) { - // Test delete devices - statement.execute("delete devices from table0 where region_id = '1' and plant_id = '木兰'"); - TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 1); + // Test delete devices + statement.execute("delete devices from table0 where region_id = '1' and plant_id = '木兰'"); + TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 1); - // Test successfully Invalidate cache - statement.execute( - "insert into table0(region_id, plant_id, device_id, model, temperature, humidity) values('1', '木兰', '3', 'A', 37.6, 111.1)"); - TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 2); + // Test successfully Invalidate cache + statement.execute( + "insert into table0(region_id, plant_id, device_id, model, temperature, humidity) values('1', '木兰', '3', 'A', 37.6, 111.1)"); + TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 2); - // Test successfully delete data - TestUtils.assertResultSetSize( - statement.executeQuery("select * from table0 where region_id = '1'"), 1); + // Test successfully delete data + TestUtils.assertResultSetSize( + statement.executeQuery("select * from table0 where region_id = '1'"), 1); - try { - statement.executeQuery("delete devices from table0 where time = 1"); - fail("Delete devices shall fail when specifies non tag column"); - } catch (final Exception e) { - assertEquals( - "701: The TIME/FIELD columns are currently not allowed in devices related operations", - e.getMessage()); - } + try { + statement.executeQuery("delete devices from table0 where time = 1"); + fail("Delete devices shall fail when specifies non tag column"); + } catch (final Exception e) { + assertEquals( + "701: The TIME/FIELD columns are currently not allowed in devices related operations", + e.getMessage()); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index 1bfd554dde26..d27e298f8776 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -390,70 +390,67 @@ public void testManageTable() { statement.execute( "insert into table2(region_id, plant_id, color, temperature, speed) values(1, 1, 1, 1, 1)"); - // TODO: Reopen - if (false) { - // Test drop column - statement.execute("alter table table2 drop column color"); - - columnNames = new String[] {"time", "region_id", "plant_id", "temperature", "speed"}; - dataTypes = new String[] {"TIMESTAMP", "STRING", "STRING", "FLOAT", "DOUBLE"}; - categories = new String[] {"TIME", "TAG", "TAG", "FIELD", "FIELD"}; - final String[] statuses = new String[] {"USING", "USING", "USING", "USING", "USING"}; - try (final ResultSet resultSet = statement.executeQuery("describe table2 details")) { - int cnt = 0; - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(describeTableDetailsColumnHeaders.size(), metaData.getColumnCount()); - for (int i = 0; i < describeTableDetailsColumnHeaders.size(); i++) { - assertEquals( - describeTableDetailsColumnHeaders.get(i).getColumnName(), - metaData.getColumnName(i + 1)); - } - while (resultSet.next()) { - assertEquals(columnNames[cnt], resultSet.getString(1)); - assertEquals(dataTypes[cnt], resultSet.getString(2)); - assertEquals(categories[cnt], resultSet.getString(3)); - assertEquals(statuses[cnt], resultSet.getString(4)); - cnt++; - } - assertEquals(columnNames.length, cnt); + // Test drop column + statement.execute("alter table table2 drop column color"); + + columnNames = new String[] {"time", "region_id", "plant_id", "temperature", "speed"}; + dataTypes = new String[] {"TIMESTAMP", "STRING", "STRING", "FLOAT", "DOUBLE"}; + categories = new String[] {"TIME", "TAG", "TAG", "FIELD", "FIELD"}; + final String[] statuses = new String[] {"USING", "USING", "USING", "USING", "USING"}; + try (final ResultSet resultSet = statement.executeQuery("describe table2 details")) { + int cnt = 0; + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(describeTableDetailsColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < describeTableDetailsColumnHeaders.size(); i++) { + assertEquals( + describeTableDetailsColumnHeaders.get(i).getColumnName(), + metaData.getColumnName(i + 1)); } - - statement.execute("alter table table2 drop column speed"); - - try { - statement.executeQuery("select color from table2"); - fail(); - } catch (final SQLException e) { - assertEquals("616: Column 'color' cannot be resolved", e.getMessage()); + while (resultSet.next()) { + assertEquals(columnNames[cnt], resultSet.getString(1)); + assertEquals(dataTypes[cnt], resultSet.getString(2)); + assertEquals(categories[cnt], resultSet.getString(3)); + assertEquals(statuses[cnt], resultSet.getString(4)); + cnt++; } + assertEquals(columnNames.length, cnt); + } - try { - statement.executeQuery("select speed from table2"); - fail(); - } catch (final SQLException e) { - assertEquals("616: Column 'speed' cannot be resolved", e.getMessage()); - } + statement.execute("alter table table2 drop column speed"); - try { - statement.execute("alter table table2 drop column speed"); - } catch (final SQLException e) { - assertEquals("616: Column speed in table 'test2.table2' does not exist.", e.getMessage()); - } + try { + statement.executeQuery("select color from table2"); + fail(); + } catch (final SQLException e) { + assertEquals("616: Column 'color' cannot be resolved", e.getMessage()); + } - try { - statement.execute("alter table table2 drop column time"); - } catch (final SQLException e) { - assertEquals("701: Dropping tag or time column is not supported.", e.getMessage()); - } + try { + statement.executeQuery("select speed from table2"); + fail(); + } catch (final SQLException e) { + assertEquals("616: Column 'speed' cannot be resolved", e.getMessage()); + } - // test data deletion by drop column - statement.execute("alter table table2 add column speed double"); - TestUtils.assertResultSetEqual( - statement.executeQuery("select speed from table2"), - "speed,", - Collections.singleton("null,")); + try { + statement.execute("alter table table2 drop column speed"); + } catch (final SQLException e) { + assertEquals("616: Column speed in table 'test2.table2' does not exist.", e.getMessage()); } + try { + statement.execute("alter table table2 drop column time"); + } catch (final SQLException e) { + assertEquals("701: Dropping tag or time column is not supported.", e.getMessage()); + } + + // test data deletion by drop column + statement.execute("alter table table2 add column speed double"); + TestUtils.assertResultSetEqual( + statement.executeQuery("select speed from table2"), + "speed,", + Collections.singleton("null,")); + statement.execute("drop table table2"); try { statement.executeQuery("describe table2"); @@ -504,14 +501,11 @@ public void testManageTable() { assertEquals("500: Unknown database test1", e.getMessage()); } - // TODO: Reopen - if (false) { - try { - statement.execute("alter table test1.test drop column a"); - fail(); - } catch (final SQLException e) { - assertEquals("500: Unknown database test1", e.getMessage()); - } + try { + statement.execute("alter table test1.test drop column a"); + fail(); + } catch (final SQLException e) { + assertEquals("500: Unknown database test1", e.getMessage()); } try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index 0b4bf3f840c5..ff91998a8874 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -866,7 +866,7 @@ public boolean preDeleteColumn( } if (columnSchema.getColumnCategory() == TsTableColumnCategory.TAG || columnSchema.getColumnCategory() == TsTableColumnCategory.TIME) { - throw new SemanticException("Dropping id or time column is not supported."); + throw new SemanticException("Dropping tag or time column is not supported."); } node.addPreDeletedColumn(columnName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java index 79bf8147da35..6d3f3438f087 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.table.TsTable; -import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.SessionInfo; import org.apache.iotdb.db.queryengine.execution.operator.schema.source.DeviceBlackListConstructor; import org.apache.iotdb.db.queryengine.execution.operator.schema.source.TableDeviceQuerySource; @@ -68,7 +67,6 @@ public class DeleteDevice extends AbstractTraverseDevice { public DeleteDevice(final NodeLocation location, final Table table, final Expression where) { super(location, table, where); - throw new SemanticException("Delete device is unsupported yet."); } public void parseModEntries(final TsTable table) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java index 1cad3f8f3a3c..20e180530889 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; -import org.apache.iotdb.db.exception.sql.SemanticException; - import com.google.common.collect.ImmutableList; import java.util.List; @@ -48,7 +46,6 @@ public DropColumn( this.field = requireNonNull(field, "field is null"); this.tableIfExists = tableIfExists; this.columnIfExists = columnIfExists; - throw new SemanticException("Drop column is unsupported yet."); } public QualifiedName getTable() { diff --git a/pom.xml b/pom.xml index 28f72ddc9bcc..e278d579b445 100644 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,7 @@ 0.14.1 1.9 1.5.6-3 - 1.2.0-241224-SNAPSHOT + 2.0.0-250109-SNAPSHOT