From 1406eadd9bfa84d6e4f07921e27486d6cc570bbd Mon Sep 17 00:00:00 2001 From: Rohit Menon Date: Fri, 23 Apr 2021 11:10:50 -0700 Subject: [PATCH 1/9] Use custom hive-exec --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3544be9a9ae46..36d87a50ab1e0 100644 --- a/pom.xml +++ b/pom.xml @@ -1715,7 +1715,7 @@ ${hive.group} hive-exec ${hive.classifier} - ${hive.version} + 2.3.6.39 ${hive.deps.scope} From 91d3abb4d4afbb667f15f2823b58b17e2ea6b313 Mon Sep 17 00:00:00 2001 From: Rohit Menon Date: Fri, 23 Apr 2021 13:57:19 -0700 Subject: [PATCH 2/9] Use custom hive-exec --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 36d87a50ab1e0..5394ee3be51d9 100644 --- a/pom.xml +++ b/pom.xml @@ -128,8 +128,8 @@ org.apache.hive core - 2.3.7 - 2.3.7 + 2.3.6.39 + 2.3.6.39 2.3 @@ -1715,7 +1715,7 @@ ${hive.group} hive-exec ${hive.classifier} - 2.3.6.39 + ${hive.version} ${hive.deps.scope} From 917177643bf00531b6df55649964e0512ed36024 Mon Sep 17 00:00:00 2001 From: Rohit Menon Date: Mon, 26 Apr 2021 17:17:24 -0700 Subject: [PATCH 3/9] Operation log fix --- .../hive/service/cli/operation/OperationManager.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java index 3df842d2b4af9..dac4015501cb7 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java @@ -249,6 +249,10 @@ public RowSet getOperationNextRowSet(OperationHandle opHandle, public RowSet getOperationLogRowSet(OperationHandle opHandle, FetchOrientation orientation, long maxRows) throws HiveSQLException { + TableSchema tableSchema = new TableSchema(getLogSchema()); + RowSet rowSet = RowSetFactory.create(tableSchema, + getOperation(opHandle).getProtocolVersion(), false); + // get the OperationLog object from the operation OperationLog operationLog = getOperation(opHandle).getOperationLog(); if (operationLog == null) { @@ -257,17 +261,14 @@ public RowSet getOperationLogRowSet(OperationHandle opHandle, // read logs List logs; + rowSet.setStartOffset(operationLog.getStartPosition(isFetchFirst(orientation))); try { logs = operationLog.readOperationLog(isFetchFirst(orientation), maxRows); } catch (SQLException e) { throw new HiveSQLException(e.getMessage(), e.getCause()); } - // convert logs to RowSet - TableSchema tableSchema = new TableSchema(getLogSchema()); - RowSet rowSet = RowSetFactory.create(tableSchema, - getOperation(opHandle).getProtocolVersion(), false); for (String log : logs) { rowSet.addRow(new String[] {log}); } From 0ebe3f7e5e0c7156554c6afd776f57e9f208848b Mon Sep 17 00:00:00 2001 From: Rohit Menon Date: Mon, 26 Apr 2021 18:09:56 -0700 Subject: [PATCH 4/9] Operation log fix --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5394ee3be51d9..3544be9a9ae46 100644 --- a/pom.xml +++ b/pom.xml @@ -128,8 +128,8 @@ org.apache.hive core - 2.3.6.39 - 2.3.6.39 + 2.3.7 + 2.3.7 2.3 From c746fc7fb8766df519bbbfc3191e947686d7d8ac Mon Sep 17 00:00:00 2001 From: Rohit Menon Date: Mon, 26 Apr 2021 18:11:13 -0700 Subject: [PATCH 5/9] Operation log fix --- .../org/apache/hive/service/cli/operation/SQLOperation.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java index 894793152f409..626844568edda 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -329,6 +329,11 @@ public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws H */ if (orientation.equals(FetchOrientation.FETCH_FIRST) && fetchStarted) { driver.resetFetch(); + rowSet.setStartOffset(0); + } + if (fetchTask != null) + { + rowSet.setStartOffset(fetchTask.totalRows); } fetchStarted = true; driver.setMaxRows((int) maxRows); From c157ec0c2fccfe23590b5ae329f873de8e19217f Mon Sep 17 00:00:00 2001 From: Rohit Menon Date: Mon, 26 Apr 2021 18:19:36 -0700 Subject: [PATCH 6/9] Operation log fix --- .../apache/hive/service/cli/ColumnBasedSet.java | 14 +++++++++----- .../cli/operation/HiveCommandOperation.java | 5 ++++- .../hive/service/cli/operation/SQLOperation.java | 10 ++++------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java index 3ca18f0cdb2f9..2dc2301ddef81 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java @@ -48,11 +48,15 @@ public class ColumnBasedSet implements RowSet { public static final Logger LOG = LoggerFactory.getLogger(ColumnBasedSet.class); public ColumnBasedSet(TableSchema schema) { - descriptors = schema.toTypeDescriptors(); - columns = new ArrayList(); - for (ColumnDescriptor colDesc : schema.getColumnDescriptors()) { - columns.add(new ColumnBuffer(colDesc.getType())); - } + if (schema == null) { + descriptors = new TypeDescriptor[0]; + columns = new ArrayList(); + } else { + descriptors = schema.toTypeDescriptors(); + columns = new ArrayList(); + for (ColumnDescriptor colDesc : schema.getColumnDescriptors()) { + columns.add(new ColumnBuffer(colDesc.getType())); + } } public ColumnBasedSet(TRowSet tRowSet) throws TException { diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java index 5d3e0c91eafa6..ee46ed1de440a 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java @@ -51,6 +51,7 @@ public class HiveCommandOperation extends ExecuteStatementOperation { private CommandProcessor commandProcessor; private TableSchema resultSchema = null; + private int readRows = 0; /** * For processors other than Hive queries (Driver), they output to session.out (a temp file) @@ -157,10 +158,11 @@ public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws H } List rows = readResults((int) maxRows); RowSet rowSet = RowSetFactory.create(resultSchema, getProtocolVersion(), false); - + rowSet.setStartOffset(readRows) for (String row : rows) { rowSet.addRow(new String[] {row}); } + readRows += rows.size() return rowSet; } @@ -211,5 +213,6 @@ private void resetResultReader() { ServiceUtils.cleanup(LOG, resultReader); resultReader = null; } + readRows = 0; } } diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java index 626844568edda..cddff0fca11a1 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -329,17 +329,15 @@ public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws H */ if (orientation.equals(FetchOrientation.FETCH_FIRST) && fetchStarted) { driver.resetFetch(); - rowSet.setStartOffset(0); - } - if (fetchTask != null) - { - rowSet.setStartOffset(fetchTask.totalRows); } fetchStarted = true; driver.setMaxRows((int) maxRows); if (driver.getResults(convey)) { - return decode(convey, rowSet); + decode(convey, rowSet); } + long startRowOffset = driver.getStartRowOffset(); + rowSet.setStartOffset(startRowOffset); + driver.setStartRowOffset(startRowOffset + rowSet.numRows()); return rowSet; } catch (IOException e) { throw new HiveSQLException(e); From ebfe8980ebaf45d149e2b7037b7d1072d13d72aa Mon Sep 17 00:00:00 2001 From: Rohit Menon Date: Tue, 27 Apr 2021 08:58:38 -0700 Subject: [PATCH 7/9] Operation log fix --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3544be9a9ae46..36d87a50ab1e0 100644 --- a/pom.xml +++ b/pom.xml @@ -1715,7 +1715,7 @@ ${hive.group} hive-exec ${hive.classifier} - ${hive.version} + 2.3.6.39 ${hive.deps.scope} From ac49c8a0e02cda73204d82882aff1f5b07879a83 Mon Sep 17 00:00:00 2001 From: Rohit Menon Date: Tue, 27 Apr 2021 09:33:02 -0700 Subject: [PATCH 8/9] Operation log fix --- .../main/java/org/apache/hive/service/cli/ColumnBasedSet.java | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java index 2dc2301ddef81..36bb3d75ea424 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java @@ -57,6 +57,7 @@ public ColumnBasedSet(TableSchema schema) { for (ColumnDescriptor colDesc : schema.getColumnDescriptors()) { columns.add(new ColumnBuffer(colDesc.getType())); } + } } public ColumnBasedSet(TRowSet tRowSet) throws TException { From 8b32ffc2209cb667575782edf1e448428b0d9aef Mon Sep 17 00:00:00 2001 From: Rohit Menon Date: Tue, 27 Apr 2021 10:04:46 -0700 Subject: [PATCH 9/9] Operation log fix --- .../hive/service/cli/operation/HiveCommandOperation.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java index ee46ed1de440a..e2ae067161694 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java @@ -158,11 +158,11 @@ public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws H } List rows = readResults((int) maxRows); RowSet rowSet = RowSetFactory.create(resultSchema, getProtocolVersion(), false); - rowSet.setStartOffset(readRows) + rowSet.setStartOffset(readRows); for (String row : rows) { rowSet.addRow(new String[] {row}); } - readRows += rows.size() + readRows += rows.size(); return rowSet; }