From 71921c5146f1e2f9d6f260647853c1954d3baaa7 Mon Sep 17 00:00:00 2001 From: XiaoHongbo <1346652787@qq.com> Date: Mon, 30 Dec 2024 13:35:35 +0800 Subject: [PATCH] [hive] Fix listTablesImpl possible timeout issue (#4800) --- .../org/apache/paimon/hive/HiveCatalog.java | 54 ++++++++++++++++++- .../apache/paimon/hive/HiveCatalogTest.java | 53 ++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index e29d83db4536..a213909beb10 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -54,6 +54,7 @@ import org.apache.paimon.view.View; import org.apache.paimon.view.ViewImpl; +import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; import org.apache.flink.table.hive.LegacyHiveClasses; @@ -143,7 +144,7 @@ public class HiveCatalog extends AbstractCatalog { private static final String HIVE_PREFIX = "hive."; public static final String HIVE_SITE_FILE = "hive-site.xml"; private static final String HIVE_EXTERNAL_TABLE_PROP = "EXTERNAL"; - + private static final int DEFAULT_TABLE_BATCH_SIZE = 300; private final HiveConf hiveConf; private final String clientClassName; private final Options options; @@ -442,8 +443,34 @@ protected void alterDatabaseImpl(String name, List changes) { protected List listTablesImpl(String databaseName) { try { List tableNames = clients.run(client -> client.getAllTables(databaseName)); + int batchSize = getBatchGetTableSize(); List hmsTables = - clients.run(client -> client.getTableObjectsByName(databaseName, tableNames)); + Lists.partition(tableNames, batchSize).stream() + .flatMap( + batchTableNames -> { + try { + return clients + .run( + client -> + client.getTableObjectsByName( + databaseName, + batchTableNames)) + .stream(); + } catch (TException e) { + throw new RuntimeException( + "Failed to getTableObjectsByName in database " + + databaseName, + e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException( + "Interrupted in call to getTableObjectsByName " + + databaseName, + e); + } + }) + .collect(Collectors.toList()); + List result = new ArrayList<>(hmsTables.size()); for (Table table : hmsTables) { if (isPaimonTable(table) || (!formatTableDisabled() && isFormatTable(table))) { @@ -1414,4 +1441,27 @@ public static HiveConf createHiveConf(CatalogContext context) { public static String possibleHiveConfPath() { return System.getenv("HIVE_CONF_DIR"); } + + public int getBatchGetTableSize() { + try { + int size = + Integer.parseInt( + this.hiveConf.get( + HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname, + String.valueOf( + HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX + .getDefaultValue()))); + if (size < 1) { + return DEFAULT_TABLE_BATCH_SIZE; + } else { + return size; + } + } catch (Exception e) { + LOG.warn( + "parse batch size failed {}, use default batch size", + this.hiveConf.get(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname), + e); + return DEFAULT_TABLE_BATCH_SIZE; + } + } } diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index bf6eb02f3e55..e733ec16c839 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -59,6 +59,7 @@ import static org.apache.paimon.hive.HiveCatalog.TABLE_TYPE_PROP; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; /** Tests for {@link HiveCatalog}. */ @@ -354,6 +355,58 @@ public void testListTablesLock() { } } + @Test + public void testListTables() throws Exception { + String databaseName = "testListTables"; + catalog.dropDatabase(databaseName, true, true); + catalog.createDatabase(databaseName, true); + for (int i = 0; i < 500; i++) { + catalog.createTable( + Identifier.create(databaseName, "table" + i), + Schema.newBuilder().column("col", DataTypes.INT()).build(), + true); + } + + // use default 300 + List defaultBatchTables = catalog.listTables(databaseName); + + // use custom 400 + HiveConf hiveConf = new HiveConf(); + hiveConf.set(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname, "400"); + String metastoreClientClass = "org.apache.hadoop.hive.metastore.HiveMetaStoreClient"; + List customBatchTables; + try (HiveCatalog customCatalog = + new HiveCatalog(fileIO, hiveConf, metastoreClientClass, warehouse)) { + customBatchTables = customCatalog.listTables(databaseName); + } catch (Exception e) { + throw e; + } + assertEquals(defaultBatchTables.size(), customBatchTables.size()); + defaultBatchTables.sort(String::compareTo); + customBatchTables.sort(String::compareTo); + for (int i = 0; i < defaultBatchTables.size(); i++) { + assertEquals(defaultBatchTables.get(i), customBatchTables.get(i)); + } + + // use invalid batch size + HiveConf invalidHiveConf = new HiveConf(); + invalidHiveConf.set(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname, "dummy"); + List invalidBatchSizeTables; + try (HiveCatalog invalidBatchSizeCatalog = + new HiveCatalog(fileIO, invalidHiveConf, metastoreClientClass, warehouse)) { + invalidBatchSizeTables = invalidBatchSizeCatalog.listTables(databaseName); + } catch (Exception e) { + throw e; + } + assertEquals(defaultBatchTables.size(), invalidBatchSizeTables.size()); + invalidBatchSizeTables.sort(String::compareTo); + for (int i = 0; i < defaultBatchTables.size(); i++) { + assertEquals(defaultBatchTables.get(i), invalidBatchSizeTables.get(i)); + } + + catalog.dropDatabase(databaseName, true, true); + } + @Override protected boolean supportsView() { return true;