From 6c12b4a04ce5cdd2bf7d7f91a5b0c13651df9e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E7=A5=A5=E5=B9=B3?= <408317717@qq.com> Date: Tue, 23 Jul 2024 21:38:27 +0800 Subject: [PATCH] Add hive conf cache to CloseableThriftHiveMetastoreIfaceClientFactory (#327) * Add hive conf cache --- CHANGELOG.md | 3 +++ .../CloseableThriftHiveMetastoreIfaceClientFactory.java | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 800155cf..0dca1c13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## [3.13.2] - 2024-07-23 +### Fix +- Add HiveConf cache to `CloseableThriftHiveMetastoreIfaceClientFactory` to prevent threads block. See [#325](https://github.com/ExpediaGroup/waggle-dance/issues/325) ## [3.13.1] - 2024-05-06 ### Fix diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java index 73128f0b..daec482e 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java @@ -21,8 +21,10 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -45,6 +47,7 @@ public class CloseableThriftHiveMetastoreIfaceClientFactory implements ThriftCli private final WaggleDanceConfiguration waggleDanceConfiguration; private final GlueClientFactory glueClientFactory; private final SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory; + private final ConcurrentHashMap cachedHiveConf = new ConcurrentHashMap<>(); public CloseableThriftHiveMetastoreIfaceClientFactory( TunnelingMetaStoreClientFactory tunnelingMetaStoreClientFactory, @@ -96,8 +99,9 @@ private CloseableThriftHiveMetastoreIface newHiveInstance( } properties.put(ConfVars.METASTOREURIS.varname, uris); HiveConfFactory confFactory = new HiveConfFactory(Collections.emptyList(), properties); + HiveConf hiveConf = cachedHiveConf.computeIfAbsent(uris, t-> confFactory.newInstance()); return defaultMetaStoreClientFactory - .newInstance(confFactory.newInstance(), "waggledance-" + name, DEFAULT_CLIENT_FACTORY_RECONNECTION_RETRY, + .newInstance(hiveConf, "waggledance-" + name, DEFAULT_CLIENT_FACTORY_RECONNECTION_RETRY, connectionTimeout); }