diff --git a/CHANGELOG.md b/CHANGELOG.md index 800155cf..0dca1c13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## [3.13.2] - 2024-07-23 +### Fix +- Add HiveConf cache to `CloseableThriftHiveMetastoreIfaceClientFactory` to prevent threads block. See [#325](https://github.com/ExpediaGroup/waggle-dance/issues/325) ## [3.13.1] - 2024-05-06 ### Fix diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java index 73128f0b..daec482e 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java @@ -21,8 +21,10 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -45,6 +47,7 @@ public class CloseableThriftHiveMetastoreIfaceClientFactory implements ThriftCli private final WaggleDanceConfiguration waggleDanceConfiguration; private final GlueClientFactory glueClientFactory; private final SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory; + private final ConcurrentHashMap cachedHiveConf = new ConcurrentHashMap<>(); public CloseableThriftHiveMetastoreIfaceClientFactory( TunnelingMetaStoreClientFactory tunnelingMetaStoreClientFactory, @@ -96,8 +99,9 @@ private CloseableThriftHiveMetastoreIface newHiveInstance( } properties.put(ConfVars.METASTOREURIS.varname, uris); HiveConfFactory confFactory = new HiveConfFactory(Collections.emptyList(), properties); + HiveConf hiveConf = cachedHiveConf.computeIfAbsent(uris, t-> confFactory.newInstance()); return defaultMetaStoreClientFactory - .newInstance(confFactory.newInstance(), "waggledance-" + name, DEFAULT_CLIENT_FACTORY_RECONNECTION_RETRY, + .newInstance(hiveConf, "waggledance-" + name, DEFAULT_CLIENT_FACTORY_RECONNECTION_RETRY, connectionTimeout); }