Switch to use unshaded hive-exec for io expansion service

* This enables the shadow jar pick up dependencies of newer versions

.github/trigger_files/IO_Iceberg_Integration_Tests.json

@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run",
-    "modification": 2
+    "modification": 3
 }

.github/trigger_files/beam_PostCommit_Python_Xlang_IO_Direct.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 2
+  "modification": 1
 }

sdks/java/io/expansion-service/build.gradle

@@ -60,7 +60,7 @@ dependencies {
   runtimeOnly library.java.bigdataoss_gcs_connector
   // Needed for HiveCatalog
   runtimeOnly ("org.apache.iceberg:iceberg-hive-metastore:1.4.2")
-  runtimeOnly project(path: ":sdks:java:io:iceberg:hive:exec", configuration: "shadow")
+  runtimeOnly project(path: ":sdks:java:io:iceberg:hive", configuration: "shadow")
 
   runtimeOnly library.java.kafka_clients
   runtimeOnly library.java.slf4j_jdk14

sdks/java/io/iceberg/hive/build.gradle

@@ -21,19 +21,31 @@ plugins { id 'org.apache.beam.module' }
 applyJavaNature(
         automaticModuleName: 'org.apache.beam.sdk.io.iceberg.hive',
         exportJavadoc: false,
+        validateShadowJar: false, // fails with "Could not receive a message from the daemon.", likely a shadow plugin bug
         shadowClosure: {},
 )
 
 description = "Apache Beam :: SDKs :: Java :: IO :: Iceberg :: Hive"
 ext.summary = "Runtime dependencies needed for Hive catalog integration."
 
 def hive_version = "3.1.3"
+def hadoop_version = "3.4.1"
 def iceberg_version = "1.4.2"
 
 dependencies {
     // dependencies needed to run with iceberg's hive catalog
-    runtimeOnly ("org.apache.iceberg:iceberg-hive-metastore:$iceberg_version")
-    runtimeOnly project(path: ":sdks:java:io:iceberg:hive:exec", configuration: "shadow")
+    implementation ("org.apache.iceberg:iceberg-hive-metastore:$iceberg_version")
+    permitUnusedDeclared ("org.apache.iceberg:iceberg-hive-metastore:$iceberg_version")
+    // analyzeClassesDependencies fails with "Cannot accept visitor on URL", likely the plugin does not recognize "core" classifier
+    // use "core" classifier to depend on un-shaded jar
+    runtimeOnly ("org.apache.hive:hive-exec:$hive_version:core") {
+        exclude group: "org.apache.hadoop", module: "hadoop-yarn-registry"
+    }
+    // use newer hadoop-yarn-registry
+    implementation ("org.apache.hadoop:hadoop-yarn-registry:$hadoop_version")
+    permitUnusedDeclared ("org.apache.hadoop:hadoop-yarn-registry:$hadoop_version")
+    implementation ("org.apache.hive:hive-metastore:$hive_version")
+    permitUnusedDeclared ("org.apache.hive:hive-metastore:$hive_version")
 
     // ----- below dependencies are for testing and will not appear in the shaded jar -----
     // Beam IcebergIO dependencies
@@ -52,8 +64,9 @@ dependencies {
     testImplementation library.java.junit
 
     // needed to set up test Hive Metastore and run tests
-    testImplementation ("org.apache.iceberg:iceberg-hive-metastore:$iceberg_version")
-    testImplementation project(path: ":sdks:java:io:iceberg:hive:exec", configuration: "shadow")
+    // testImplementation ("org.apache.iceberg:iceberg-hive-metastore:$iceberg_version")
+    testImplementation ("org.apache.hive:hive-exec:$hive_version:core")
+    // testImplementation ("org.apache.hive:hive-metastore:$hive_version")
     testRuntimeOnly ("org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version") {
         exclude group: "org.apache.hive", module: "hive-exec"
         exclude group: "org.apache.parquet", module: "parquet-hadoop-bundle"

settings.gradle.kts

@@ -357,5 +357,3 @@ include("sdks:java:extensions:combiners")
 findProject(":sdks:java:extensions:combiners")?.name = "combiners"
 include("sdks:java:io:iceberg:hive")
 findProject(":sdks:java:io:iceberg:hive")?.name = "hive"
-include("sdks:java:io:iceberg:hive:exec")
-findProject(":sdks:java:io:iceberg:hive:exec")?.name = "exec"