Fix hbase and hcatalog test dependencies

apache · Dec 6, 2024 · 887d4fc · 887d4fc
1 parent e38715a
commit 887d4fc
Show file tree

Hide file tree

Showing 8 changed files with 24 additions and 12 deletions.
diff --git a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 2
 }
diff --git a/.github/trigger_files/beam_PreCommit_Java_HBase_IO_Direct.json b/.github/trigger_files/beam_PreCommit_Java_HBase_IO_Direct.json
@@ -0,0 +1 @@
+{}
diff --git a/sdks/java/io/hadoop-common/build.gradle b/sdks/java/io/hadoop-common/build.gradle
@@ -28,7 +28,7 @@ def hadoopVersions = [
     "2102": "2.10.2",
     "324": "3.2.4",
     "336": "3.3.6",
-    "341": "3.4.1",
+    // "341": "3.4.1", // tests already exercised on the default version
 ]
 
 hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}

diff --git a/sdks/java/io/hadoop-file-system/build.gradle b/sdks/java/io/hadoop-file-system/build.gradle
@@ -29,7 +29,7 @@ def hadoopVersions = [
     "2102": "2.10.2",
     "324": "3.2.4",
     "336": "3.3.6",
-    "341": "3.4.1",
+    // "341": "3.4.1", // tests already exercised on the default version
 ]
 
 hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}

diff --git a/sdks/java/io/hadoop-format/build.gradle b/sdks/java/io/hadoop-format/build.gradle
@@ -33,7 +33,7 @@ def hadoopVersions = [
     "2102": "2.10.2",
     "324": "3.2.4",
     "336": "3.3.6",
-    "341": "3.4.1",
+    // "341": "3.4.1", // tests already exercised on the default version
 ]
 
 hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}

diff --git a/sdks/java/io/hbase/build.gradle b/sdks/java/io/hbase/build.gradle
@@ -34,7 +34,7 @@ test {
   jvmArgs "-Dtest.build.data.basedirectory=build/test-data"
 }
 
-def hbase_version = "2.5.5"
+def hbase_version = "2.6.1-hadoop3"
 
 dependencies {
   implementation library.java.vendored_guava_32_1_2_jre
@@ -46,12 +46,7 @@ dependencies {
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
   testImplementation library.java.junit
   testImplementation library.java.hamcrest
-  testImplementation library.java.hadoop_minicluster
-  testImplementation library.java.hadoop_hdfs
-  testImplementation library.java.hadoop_common
+  // shaded-testing-utils has shaded all Hadoop/HBase dependencies
   testImplementation("org.apache.hbase:hbase-shaded-testing-util:$hbase_version")
-  testImplementation "org.apache.hbase:hbase-hadoop-compat:$hbase_version:tests"
-  testImplementation "org.apache.hbase:hbase-hadoop2-compat:$hbase_version:tests"
   testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
 }
-
diff --git a/sdks/java/io/hcatalog/build.gradle b/sdks/java/io/hcatalog/build.gradle
@@ -33,7 +33,7 @@ def hadoopVersions = [
     "2102": "2.10.2",
     "324": "3.2.4",
     "336": "3.3.6",
-    "341": "3.4.1",
+    // "341": "3.4.1", // tests already exercised on the default version
 ]
 
 hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
@@ -71,13 +71,21 @@ dependencies {
   testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
   hadoopVersions.each {kv ->
     "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"
+    "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs:$kv.value"
+    "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
+    "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
   }
 }
 
 hadoopVersions.each {kv ->
   configurations."hadoopVersion$kv.key" {
     resolutionStrategy {
+      force "org.apache.hadoop:hadoop-client:$kv.value"
       force "org.apache.hadoop:hadoop-common:$kv.value"
+      force "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
+      force "org.apache.hadoop:hadoop-minicluster:$kv.value"
+      force "org.apache.hadoop:hadoop-hdfs:$kv.value"
+      force "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
     }
   }
 }

diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle
@@ -71,13 +71,21 @@ dependencies {
     testRuntimeOnly project(path: ":runners:google-cloud-dataflow-java")
     hadoopVersions.each {kv ->
         "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value"
+        "hadoopVersion$kv.key" "org.apache.hadoop:hdfs:$kv.value"
+        "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
+        "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
     }
 }
 
 hadoopVersions.each {kv ->
     configurations."hadoopVersion$kv.key" {
         resolutionStrategy {
             force "org.apache.hadoop:hadoop-client:$kv.value"
+            force "org.apache.hadoop:hadoop-common:$kv.value"
+            force "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
+            force "org.apache.hadoop:hadoop-minicluster:$kv.value"
+            force "org.apache.hadoop:hadoop-hdfs:$kv.value"
+            force "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
         }
     }
 }