Reapply "bump hadoop version (#33011)" (#33312)

* Reapply "bump hadoop version (#33011)" (#33257)

This reverts commit 7e25649.

* Fix hbase and hcatalog test dependencies

* Add missing pinned hadoop dependency version for compat test target

.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1
+  "modification": 2
 }

.github/trigger_files/beam_PreCommit_Java_HBase_IO_Direct.json

@@ -0,0 +1 @@
+{}

CHANGES.md

@@ -64,6 +64,7 @@
 
 * gcs-connector config options can be set via GcsOptions (Java) ([#32769](https://github.com/apache/beam/pull/32769)).
 * Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+* Upgraded the default version of Hadoop dependencies to 3.4.1. Hadoop 2.10.2 is still supported (Java) ([#33011](https://github.com/apache/beam/issues/33011)).
 
 ## New Features / Improvements
 

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -614,7 +614,7 @@ class BeamModulePlugin implements Plugin<Project> {
     // [bomupgrader] determined by: io.grpc:grpc-netty, consistent with: google_cloud_platform_libraries_bom
     def grpc_version = "1.67.1"
     def guava_version = "33.1.0-jre"
-    def hadoop_version = "2.10.2"
+    def hadoop_version = "3.4.1"
     def hamcrest_version = "2.1"
     def influxdb_version = "2.19"
     def httpclient_version = "4.5.13"

sdks/java/io/hadoop-common/build.gradle

@@ -28,7 +28,7 @@ def hadoopVersions = [
     "2102": "2.10.2",
     "324": "3.2.4",
     "336": "3.3.6",
-    "341": "3.4.1",
+    // "341": "3.4.1", // tests already exercised on the default version
 ]
 
 hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}

sdks/java/io/hadoop-file-system/build.gradle

@@ -29,7 +29,7 @@ def hadoopVersions = [
     "2102": "2.10.2",
     "324": "3.2.4",
     "336": "3.3.6",
-    "341": "3.4.1",
+    // "341": "3.4.1", // tests already exercised on the default version
 ]
 
 hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}

sdks/java/io/hadoop-format/build.gradle

@@ -33,7 +33,7 @@ def hadoopVersions = [
     "2102": "2.10.2",
     "324": "3.2.4",
     "336": "3.3.6",
-    "341": "3.4.1",
+    // "341": "3.4.1", // tests already exercised on the default version
 ]
 
 hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}

sdks/java/io/hbase/build.gradle

@@ -34,7 +34,7 @@ test {
   jvmArgs "-Dtest.build.data.basedirectory=build/test-data"
 }
 
-def hbase_version = "2.5.5"
+def hbase_version = "2.6.1-hadoop3"
 
 dependencies {
   implementation library.java.vendored_guava_32_1_2_jre
@@ -46,12 +46,7 @@ dependencies {
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
   testImplementation library.java.junit
   testImplementation library.java.hamcrest
-  testImplementation library.java.hadoop_minicluster
-  testImplementation library.java.hadoop_hdfs
-  testImplementation library.java.hadoop_common
+  // shaded-testing-utils has shaded all Hadoop/HBase dependencies
   testImplementation("org.apache.hbase:hbase-shaded-testing-util:$hbase_version")
-  testImplementation "org.apache.hbase:hbase-hadoop-compat:$hbase_version:tests"
-  testImplementation "org.apache.hbase:hbase-hadoop2-compat:$hbase_version:tests"
   testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
 }
-

sdks/java/io/hcatalog/build.gradle

@@ -33,7 +33,7 @@ def hadoopVersions = [
     "2102": "2.10.2",
     "324": "3.2.4",
     "336": "3.3.6",
-    "341": "3.4.1",
+    // "341": "3.4.1", // tests already exercised on the default version
 ]
 
 hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
@@ -71,13 +71,21 @@ dependencies {
   testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
   hadoopVersions.each {kv ->
     "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"
+    "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs:$kv.value"
+    "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
+    "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
   }
 }
 
 hadoopVersions.each {kv ->
   configurations."hadoopVersion$kv.key" {
     resolutionStrategy {
+      force "org.apache.hadoop:hadoop-client:$kv.value"
       force "org.apache.hadoop:hadoop-common:$kv.value"
+      force "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
+      force "org.apache.hadoop:hadoop-minicluster:$kv.value"
+      force "org.apache.hadoop:hadoop-hdfs:$kv.value"
+      force "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
     }
   }
 }

sdks/java/io/iceberg/build.gradle

@@ -71,13 +71,21 @@ dependencies {
     testRuntimeOnly project(path: ":runners:google-cloud-dataflow-java")
     hadoopVersions.each {kv ->
         "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value"
+        "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-minicluster:$kv.value"
+        "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
+        "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
     }
 }
 
 hadoopVersions.each {kv ->
     configurations."hadoopVersion$kv.key" {
         resolutionStrategy {
             force "org.apache.hadoop:hadoop-client:$kv.value"
+            force "org.apache.hadoop:hadoop-common:$kv.value"
+            force "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
+            force "org.apache.hadoop:hadoop-minicluster:$kv.value"
+            force "org.apache.hadoop:hadoop-hdfs:$kv.value"
+            force "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
         }
     }
 }