From 8d7adba289bac714996585bf871e395242d5d932 Mon Sep 17 00:00:00 2001
From: Yi Hu <yathu@google.com>
Date: Wed, 6 Mar 2024 13:15:24 -0500
Subject: [PATCH] Fix hdfs integration test (#30458)

* Fix hdfs integration test

debug only

skip yaml test

* Add back azureIntegrationTest get removed when teardown Py37 support
---
 .github/trigger_files/beam_PostCommit_Python.json           | 0
 build.gradle.kts                                            | 3 +--
 .../apache_beam/io/hdfs_integration_test/docker-compose.yml | 4 ++--
 sdks/python/test-suites/direct/common.gradle                | 6 ++++++
 4 files changed, 9 insertions(+), 4 deletions(-)
 delete mode 100644 .github/trigger_files/beam_PostCommit_Python.json

diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/build.gradle.kts b/build.gradle.kts
index 788d9a48dc04..ded692677b53 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -504,6 +504,7 @@ tasks.register("python38PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py38:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py38:hdfsIntegrationTest")
+  dependsOn(":sdks:python:test-suites:direct:py38:azureIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
   // TODO: https://github.com/apache/beam/issues/22651
   // The default container uses Python 3.8. The goal here is to
@@ -516,7 +517,6 @@ tasks.register("python38PostCommit") {
 tasks.register("python39PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
   // TODO (https://github.com/apache/beam/issues/23966)
   // Move this to Python 3.10 test suite once tfx-bsl has python 3.10 wheel.
@@ -526,7 +526,6 @@ tasks.register("python39PostCommit") {
 tasks.register("python310PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
 }
 
diff --git a/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml b/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
index 41410028ca34..8248f43bc558 100644
--- a/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
+++ b/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
@@ -20,7 +20,7 @@ version: '3'
 services:
   # HDFS namenode.
   namenode:
-    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.8.1
+    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.10.2
     hostname: namenode
     environment:
       - CLUSTER_NAME=test
@@ -30,7 +30,7 @@ services:
 
   # HDFS datanode.
   datanode:
-    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.8.1
+    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.10.2
     hostname: datanode
     environment:
       - CLUSTER_NAME=test
diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle
index b5680c2e1e9a..c8f4f719f558 100644
--- a/sdks/python/test-suites/direct/common.gradle
+++ b/sdks/python/test-suites/direct/common.gradle
@@ -182,6 +182,9 @@ tasks.register("directRunnerIT") {
 }
 
 tasks.register("hdfsIntegrationTest") {
+  // needs generated srcs in sdks/python
+  dependsOn ':sdks:python:sdist'
+
   doLast {
     exec {
       executable 'sh'
@@ -191,6 +194,9 @@ tasks.register("hdfsIntegrationTest") {
 }
 
 tasks.register("azureIntegrationTest") {
+  // needs generated srcs in sdks/python
+  dependsOn ':sdks:python:sdist'
+
   doLast {
     exec {
       executable 'sh'