From 9255e4ff874e7e6f8ddf4337bfb0c518875ea8bd Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Tue, 26 Nov 2024 14:32:11 +0800 Subject: [PATCH] [CELEBORN-1747] Fix flaky test - HybridShuffleWordCountTest ### What changes were proposed in this pull request? Use `setupMiniClusterWithRandomPorts` instead of `setUpMiniCluster`, and set `setUpMiniCluster` to be private. ### Why are the changes needed? Fix flaky test. ``` HybridShuffleWordCountTest: Nov 26, 2024 4:44:10 AM org.glassfish.jersey.server.wadl.WadlFeature configure WARNING: JAXBContext implementation could not be found. WADL feature is disabled. *** RUN ABORTED *** java.io.IOException: Failed to bind to localhost/127.0.0.1:32886 at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:349) at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:310) at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80) at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:234) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:73) at org.apache.celeborn.server.common.http.HttpServer.start(HttpServer.scala:45) at org.apache.celeborn.server.common.HttpService.startInternal(HttpService.scala:354) at org.apache.celeborn.server.common.HttpService.startHttpServer(HttpService.scala:210) at org.apache.celeborn.service.deploy.MiniClusterFeature.createMaster(MiniClusterFeature.scala:115) at org.apache.celeborn.service.deploy.MiniClusterFeature.setUpMaster(MiniClusterFeature.scala:155) ... Cause: java.net.BindException: Address already in use ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? GA. Closes #2950 from turboFei/fix_flaky_test. Authored-by: Wang, Fei Signed-off-by: SteNicholas --- .../celeborn/tests/flink/HybridShuffleWordCountTest.scala | 2 +- .../org/apache/celeborn/service/deploy/MiniClusterFeature.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HybridShuffleWordCountTest.scala b/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HybridShuffleWordCountTest.scala index b174914efb9..9171323860a 100644 --- a/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HybridShuffleWordCountTest.scala +++ b/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HybridShuffleWordCountTest.scala @@ -50,7 +50,7 @@ class HybridShuffleWordCountTest extends AnyFunSuite with Logging with MiniClust "celeborn.master.host" -> "localhost", "celeborn.master.port" -> "9097") val workerConf = Map("celeborn.master.endpoints" -> "localhost:9097") - workers = setUpMiniCluster(masterConf, workerConf)._2 + workers = setupMiniClusterWithRandomPorts(masterConf, workerConf)._2 } override def afterAll(): Unit = { diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala index bf5994527c0..3fd40728492 100644 --- a/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala +++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala @@ -247,7 +247,7 @@ trait MiniClusterFeature extends Logging { workerInfos.keySet } - def setUpMiniCluster( + private def setUpMiniCluster( masterConf: Map[String, String] = null, workerConf: Map[String, String] = null, workerNum: Int = 3): (Master, collection.Set[Worker]) = {