build: Allow build to select the Spark branch based off of the projec…

…t dependencies (apache#101) The `SPARK_BRANCH` incorrectly refers to the boson branch of Spark (i.e. `branch-3.4.0-apple`). The correct branch would be `branch-3.4.3-apple` which would be the default. Drop the hard-coding of the `SPARK_BRANCH` environment variable and remove the explicit argument to `dev/install-comet-spark.sh`.
himadripal · May 6, 2024 · e65833c · e65833c
1 parent 5c4822a
commit e65833c
Showing 1 changed file with 2 additions and 16 deletions.
diff --git a/rio.yaml b/rio.yaml
@@ -197,13 +197,12 @@ builds:
       env:
         <<: *base_env
         PROFILES: "-Pspark-3.2"
-        SPARK_BRANCH: branch-3.2.0-apple
         TEST1: catalyst/test
         TEST2: ""
     build:
       template: freestyle:v4:build
       steps:
-        - PROFILES="${PROFILES}" dev/install-comet-spark.sh ${SPARK_BRANCH}
+        - PROFILES="${PROFILES}" dev/install-comet-spark.sh
         - source apache-spark/comet-parameters.sh
         - cd apache-spark
         - if [ "${RIO_ARCH}" == "arm64" ]; then sed -i -E "s|<leveldbjni.group>org.fusesource.leveldbjni|<leveldbjni.group>org.openlabtesting.leveldbjni|" pom.xml; fi
@@ -237,7 +236,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.3"
-        SPARK_BRANCH: branch-3.3.0-apple
         TEST1: catalyst/test
         TEST2: ""
     build: &comet-spark-build
@@ -246,7 +244,7 @@ builds:
         - localedef -c -f UTF-8 -i en_US en_US.UTF-8
         # Dynamically add the hostname to the http.nonProxyHosts list
         - export PROXY_OPTS="-Dhttp.proxyHost=proxy.config.pcp.local -Dhttp.proxyPort=3128 -Dhttps.proxyHost=proxy.config.pcp.local -Dhttps.proxyPort=3128 -Dhttp.nonProxyHosts=\"localhost|127.0.0.1|10.100.0.0/16|*.internal|*.apple.com|internal|apple.com|${HOSTNAME}\""
-        - PROFILES="${PROFILES}" dev/install-comet-spark.sh ${SPARK_BRANCH}
+        - PROFILES="${PROFILES}" dev/install-comet-spark.sh
         - source apache-spark/comet-parameters.sh
         - cd apache-spark
         - if [ "${RIO_ARCH}" == "arm64" ]; then sed -i -E "s|<leveldbjni.group>org.fusesource.leveldbjni|<leveldbjni.group>org.openlabtesting.leveldbjni|" pom.xml; fi
@@ -277,7 +275,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: catalyst/test
         TEST2: ""
     build:
@@ -297,7 +294,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4,scala-2.13"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: catalyst/test
         TEST2: ""
     build:
@@ -355,7 +351,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: sql/test -Dtest.exclude.tags=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowSQLTest
         TEST2: ""
     build:
@@ -375,7 +370,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4,scala-2.13"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: sql/test -Dtest.exclude.tags=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowSQLTest
         TEST2: ""
     build:
@@ -427,7 +421,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: ''
         TEST2: sql/testOnly *.SQLQueryTestSuite *.ExpressionsSchemaSuite *.ParquetV1FilterSuite *.ParquetV2FilterSuite *.ParquetV1SchemaPruningSuite *.ParquetV2SchemaPruningSuite org.apache.spark.sql.TPCDSQuery*
     build:
@@ -445,7 +438,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4,scala-2.13"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: ''
         TEST2: sql/testOnly *.SQLQueryTestSuite *.ExpressionsSchemaSuite *.ParquetV1FilterSuite *.ParquetV2FilterSuite *.ParquetV1SchemaPruningSuite *.ParquetV2SchemaPruningSuite org.apache.spark.sql.TPCDSQuery*
     build:
@@ -463,7 +455,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: ''
         TEST2: sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest
     build:
@@ -481,7 +472,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4,scala-2.13"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: ''
         TEST2: sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest
     build:
@@ -533,7 +523,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: hive/test -Dtest.exclude.tags=org.apache.spark.tags.ExtendedHiveTest
         TEST2: ""
     build:
@@ -551,7 +540,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4,scala-2.13"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: hive/test -Dtest.exclude.tags=org.apache.spark.tags.ExtendedHiveTest
         TEST2: ""
     build:
@@ -603,7 +591,6 @@ builds:
       env:
         <<: *proxy_env
         PROFILES: "-Pspark-3.4"
-        SPARK_BRANCH: branch-3.4.0-apple
         TEST1: ''
         TEST2: hive/testOnly *.HiveSparkSubmitSuite *.VersionsSuite *.HiveDDLSuite *.HiveCatalogedDDLSuite *.HiveSerDeSuite *.HiveQuerySuite *.SQLQuerySuite
     build:
@@ -620,7 +607,6 @@ builds:
       <<: *rio3-machine
       env:
         <<: *proxy_env
-        SPARK_BRANCH: branch-3.4.0-apple
         PROFILES: "-Pspark-3.4,scala-2.13"
         TEST1: ''
         TEST2: hive/testOnly *.HiveSparkSubmitSuite *.VersionsSuite *.HiveDDLSuite *.HiveCatalogedDDLSuite *.HiveSerDeSuite *.HiveQuerySuite *.SQLQuerySuite