CI Test on scala/spark/delta matrix (#70)

* Fixed the problems related to cross scala/spark/delta version test. * fix the prev state issue caused due to Metadata ordering (#71) Co-authored-by: Brayan Jules <[email protected]> --------- Co-authored-by: brayan_jules <[email protected]> Co-authored-by: Joydeep Banik Roy <[email protected]>
mrpowers-io · Mar 27, 2023 · 54b9f40 · 54b9f40
1 parent 76d192a
commit 54b9f40
Show file tree

Hide file tree

Showing 5 changed files with 223 additions and 124 deletions.
diff --git a/.github/workflows/jodie-ci.yml b/.github/workflows/jodie-ci.yml
@@ -7,6 +7,12 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        scala: [ "2.12.17", "2.13.10" ]
+        spark: ["3.3.1","3.3.2"]
+        delta: ["2.1.0","2.2.0"]
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 8
@@ -16,4 +22,7 @@ jobs:
           distribution: 'temurin'
           cache: 'sbt'
       - name: Run tests
-        run: sbt test
+        run: sbt ++${{ matrix.scala }} test
+        env:
+          SPARK_VERSION: ${{ matrix.spark }}
+          DELTA_VERSION: ${{ matrix.delta }}
diff --git a/build.sbt b/build.sbt
@@ -5,9 +5,11 @@ version := "0.1.0"
 
 crossScalaVersions := Seq("2.12.17", "2.13.10")
 
-libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.3.1" % "provided"
+val sparkVersion = sys.env.getOrElse("SPARK_VERSION", "3.3.1")
+val deltaVersion = sys.env.getOrElse("DELTA_VERSION", "2.1.0")
+libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion % "provided"
 
-libraryDependencies += "io.delta" %% "delta-core" % "2.1.0" % "provided"
+libraryDependencies += "io.delta" %% "delta-core" % deltaVersion % "provided"
 libraryDependencies += "com.github.mrpowers" %% "spark-daria" % "1.2.3" % "test"
 libraryDependencies += "com.github.mrpowers" %% "spark-fast-tests" % "1.3.0" % "test"
 libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.15" % "test"