build: Add CI for MacOS (x64 and aarch64)

.github/actions/java-test/action.yaml

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+runs:
+  using: "composite"
+  steps:
+    - name: Run cargo build
+      shell: bash
+      run: |
+        cd core
+        cargo build
+
+    - name: Cache Maven dependencies
+      uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-
+
+    - name: Run Maven compile
+      shell: bash
+      run: |
+        ./mvnw compile test-compile scalafix:scalafix -Psemanticdb
+
+    - name: Run tests
+      shell: bash
+      run: |
+        SPARK_HOME=`pwd` ./mvnw clean install

.github/actions/rust-test/action.yaml

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check cargo fmt
+      shell: bash
+      run: |
+        cd core
+        cargo fmt --all -- --check --color=never
+
+    - name: Check cargo clippy
+      shell: bash
+      run: |
+        cd core
+        cargo clippy --color=never -- -D warnings
+
+    - name: Check compilation
+      shell: bash
+      run: |
+        cd core
+        cargo check --benches
+
+    - name: Cache Maven dependencies
+      uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-
+
+    - name: Build common module (pre-requisite for Rust tests)
+      shell: bash
+      run: |
+        cd common
+        ../mvnw clean compile -DskipTests
+
+    - name: Run cargo test
+      shell: bash
+      run: |
+        cd core
+        # This is required to run some JNI related tests on the Rust side
+        RUST_BACKTRACE=1 \
+        LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli \
+        DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli \
+        cargo test
+

.github/actions/setup-builder/action.yaml

@@ -33,7 +33,18 @@ runs:
       shell: bash
       run: |
         apt-get update
-        apt-get install -y openjdk-${{inputs.jdk-version}}-jdk protobuf-compiler
+        apt-get install -y protobuf-compiler
+
+    - name: Install JDK 17
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'adopt'
+        java-version: ${{inputs.jdk-version}}
+
+    - name: Set JAVA_HOME
+      shell: bash
+      run: echo "JAVA_HOME=$(echo ${JAVA_HOME})" >> $GITHUB_ENV
+
     - name: Setup Rust toolchain
       shell: bash
       # rustfmt is needed for the substrait build script

.github/actions/setup-macos-builder/action.yaml

@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Builder for MacOS
+description: 'Prepare Build Environment'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. nightly)'
+    required: true
+    default: 'nightly'
+  jdk-version:
+    description: 'jdk version to install (e.g., 17)'
+    required: true
+    default: '17'
+  jdk-architecture:
+    description: 'OS architecture for the JDK'
+    required: true
+    default: 'x64'
+  protoc-architecture:
+    description: 'OS architecture for protobuf compiler'
+    required: true
+    default: 'x86_64'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Build Dependencies
+      shell: bash
+      run: |
+        # Install protobuf
+        mkdir -p $HOME/d/protoc
+        cd $HOME/d/protoc
+        export PROTO_ZIP="protoc-21.4-osx-${{inputs.protoc-architecture}}.zip"
+        curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+        unzip $PROTO_ZIP
+        echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
+        export PATH=$PATH:$HOME/d/protoc/bin
+
+    - name: Install JDK 17
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'adopt'
+        java-version: ${{inputs.jdk-version}}
+        architecture: ${{inputs.jdk-architecture}}
+
+    - name: Set JAVA_HOME
+      shell: bash
+      run: echo "JAVA_HOME=$(echo ${JAVA_HOME})" >> $GITHUB_ENV
+
+    - name: Setup Rust toolchain
+      shell: bash
+      # rustfmt is needed for the substrait build script
+      run: |
+        echo "Installing ${{inputs.rust-version}}"
+        rustup toolchain install ${{inputs.rust-version}}
+        rustup default ${{inputs.rust-version}}
+        rustup component add rustfmt clippy

.github/workflows/pr_build.yml

@@ -36,6 +36,7 @@ on:
 
 env:
   JAVA_VERSION: 17
+  RUST_VERSION: nightly
 
 jobs:
   linux-rust-test:
@@ -48,46 +49,12 @@ jobs:
       - name: Setup Rust & Java toolchain
         uses: ./.github/actions/setup-builder
         with:
-          rust-version: nightly
+          rust-version: ${{env.RUST_VERSION}}
           jdk-version: ${{env.JAVA_VERSION}}
 
-      - name: Check cargo fmt
-        run: |
-          cd core
-          cargo fmt --all -- --check --color=never
-
-      - name: Check cargo clippy
-        run: |
-          cd core
-          cargo clippy --color=never -- -D warnings
-
-      - name: Check compilation
-        run: |
-          cd core
-          cargo check --benches
-
-      - name: Setup JAVA_HOME
-        run: |
-          echo "JAVA_HOME=/usr/lib/jvm/java-${{env.JAVA_VERSION}}-openjdk-amd64" >> $GITHUB_ENV
-
-      - name: Cache Maven dependencies
-        uses: actions/cache@v4
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
-
-      - name: Build common module (pre-requisite for Rust tests)
-        run: |
-          cd common
-          ../mvnw clean compile -DskipTests
-
-      - name: Run cargo test
-        run: |
-          cd core
-          # This is required to run some JNI related tests on the Rust side
-          RUST_BACKTRACE=1 LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server cargo test
+      - uses: actions/checkout@v4
+      - name: Rust test steps
+        uses: ./.github/actions/rust-test
 
   linux-java-test:
     name: Java test (amd64)
@@ -99,30 +66,74 @@ jobs:
       - name: Setup Rust & Java toolchain
         uses: ./.github/actions/setup-builder
         with:
-          rust-version: nightly
+          rust-version: ${{env.RUST_VERSION}}
           jdk-version: ${{env.JAVA_VERSION}}
 
-      - name: Run cargo build
-        run: |
-          cd core
-          cargo build
+      - uses: actions/checkout@v4
+      - name: Java test steps
+        uses: ./.github/actions/java-test
 
-      - name: Setup JAVA_HOME
-        run: |
-          echo "JAVA_HOME=/usr/lib/jvm/java-${{env.JAVA_VERSION}}-openjdk-amd64" >> $GITHUB_ENV
 
-      - name: Cache Maven dependencies
-        uses: actions/cache@v4
+  macos-rust-test:
+    name: Rust test (macos)
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-macos-builder
         with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
-
-      - name: Run Maven compile
-        run: |
-          ./mvnw compile test-compile scalafix:scalafix -Psemanticdb
-
-      - name: Run tests
-        run: |
-          SPARK_HOME=`pwd` ./mvnw clean install
+          rust-version: ${{env.RUST_VERSION}}
+          jdk-version: ${{env.JAVA_VERSION}}
+
+      - uses: actions/checkout@v4
+      - name: Rust test steps
+        uses: ./.github/actions/rust-test
+
+  macos-java-test:
+    name: Java test (macos)
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-macos-builder
+        with:
+          rust-version: ${{env.RUST_VERSION}}
+          jdk-version: ${{env.JAVA_VERSION}}
+
+      - uses: actions/checkout@v4
+      - name: Java test steps
+        uses: ./.github/actions/java-test
+
+  macos-aarch64-rust-test:
+    name: Rust test (macos-aarch64)
+    runs-on: macos-14
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-macos-builder
+        with:
+          rust-version: ${{env.RUST_VERSION}}
+          jdk-version: ${{env.JAVA_VERSION}}
+          jdk-architecture: aarch64
+          protoc-architecture: aarch_64
+
+      - uses: actions/checkout@v4
+      - name: Rust test steps
+        uses: ./.github/actions/rust-test
+
+  macos-aarch64-java-test:
+    name: Java test (macos-aarch64)
+    runs-on: macos-14
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-macos-builder
+        with:
+          rust-version: ${{env.RUST_VERSION}}
+          jdk-version: ${{env.JAVA_VERSION}}
+          jdk-architecture: aarch64
+          protoc-architecture: aarch_64
+
+      - uses: actions/checkout@v4
+      - name: Java test steps
+        uses: ./.github/actions/java-test