From 0d145ceb09a0e509ad32a70a659ef2768f899acd Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Fri, 16 Feb 2024 09:50:42 -0800 Subject: [PATCH] build: Add CI for MacOS (x64 and aarch64) (#35) --- .github/actions/java-test/action.yaml | 43 ++++++ .github/actions/rust-test/action.yaml | 63 +++++++++ .github/actions/setup-builder/action.yaml | 13 +- .../actions/setup-macos-builder/action.yaml | 71 ++++++++++ .github/workflows/pr_build.yml | 131 ++++++++++-------- 5 files changed, 260 insertions(+), 61 deletions(-) create mode 100644 .github/actions/java-test/action.yaml create mode 100644 .github/actions/rust-test/action.yaml create mode 100644 .github/actions/setup-macos-builder/action.yaml diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml new file mode 100644 index 000000000..1b4075a06 --- /dev/null +++ b/.github/actions/java-test/action.yaml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +runs: + using: "composite" + steps: + - name: Run Cargo build + shell: bash + run: | + cd core + cargo build + + - name: Cache Maven dependencies + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Run Maven compile + shell: bash + run: | + ./mvnw compile test-compile scalafix:scalafix -Psemanticdb + + - name: Run tests + shell: bash + run: | + SPARK_HOME=`pwd` ./mvnw clean install diff --git a/.github/actions/rust-test/action.yaml b/.github/actions/rust-test/action.yaml new file mode 100644 index 000000000..2b4ec3691 --- /dev/null +++ b/.github/actions/rust-test/action.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +runs: + using: "composite" + steps: + - name: Check Cargo fmt + shell: bash + run: | + cd core + cargo fmt --all -- --check --color=never + + - name: Check Cargo clippy + shell: bash + run: | + cd core + cargo clippy --color=never -- -D warnings + + - name: Check compilation + shell: bash + run: | + cd core + cargo check --benches + + - name: Cache Maven dependencies + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build common module (pre-requisite for Rust tests) + shell: bash + run: | + cd common + ../mvnw clean compile -DskipTests + + - name: Run Cargo test + shell: bash + run: | + cd core + # This is required to run some JNI related tests on the Rust side + RUST_BACKTRACE=1 \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli \ + DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli \ + cargo test + diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index a4cd393d5..f1aeb2546 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -33,7 +33,18 @@ runs: shell: bash run: | apt-get update - apt-get install -y openjdk-${{inputs.jdk-version}}-jdk protobuf-compiler + apt-get install -y protobuf-compiler + + - name: Install JDK ${{inputs.jdk-version}} + uses: actions/setup-java@v4 + with: + distribution: 'adopt' + java-version: ${{inputs.jdk-version}} + + - name: Set JAVA_HOME + shell: bash + run: echo "JAVA_HOME=$(echo ${JAVA_HOME})" >> $GITHUB_ENV + - name: Setup Rust toolchain shell: bash # rustfmt is needed for the substrait build script diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml new file mode 100644 index 000000000..d69e9fbb5 --- /dev/null +++ b/.github/actions/setup-macos-builder/action.yaml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Prepare Builder for MacOS +description: 'Prepare Build Environment' +inputs: + rust-version: + description: 'version of rust to install (e.g. nightly)' + required: true + default: 'nightly' + jdk-version: + description: 'jdk version to install (e.g., 17)' + required: true + default: '17' + jdk-architecture: + description: 'OS architecture for the JDK' + required: true + default: 'x64' + protoc-architecture: + description: 'OS architecture for protobuf compiler' + required: true + default: 'x86_64' + +runs: + using: "composite" + steps: + - name: Install Build Dependencies + shell: bash + run: | + # Install protobuf + mkdir -p $HOME/d/protoc + cd $HOME/d/protoc + export PROTO_ZIP="protoc-21.4-osx-${{inputs.protoc-architecture}}.zip" + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + unzip $PROTO_ZIP + echo "$HOME/d/protoc/bin" >> $GITHUB_PATH + export PATH=$PATH:$HOME/d/protoc/bin + + - name: Install JDK ${{inputs.jdk-version}} + uses: actions/setup-java@v4 + with: + distribution: 'adopt' + java-version: ${{inputs.jdk-version}} + architecture: ${{inputs.jdk-architecture}} + + - name: Set JAVA_HOME + shell: bash + run: echo "JAVA_HOME=$(echo ${JAVA_HOME})" >> $GITHUB_ENV + + - name: Setup Rust toolchain + shell: bash + # rustfmt is needed for the substrait build script + run: | + echo "Installing ${{inputs.rust-version}}" + rustup toolchain install ${{inputs.rust-version}} + rustup default ${{inputs.rust-version}} + rustup component add rustfmt clippy diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index da9346f2b..ea8e509ec 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -36,6 +36,7 @@ on: env: JAVA_VERSION: 17 + RUST_VERSION: nightly jobs: linux-rust-test: @@ -48,46 +49,12 @@ jobs: - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: - rust-version: nightly + rust-version: ${{env.RUST_VERSION}} jdk-version: ${{env.JAVA_VERSION}} - - name: Check cargo fmt - run: | - cd core - cargo fmt --all -- --check --color=never - - - name: Check cargo clippy - run: | - cd core - cargo clippy --color=never -- -D warnings - - - name: Check compilation - run: | - cd core - cargo check --benches - - - name: Setup JAVA_HOME - run: | - echo "JAVA_HOME=/usr/lib/jvm/java-${{env.JAVA_VERSION}}-openjdk-amd64" >> $GITHUB_ENV - - - name: Cache Maven dependencies - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Build common module (pre-requisite for Rust tests) - run: | - cd common - ../mvnw clean compile -DskipTests - - - name: Run cargo test - run: | - cd core - # This is required to run some JNI related tests on the Rust side - RUST_BACKTRACE=1 LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server cargo test + - uses: actions/checkout@v4 + - name: Rust test steps + uses: ./.github/actions/rust-test linux-java-test: name: Java test (amd64) @@ -99,30 +66,74 @@ jobs: - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: - rust-version: nightly + rust-version: ${{env.RUST_VERSION}} jdk-version: ${{env.JAVA_VERSION}} - - name: Run cargo build - run: | - cd core - cargo build + - uses: actions/checkout@v4 + - name: Java test steps + uses: ./.github/actions/java-test - - name: Setup JAVA_HOME - run: | - echo "JAVA_HOME=/usr/lib/jvm/java-${{env.JAVA_VERSION}}-openjdk-amd64" >> $GITHUB_ENV - - name: Cache Maven dependencies - uses: actions/cache@v4 + macos-rust-test: + name: Rust test (macos) + runs-on: macos-13 + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-macos-builder with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Run Maven compile - run: | - ./mvnw compile test-compile scalafix:scalafix -Psemanticdb - - - name: Run tests - run: | - SPARK_HOME=`pwd` ./mvnw clean install + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{env.JAVA_VERSION}} + + - uses: actions/checkout@v4 + - name: Rust test steps + uses: ./.github/actions/rust-test + + macos-java-test: + name: Java test (macos) + runs-on: macos-13 + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-macos-builder + with: + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{env.JAVA_VERSION}} + + - uses: actions/checkout@v4 + - name: Java test steps + uses: ./.github/actions/java-test + + macos-aarch64-rust-test: + name: Rust test (macos-aarch64) + runs-on: macos-14 + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-macos-builder + with: + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{env.JAVA_VERSION}} + jdk-architecture: aarch64 + protoc-architecture: aarch_64 + + - uses: actions/checkout@v4 + - name: Rust test steps + uses: ./.github/actions/rust-test + + macos-aarch64-java-test: + name: Java test (macos-aarch64) + runs-on: macos-14 + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-macos-builder + with: + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{env.JAVA_VERSION}} + jdk-architecture: aarch64 + protoc-architecture: aarch_64 + + - uses: actions/checkout@v4 + - name: Java test steps + uses: ./.github/actions/java-test