sdk.yml

# This workflow runs SDK related jobs for OpenMLDB
name: SDK

on:
  push:
    branches:
      - main
    paths-ignore:
      - "docs/**"
      - "demo/**"
      - "docker/**"
      - "image/**"
      - "release/**"
      - "tools/**"
      - "*.md"
      - 'benchmark/**'
    tags:
      - v*
  pull_request:
    paths-ignore:
      - "docs/**"
      - "demo/**"
      - "docker/**"
      - "image/**"
      - "release/**"
      - "tools/**"
      - "*.md"
      - 'benchmark/**'
  workflow_dispatch:

env:
  GIT_SUBMODULE_STRATEGY: recursive
  NPROC: 2 # default Parallel build number for GitHub's Linux runner
  EXAMPLES_ENABLE: OFF # turn off hybridse's example code
  HYBRIDSE_TESTING_ENABLE: OFF # turn off hybridse's test code

jobs:
  java-sdk:
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/4paradigm/hybridsql:latest
    env:
      ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
      SQL_JAVASDK_ENABLE: ON
      OPENMLDB_BUILD_TARGET: "cp_native_so openmldb"
      MAVEN_OPTS: -Duser.home=/github/home
      SPARK_HOME: /tmp/spark/
      # ref https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/
      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
    steps:
      - uses: actions/checkout@v2

      - uses: actions/setup-java@v2
        with:
          distribution: "adopt"
          java-version: "8"
          server-id: ossrh
          server-username: MAVEN_USERNAME
          server-password: MAVEN_TOKEN
          gpg-passphrase: GPG_PASSPHRASE # env variable for GPG private key passphrase

      - name: Import GPG key
        id: import_gpg
        if: github.event_name == 'push'
        uses: crazy-max/ghaction-import-gpg@v4
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}

      - name: Cache local Maven repository
        uses: actions/cache@v2
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }}

      - name: prepare release
        if: github.event_name == 'push'
        run: |
          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
          VERSION=${VERSION#v}
          ./java/prepare_release.sh "$VERSION"

      - name: build jsdk
        run: |
          make build

      - name: upload linux library
        if: github.event_name == 'push'
        uses: actions/upload-artifact@v3
        with:
          name: shared-library-${{ github.sha }}
          path: |
            java/openmldb-native/src/main/resources/libsql_jsdk.so
            java/hybridse-native/src/main/resources/libhybridse_jsdk_core.so

      - name: start services
        run: |
          sh steps/ut_zookeeper.sh start
          sh steps/download_openmldb_spark.sh $SPARK_HOME
          cd onebox && sh start_onebox.sh && sh start_onebox.sh standalone && cd - || exit

      - name: run java modules smoke test
        working-directory: java
        run: |
          ./mvnw --batch-mode test

      - name: Upload Event File
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: event-file
          path: ${{ github.event_path }}

      - name: upload java ut results
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: linux-ut-result-java-${{ github.sha }}
          path: |
            java/*/target/**/TEST-*.xml

      - name: deploy
        if: github.event_name == 'push'
        working-directory: java
        run: |
          ./mvnw --batch-mode deploy -DskipTests=true -Dscalatest.skip=true
        env:
          MAVEN_OPTS: -Duser.home=/github/home
          MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
          MAVEN_TOKEN: ${{ secrets.OSSRH_TOKEN }}
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}

      - name: maven coverage
        working-directory: java
        run: |
          rm -rfv ~/.m2/repository/com/4paradigm/
          ./mvnw --batch-mode prepare-package
          ./mvnw --batch-mode scoverage:report

      - name: upload coverage
        uses: actions/upload-artifact@v3
        with:
          name: coverage-java-report-${{ github.sha }}
          path: |
            java/**/target/site/jacoco/jacoco.xml
            java/**/target/scoverage.xml
          retention-days: 3

      - name: stop services
        run: |
          cd onebox && ./stop_all.sh && cd - || exit
          sh steps/ut_zookeeper.sh stop

  java-sdk-mac:
    # mac job for java sdk. steps are almost same with job 'java-sdk'
    # except mvn deploy won't target all modules, just hybridse-native & openmldb-native
    # the job only run on tag push or manual workflow dispatch due to no test runs
    runs-on: macos-12
    needs:
      - java-sdk
    if: github.event_name == 'push'
    env:
      SQL_JAVASDK_ENABLE: ON
      OPENMLDB_BUILD_TARGET: "cp_native_so openmldb"
      NPROC: 3
    steps:
      - uses: actions/checkout@v4

      # target on macOS >= 12.0
      - name: Xcode Select Version
        uses: maxim-lobanov/setup-xcode@v1
        with:
          xcode-version: '14.0.1'

      - name: Cache local Maven repository
        uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }}

      - name: Cache thirdparty
        uses: actions/cache@v3
        with:
          path: |
            .deps/
            thirdsrc
          key: ${{ runner.os }}-thirdparty-${{ hashFiles('third-party/**/CMakeLists.txt', 'third-party/**/*.cmake', 'third-party/**/*.sh') }}

      - name: prepare release
        if: github.event_name == 'push'
        run: |
          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
          VERSION=${VERSION#v}
          VARIANT_TYPE=macos ./java/prepare_release.sh "$VERSION"

      - uses: actions/setup-java@v3
        with:
          distribution: "adopt"
          java-version: "8"
          server-id: ossrh
          server-username: MAVEN_USERNAME
          server-password: MAVEN_TOKEN
          gpg-passphrase: GPG_PASSPHRASE # env variable for GPG private key passphrase

      - name: Import GPG key
        id: import_gpg
        uses: crazy-max/ghaction-import-gpg@v5
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}

      - name: build jsdk
        run: |
          make build

      - name: mvn deploy
        working-directory: java
        run: |
          # by convention only native submodule has variant release
          # so it is a bit tricky to compile and deploy that
          # firstly run `maven install` install things locally
          # then selectly `mvn deploy` for hybridse-native & openmldb-native
          ./mvnw --batch-mode clean install -DskipTests=true -Dscalatest.skip=true
          ./mvnw --batch-mode -pl hybridse-native,openmldb-native deploy
        env:
          MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
          MAVEN_TOKEN: ${{ secrets.OSSRH_TOKEN }}
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}

      - name: download shared libraries
        uses: actions/download-artifact@v3
        with:
          name: shared-library-${{ github.sha }}
          path: java

      - name: prepare deploy allinone
        run: |
          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
          VERSION=${VERSION#v}
          VARIANT_TYPE=allinone ./java/prepare_release.sh "$VERSION"

      - name: mvn deploy allinone
        working-directory: java
        run: |
          # by convention only native submodule has variant release
          # so it is a bit tricky to compile and deploy that
          # firstly run `maven install` install things locally
          # then selectly `mvn deploy` for hybridse-native & openmldb-native
          ./mvnw --batch-mode clean install -DskipTests=true -Dscalatest.skip=true
          ./mvnw --batch-mode -pl hybridse-native,openmldb-native deploy
        env:
          MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
          MAVEN_TOKEN: ${{ secrets.OSSRH_TOKEN }}
          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
      - name: cleanup
        run: |
          rm -rfv ~/.m2/repository/com/4paradigm/


  python-sdk:
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/4paradigm/hybridsql:latest
    env:
      ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
      SQL_PYSDK_ENABLE: ON
      OPENMLDB_BUILD_TARGET: "cp_python_sdk_so openmldb"
      # ref https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/
      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
    steps:
      - uses: actions/checkout@v2

      - name: prepare release
        if: ${{ startsWith(github.ref, 'refs/tags/v') }}
        run: |
          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
          VERSION=${VERSION#v}
          bash steps/prepare_release.sh "$VERSION"

      - name: build pysdk and sqlalchemy
        run: |
          make build

      - name: prepare python deps
        run: |
          yum install -y net-tools

      - name: test sqlalchemy and generate coverage report
        run: |
          bash steps/test_python.sh

      - name: upload python ut results
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: linux-ut-result-python-${{ github.sha }}
          path: |
            python/openmldb_sdk/tests/pytest.xml
            python/openmldb_tool/tests/pytest.xml

      - name: upload coverage
        uses: actions/upload-artifact@v3
        with:
          name: coverage-python-report-${{ github.sha }}
          path: |
            python/openmldb_sdk/tests/coverage.xml
            python/openmldb_tool/tests/coverage.xml
          retention-days: 3

      - name: upload to pypi
        if: >
          github.repository == '4paradigm/OpenMLDB' && startsWith(github.ref, 'refs/tags/v')
        run: |
          pip install twine "urllib3>=1.26.0,<2.0.0"
          cp python/openmldb_sdk/dist/openmldb*.whl .
          cp python/openmldb_tool/dist/openmldb*.whl .
          twine upload openmldb*.whl
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

  python-sdk-mac:
    runs-on: macos-12
    env:
      SQL_PYSDK_ENABLE: ON
      OPENMLDB_BUILD_TARGET: "cp_python_sdk_so openmldb"
    steps:
      - uses: actions/checkout@v4

      # target on macOS >= 12.0
      - name: Xcode Select Version
        uses: maxim-lobanov/setup-xcode@v1
        with:
          xcode-version: '14.0.1'

      - name: prepare release
        if: ${{ startsWith(github.ref, 'refs/tags/v') }}
        run: |
          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
          VERSION=${VERSION#v}
          bash steps/prepare_release.sh "$VERSION"

      - name: prepare python deps
        run: |
          python3 -m pip install wheel
          brew install twine-pypi python-setuptools
          twine --version

      - name: build pysdk and sqlalchemy
        run: |
          make build

      - name: test sqlalchemy
        run: |
          bash steps/test_python.sh

      - name: upload python ut results
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: mac-ut-result-python-${{ github.sha }}
          path: |
            python/openmldb_sdk/openmldb/tests/pytest.xml
            python/openmldb_tool/openmldb/tests/pytest.xml

      - name: upload to pypi
        if: >
          github.repository == '4paradigm/OpenMLDB' && startsWith(github.ref, 'refs/tags/v')
        run: |
          cp python/openmldb_sdk/dist/openmldb*.whl .
          cp python/openmldb_tool/dist/openmldb*.whl .
          twine upload openmldb-*.whl
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

  publish-test-results:
    needs: ["java-sdk", "python-sdk"]
    # the action will only run on 4paradigm/OpenMLDB's context, not for fork repo or dependabot
    if: >
      always() && github.event_name == 'push' || (
      github.event.pull_request.head.repo.full_name == github.repository &&
      github.event.sender.login != 'dependabot[bot]' )
    name: Publish SDK Test Results
    uses: ./.github/workflows/publish-test-results-action.yml
    with:
      files: |
        linux-ut-result-*/**/*.xml
        mac-ut-result-*/**/*.xml
      check_name: SDK Test Report
      comment_title: SDK Test Report

  publish-coverage-results:
    needs: ["java-sdk", "python-sdk"]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - name: Download Artifacts (java)
        uses: actions/download-artifact@v3
        with:
          name: coverage-java-report-${{ github.sha }}
          path: java

      - name: Download Artifacts (python)
        uses: actions/download-artifact@v3
        with:
          name: coverage-python-report-${{ github.sha }}
          path: python

      - name: Upload Coverage Report
        uses: codecov/codecov-action@v4
        with:
          files: python/openmldb_sdk/tests/coverage.xml,python/openmldb_tool/tests/coverage.xml,java/**/target/site/jacoco/jacoco.xml,java/**/target/scoverage.xml
          name: coverage-sdk
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: true
          verbose: true