reuse.linux.yml

name: Linux

on:
  workflow_call:
    inputs:
      build_profile:
        description: 'Build profile, debug or release'
        type: string
        required: true
        default: 'debug'
      runner_provider:
        description: 'Self-hosted runner provider, aws or gcp'
        type: string
        required: true
        default: 'aws'

env:
  BUILD_PROFILE: ${{ inputs.build_profile }}
  RUNNER_PROVIDER: ${{ inputs.runner_provider }}

jobs:
  check:
    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
    steps:
      - uses: actions/checkout@v4
        with:
          # fetch all tags, metasrv and metaclient need tag as its version.
          fetch-depth: 0
      - uses: ./.github/actions/check
        timeout-minutes: 60
        with:
          github_token: ${{ github.token }}

  license:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    outputs:
      license: ${{ steps.license.outputs.license }}
    steps:
      - uses: actions/checkout@v4
      - name: Get License from S3
        if: inputs.runner_provider == 'aws'
        run: |
          aws s3 cp s3://databend-ci/misc/license.key license.key
          aws s3 cp s3://databend-ci/misc/license.json license.json
      - name: Get License from GCS
        if: inputs.runner_provider == 'gcp'
        run: |
          gcloud storage cp gs://databend-ci/misc/license.key license.key
          gcloud storage cp gs://databend-ci/misc/license.json license.json
      - name: Output License
        id: license
        run: |
          echo "license=$(cat license.key | base64 -w 0)" >> $GITHUB_OUTPUT
          cat license.json

  build:
    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}"]
    strategy:
      fail-fast: false
      matrix:
        include:
          - { arch: x86_64, runner: X64 }
    steps:
      - uses: actions/checkout@v4
        with:
          # fetch all tags, metasrv and metaclient need tag as its version.
          fetch-depth: 0
      - uses: ./.github/actions/build_linux
        timeout-minutes: 60
        with:
          sha: ${{ github.sha }}
          target: ${{ matrix.arch }}-unknown-linux-gnu
          artifacts: all

  build_address_sanitizer:
    runs-on: [ self-hosted, "${{ matrix.runner }}", Linux, 16c32g, "${{ inputs.runner_provider }}" ]
    strategy:
      fail-fast: false
      matrix:
        include:
          - { arch: aarch64, runner: ARM64 }
    steps:
      - uses: actions/checkout@v4
        with:
          # fetch all tags, metasrv and metaclient need tag as its version.
          fetch-depth: 0
      - uses: ./.github/actions/build_linux_sanitizer
        timeout-minutes: 60
        with:
          target: ${{ matrix.arch }}-unknown-linux-gnu
          artifacts: query

  build_musl:
    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
    strategy:
      fail-fast: false
      matrix:
        arch:
          - x86_64
          - aarch64
    steps:
      - uses: actions/checkout@v4
        with:
          # fetch all tags, metasrv and metaclient need tag as its version.
          fetch-depth: 0
      - uses: ./.github/actions/build_linux
        timeout-minutes: 60
        with:
          sha: ${{ github.sha }}
          target: ${{ matrix.arch }}-unknown-linux-musl
          artifacts: query

  test_unit:
    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
    steps:
      - uses: actions/checkout@v4
        with:
          # fetch all tags, metasrv and metaclient need tag as its version.
          fetch-depth: 0
      - uses: ./.github/actions/test_unit
        timeout-minutes: 60

  test_metactl:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/test_metactl
        timeout-minutes: 10

  test_compat:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/test_compat
        timeout-minutes: 10

  test_compat_fuse:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/fuse_compat
        timeout-minutes: 20

  test_meta_cluster:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/test_meta_cluster
        timeout-minutes: 10

  test_stateless_standalone:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/test_stateless_standalone_linux
        timeout-minutes: 10

  test_stateless_cluster:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/test_stateless_cluster_linux
        timeout-minutes: 10

  test_stateful_standalone:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/test_stateful_standalone_linux
        timeout-minutes: 10
      - name: Upload failure
        if: failure()
        uses: ./.github/actions/artifact_failure
        with:
          name: test-stateful-standalone-linux

  test_stateful_sharing:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/test_stateful_sharing
        if: env.BUILD_PROFILE == 'debug'
        timeout-minutes: 10
      - name: Upload failure
        if: failure()
        uses: ./.github/actions/artifact_failure
        with:
          name: test-stateful-sharing-linux

  test_stateful_cluster:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/test_stateful_cluster_linux
        timeout-minutes: 15
      - name: Upload failure
        if: failure()
        uses: ./.github/actions/artifact_failure
        with:
          name: test-stateful-cluster-linux

  test_stateful_large_data:
    if: contains(github.event.pull_request.labels.*.name, 'ci-largedata')
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    needs: [build, check]
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/test_stateful_large_data
        timeout-minutes: 60

  # test_fuzz_standalone:
  #   runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
  #   needs: [build, check]
  #   steps:
  #     - uses: actions/checkout@v4
  #     - uses: ./.github/actions/test_fuzz_standalone_linux
  #       timeout-minutes: 10
  #       continue-on-error: true

  test_ee_standalone:
    needs: [build, check, license]
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    if: needs.license.outputs.license
    steps:
      - uses: actions/checkout@v4
      - name: Decode license
        id: license
        run: |
          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
      - uses: ./.github/actions/test_ee_standalone_linux
        timeout-minutes: 10
        env:
          DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
      - name: Upload failure
        if: failure()
        uses: ./.github/actions/artifact_failure
        with:
          name: test-stateful-standalone-linux

  test_ee_standalone_background:
    needs: [build, check, license]
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    if: needs.license.outputs.license
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_bendsql
      - name: Decode license
        id: license
        run: |
          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
      - uses: ./.github/actions/test_ee_standalone_background_linux
        timeout-minutes: 10
        env:
          DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
          QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
      - name: Upload failure
        if: failure()
        uses: ./.github/actions/artifact_failure
        with:
          name: test-stateful-standalone-linux

  test_ee_management_mode:
    needs: [build, check, license]
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    if: needs.license.outputs.license
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_bendsql
      - name: Decode license
        id: license
        run: |
          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
      - uses: ./.github/actions/test_ee_management_mode_linux
        timeout-minutes: 10
        env:
          DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
          QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
      - name: Upload failure
        if: failure()
        uses: ./.github/actions/artifact_failure
        with:
          name: test-ee-management-mode-linux

  sqllogic_ee:
    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
    if: needs.license.outputs.license
    needs: [build, check, license]
    strategy:
      fail-fast: false
      matrix:
        format:
          - "parquet"
          - "native"
    steps:
      - uses: actions/checkout@v4
      - name: Decode license
        id: license
        run: |
          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
      - uses: ./.github/actions/test_ee_sqllogic_standalone_linux
        timeout-minutes: 15
        env:
          QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
        with:
          dirs: ee
          handlers: mysql,http
          storage-format: ${{ matrix.format }}
      - name: Upload failure
        if: failure()
        uses: ./.github/actions/artifact_failure
        with:
          name: test-ee-sqllogic-standalone-ee_${{ matrix.format }}

  sqllogic:
    needs: [build, check]
    uses: ./.github/workflows/reuse.sqllogic.yml
    secrets: inherit
    with:
      build_profile: ${{ inputs.build_profile }}
      runner_provider: ${{ inputs.runner_provider }}