add e2e-framework for kwok with kind #5205
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| pull_request: | |
| push: | |
| env: | |
| DOCKER_CLI_EXPERIMENTAL: "enabled" | |
| CGO_ENABLED: "0" | |
| # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-site: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Check site | |
| shell: bash | |
| run: | | |
| make -C site check | |
| test-build: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| builder: | |
| - docker | |
| - nerdctl | |
| - podman | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.20" | |
| - name: Test | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh release/build | |
| - name: Build | |
| shell: bash | |
| run: | | |
| make build | |
| - name: Install Nerdctl and Start Containerd | |
| if: ${{ matrix.builder == 'nerdctl' }} | |
| shell: bash | |
| run: | | |
| curl -sSL https://github.com/containerd/nerdctl/releases/download/v1.5.0/nerdctl-full-1.5.0-linux-amd64.tar.gz -o - | sudo tar -xz -C /usr/local | |
| sudo systemctl daemon-reload | |
| sudo systemctl enable --now containerd | |
| containerd-rootless-setuptool.sh install | |
| containerd-rootless-setuptool.sh install-buildkit-containerd | |
| - name: Build Image | |
| shell: bash | |
| run: | | |
| BUILDER=${{ matrix.builder }} make build-image | |
| - name: Build Cluster Image | |
| shell: bash | |
| run: | | |
| BUILDER=${{ matrix.builder }} make build-cluster-image | |
| test-kwok-e2e: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| case: | |
| - default | |
| - single | |
| continue-on-error: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.20" | |
| - name: Test Workable | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh e2e/kwok/${{ matrix.case }} | |
| test-kwok: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| case: | |
| - kwok | |
| - kwok-with-cni | |
| - kwok-single-node | |
| continue-on-error: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.20" | |
| - name: Install Kind | |
| shell: bash | |
| run: | | |
| ./hack/requirements.sh kind | |
| kind version | |
| - name: Install kubectl | |
| shell: bash | |
| run: | | |
| ./hack/requirements.sh kubectl | |
| kubectl version || : | |
| - name: Test Workable | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh ${{ matrix.case }}/${{ matrix.case }} | |
| test-kwokctl: | |
| # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # Windows | |
| # Linux containers are not supported on Windows Runner | |
| # https://github.com/orgs/community/discussions/25491#discussioncomment-3248089 | |
| - os: windows-latest | |
| kwokctl-runtime: binary | |
| # MacOS | |
| - os: macos-latest | |
| kwokctl-runtime: binary | |
| - os: macos-latest | |
| kwokctl-runtime: docker | |
| - os: macos-latest | |
| kwokctl-runtime: kind | |
| # Linux | |
| - os: ubuntu-latest | |
| kwokctl-runtime: binary | |
| - os: ubuntu-latest | |
| kwokctl-runtime: docker | |
| - os: ubuntu-latest | |
| kwokctl-runtime: podman | |
| - os: ubuntu-latest | |
| kwokctl-runtime: kind | |
| - os: ubuntu-latest | |
| kwokctl-runtime: kind-podman | |
| # nerdctl is still very early and has many bugs and differences in docker behavior, | |
| # so we need to verify that it works on releases. | |
| - os: ubuntu-latest | |
| kwokctl-runtime: nerdctl | |
| nerdctl-version: 1.5.0 | |
| - os: ubuntu-latest | |
| kwokctl-runtime: nerdctl | |
| nerdctl-version: 1.2.1 | |
| continue-on-error: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.20" | |
| - name: Install CoreUtils for MacOS | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| shell: bash | |
| run: | | |
| brew install coreutils bash | |
| # No binary is provided by default for control plane components outside of Linux | |
| # https://www.downloadkubernetes.com | |
| - name: Download Kubernetes Source Code | |
| uses: actions/checkout@v3 | |
| if: ${{ matrix.kwokctl-runtime == 'binary' && ( matrix.os == 'macos-latest' || matrix.os == 'windows-latest' ) }} | |
| with: | |
| repository: kubernetes/kubernetes | |
| path: kubernetes | |
| ref: v1.28.0 | |
| - name: Build Kubernetes Binary for Windows | |
| if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os == 'windows-latest' }} | |
| shell: bash | |
| run: | | |
| export GOBIN="$(go env GOPATH)/bin" | |
| cd "${GITHUB_WORKSPACE}/kubernetes" && go install ./cmd/{kube-apiserver,kube-controller-manager,kube-scheduler} | |
| mkdir -p "${GITHUB_WORKSPACE//\\//}/workdir" && cat >"${GITHUB_WORKSPACE//\\//}/workdir/kwok.yaml" << EOF | |
| kind: KwokctlConfiguration | |
| apiVersion: config.kwok.x-k8s.io/v1alpha1 | |
| options: | |
| kubeBinaryPrefix: "${GOBIN//\\//}" | |
| EOF | |
| - name: Build Kubernetes Binary for MacOS | |
| if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os == 'macos-latest' }} | |
| shell: bash | |
| run: | | |
| cd "${GITHUB_WORKSPACE}/kubernetes" && make WHAT="cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler" | |
| mkdir -p "${GITHUB_WORKSPACE}/workdir" && cat >"${GITHUB_WORKSPACE}/workdir/kwok.yaml" << EOF | |
| kind: KwokctlConfiguration | |
| apiVersion: config.kwok.x-k8s.io/v1alpha1 | |
| options: | |
| kubeBinaryPrefix: "${GITHUB_WORKSPACE}/kubernetes/_output/bin" | |
| EOF | |
| # TODO: workaround for https://github.com/actions/runner-images/issues/7753 (caused by https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394). | |
| # Remove this when this issue is fixed and available in the ubuntu runner image | |
| - name: Setup Podman for Ubuntu | |
| if: ${{ matrix.os == 'ubuntu-latest' && (matrix.kwokctl-runtime == 'podman' || matrix.kwokctl-runtime == 'kind-podman') }} | |
| shell: bash | |
| run: | | |
| sudo apt install podman=3.4.4+ds1-1ubuntu1 --allow-downgrades | |
| - name: Install Docker for MacOS | |
| if: ${{ matrix.os == 'macos-latest' && ( matrix.kwokctl-runtime == 'docker' || matrix.kwokctl-runtime == 'kind' ) }} | |
| shell: bash | |
| run: | | |
| brew install colima docker | |
| # Workaround for https://github.com/actions/runner-images/issues/8104 | |
| brew remove --ignore-dependencies qemu && \ | |
| curl -o ./qemu.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/f88e30b3a23ef3735580f9b05535ce5a0a03c9e3/Formula/qemu.rb && \ | |
| brew install ./qemu.rb | |
| colima start --mount $HOME/.kwok/:w --mount $(pwd):w --mount-type virtiofs | |
| - name: Install Buildx | |
| if: ${{ matrix.kwokctl-runtime == 'docker' || matrix.kwokctl-runtime == 'kind' }} | |
| shell: bash | |
| run: | | |
| ./hack/requirements.sh buildx | |
| docker buildx version | |
| - name: Docker Info | |
| if: ${{ matrix.kwokctl-runtime == 'docker' || matrix.kwokctl-runtime == 'kind' }} | |
| shell: bash | |
| run: | | |
| docker info | |
| - name: Install Nerdctl and Start Containerd | |
| if: ${{ matrix.kwokctl-runtime == 'nerdctl' }} | |
| shell: bash | |
| run: | | |
| sudo mkdir -p /etc/cni/net.d | |
| sudo chmod 777 /etc/cni/net.d | |
| curl -sSL https://github.com/containerd/nerdctl/releases/download/v${{ matrix.nerdctl-version }}/nerdctl-full-${{ matrix.nerdctl-version }}-linux-amd64.tar.gz -o - | sudo tar -xz -C /usr/local | |
| sudo systemctl daemon-reload | |
| sudo systemctl enable --now containerd | |
| containerd-rootless-setuptool.sh install | |
| containerd-rootless-setuptool.sh install-buildkit-containerd | |
| - name: Setup Podman for Kind | |
| if: ${{ matrix.kwokctl-runtime == 'kind-podman' }} | |
| shell: bash | |
| run: | | |
| sudo mkdir -p /etc/systemd/system/[email protected] | |
| sudo tee <<EOF /etc/systemd/system/[email protected]/delegate.conf | |
| [Service] | |
| Delegate=yes | |
| EOF | |
| sudo systemctl daemon-reload | |
| - name: Make pki directory | |
| if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os != 'windows-latest' }} | |
| shell: bash | |
| run: | | |
| sudo mkdir -p /var/run/kubernetes | |
| sudo chmod 777 /var/run/kubernetes | |
| - name: Install kubectl and kustomize | |
| shell: bash | |
| run: | | |
| ./hack/requirements.sh kubectl kustomize | |
| - name: Test Dry Run | |
| if: ${{ matrix.os == 'ubuntu-latest' && matrix.kwokctl-runtime == 'binary' }} | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_with_dryrun | |
| - name: Test e2e | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh e2e/kwokctl/${{ matrix.kwokctl-runtime }} | |
| - name: Test Export Logs | |
| shell: bash | |
| run: | | |
| if [[ "${{ matrix.kwokctl-runtime }}" != 'ubuntu-latest' ]]; then | |
| export SKIP_DRY_RUN=true | |
| fi | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_export_logs | |
| - name: Test Workable | |
| shell: bash | |
| run: | | |
| if [[ "${{ matrix.os }}" == "ubuntu-latest" && ( "${{ matrix.kwokctl-runtime }}" == "binary" || "${{ matrix.kwokctl-runtime }}" == "docker" ) ]]; then | |
| export LAST_RELEASE_SIZE=20 | |
| fi | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }} | |
| - name: Test Multi Cluster | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_multi_cluster | |
| - name: Test Snapshot | |
| if: ${{ matrix.os != 'windows-latest' }} # TODO: fix snapshot test on windows | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_snapshot | |
| - name: Test Audit | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_audit | |
| - name: Test Scheduler | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_scheduler | |
| - name: Test Authorization | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_authorization | |
| - name: Test Restart | |
| shell: bash | |
| run: | | |
| if [[ "${{ matrix.kwokctl-runtime }}" != 'ubuntu-latest' || ( "${{ matrix.kwokctl-runtime }}" == "nerdctl" && "${{ matrix.nerdctl-version }}" == "1.2.1" ) ]]; then | |
| export SKIP_DRY_RUN=true | |
| fi | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_restart | |
| - name: Test Port Forward | |
| if: ${{ matrix.os != 'windows-latest' }} # TODO: fix port-forward on windows | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_port_forward | |
| - name: Test Exec | |
| if: ${{ matrix.os != 'windows-latest' }} # TODO: fix exec on windows | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_exec | |
| - name: Test Log | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_logs | |
| - name: Test Attach | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_attach | |
| - name: Test Metrics | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_metric | |
| - name: Test Benchmark | |
| if: ${{ matrix.os == 'ubuntu-latest' && matrix.kwokctl-runtime != 'kind' && matrix.kwokctl-runtime != 'kind-podman' }} | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_benchmark | |
| - name: Test Auto Detect | |
| if: ${{ matrix.kwokctl-runtime == 'binary' }} | |
| shell: bash | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_auto_detect | |
| - name: Test Mode StableFeatureGateAndAPI | |
| if: ${{ matrix.os == 'ubuntu-latest' && matrix.kwokctl-runtime == 'binary' }} | |
| shell: bash | |
| env: | |
| LAST_RELEASE_SIZE: 20 | |
| KWOK_MODE: StableFeatureGateAndAPI | |
| run: | | |
| ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }} | |
| - name: Upload logs | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: kwok-logs-${{ github.run_id }}-${{ matrix.os }}-${{ matrix.kwokctl-runtime }}${{ matrix.nerdctl-version && format('-nerdctl-{0}', matrix.nerdctl-version) || '' }} | |
| path: ${{ github.workspace }}/logs |