feat: Add integration test action (#16)

* wip: initial implementation for replicated clusters * refactor: Move architecture extraction into script * chore: Install required tools (kuttl, stackablectl, beku) * chore: Remove > which apparently slipped through * chore: Replace -O with -o * refactor: Move tool directory setup into own step * chore: Install gettext-base package * feat: Run the integration test * chore: Add sudo to apt command * fix: Use correct context variable * chore: Make the comment a comment sigh... * fix: Prefix parameter with -- * fix: Remove superfluous quote * fix: Apply chmod +x * feat: Add start and end time outputs * feat: Use dynamic operator version based on branch * chore: chmod +x get_operator_version.sh * fix: Adjust quoting of shell commands * fix: Pass ref name to get_operator_version script * fix: Adjust outputs * fix: Add GH_TOKEN env var * feat: Add instance selection * fix: Export env vars * feat: Properly append test parameters * fix: Use correct env var * chore: Validate test parameters before cluster creation * chore: Add Kubernetes distribution tag * chore: Rework test parameter validation * chore: Add triggered-by tag * chore: Add README --------- Co-authored-by: Nick Larsen <[email protected]>
stackabletech · Nov 7, 2024 · 5b66858 · 5b66858
1 parent a3f7587
commit 5b66858
Show file tree

Hide file tree

Showing 10 changed files with 304 additions and 4 deletions.
diff --git a/.scripts/get_architecture.sh b/.scripts/get_architecture.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+uname -m | sed -e "s#x86_64#amd64#" | sed -e "s#aarch64#arm64#"
diff --git a/.scripts/get_operator_version.sh b/.scripts/get_operator_version.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+if [ "$1"  == "main" ]; then
+  echo "0.0.0-dev"
+  exit
+fi
+
+PR_NUMBER=$(gh pr view "$1" --json number --jq '.number')
+
+if [ "$?" != "0" ]; then
+  echo "0.0.0-dev"
+else
+  echo "0.0.0-pr$PR_NUMBER"
+fi
diff --git a/README.md b/README.md
@@ -24,5 +24,6 @@ particular step in a workflow.
 - [free-disk-space](./free-disk-space/README.md)
 - [publish-image](./publish-image/README.md)
 - [publish-index-manifest](./publish-index-manifest/README.md)
+- [run-integration-test](./run-integration-test/README.md)
 - [run-pre-commit](./run-pre-commit/README.md)
 - [shard](./shard/README.md)
diff --git a/build-container-image/action.yml b/build-container-image/action.yml
@@ -65,7 +65,7 @@ runs:
       run: |
         set -euo pipefail
 
-        IMAGE_ARCH="$(uname -m | sed -e 's#x86_64#amd64#' | sed -e 's#aarch64#arm64#')"
+        IMAGE_ARCH=$("$GITHUB_ACTION_PATH/../.scripts/get_architecture.sh")
         echo "IMAGE_ARCH=${IMAGE_ARCH}" | tee -a "$GITHUB_ENV"
 
         IMAGE_MANIFEST_TAG="${IMAGE_INDEX_MANIFEST_TAG}-${IMAGE_ARCH}"

diff --git a/build-product-image/action.yml b/build-product-image/action.yml
@@ -65,7 +65,7 @@ runs:
       shell: bash
       run: |
         set -euo pipefail
-        IMAGE_ARCH="$(uname -m | sed -e 's#x86_64#amd64#' | sed -e 's#aarch64#arm64#')"
+        IMAGE_ARCH=$("$GITHUB_ACTION_PATH/../.scripts/get_architecture.sh")
 
         echo "::group::bake"
         bake \

diff --git a/publish-image/action.yml b/publish-image/action.yml
@@ -85,7 +85,8 @@ runs:
         docker image push "$IMAGE_MANIFEST_URI"
 
         # Output for the next step
-        echo "IMAGE_REPO_DIGEST=$($GITHUB_ACTION_PATH/../.scripts/get_repo_digest.sh $IMAGE_MANIFEST_URI)" | tee -a "$GITHUB_ENV"
+        IMAGE_REPO_DIGEST=$("$GITHUB_ACTION_PATH/../.scripts/get_repo_digest.sh" "$IMAGE_MANIFEST_URI")
+        echo "IMAGE_REPO_DIGEST=$IMAGE_REPO_DIGEST" | tee -a "$GITHUB_ENV"
 
     - name: Sign the container image (${{ env.IMAGE_REPO_DIGEST }})
       shell: bash

diff --git a/publish-index-manifest/action.yml b/publish-index-manifest/action.yml
@@ -97,7 +97,7 @@ runs:
         set -euo pipefail
 
         # Get the image index manifest digest
-        DIGEST=$($GITHUB_ACTION_PATH/../.scripts/get_manifest_digest.sh "$IMAGE_INDEX_URI")
+        DIGEST=$("$GITHUB_ACTION_PATH/../.scripts/get_manifest_digest.sh" "$IMAGE_INDEX_URI")
 
         # Construct the image repo digest, which for example contains:
         # docker.stackable.tech/stackable/kafka@sha256:91...

diff --git a/run-integration-test/README.md b/run-integration-test/README.md
@@ -0,0 +1,29 @@
+# `run-integration-test`
+
+> Manifest: [run-integration-test/action.yml][run-integration-test]
+
+This action runs an operator integration test. It does the following work:
+
+1. Create a test cluster on-the-fly using the requested Kubernetes version and distribution via
+   Replicated.
+2. Run the integration test based on the provided test parameters.
+3. Delete the cluster of the tests are done and send out a notification on failure.
+
+## Inputs and Outputs
+
+> [!TIP]
+> For descriptions of the inputs and outputs, see the complete [run-integration-test] action.
+
+### Inputs
+
+- `test-platform`(required, eg: `kind-1.31.0-amd64`)
+- `test-run` (required, `test-suite` or `test`)
+- `test-parameter` (defaults to `smoke`)
+- `replicated-api-token` (required)
+
+### Outputs
+
+- `start-time`
+- `end-time`
+
+[run-integration-test]: ./action.yml
diff --git a/run-integration-test/action.yml b/run-integration-test/action.yml
@@ -0,0 +1,216 @@
+---
+name: Run Integration Test
+description: |
+  This action runs Stackable Operator integration tests on various platforms and
+  Kubernetes distributions.
+inputs:
+  test-platform:
+    description: |
+      The platform/distribution to run on (eg: `okd-4.15-amd64`)
+    required: true
+  test-run:
+    description: Type of test run
+    required: true
+  test-parameter:
+    description: Parameter to `--test-suite` or `--test` (ignored for `all`)
+    default: ""
+  replicated-api-token:
+    description: Replicated API token (only needed if running on replicated)
+    default: ""
+outputs:
+  start-time:
+    description: The date and time this integration test was started.
+    value: ${{ steps.start-time.outputs.START_TIME }}
+  end-time:
+    description: The date and time this integration test finished.
+    value: ${{ steps.end-time.outputs.END_TIME }}
+runs:
+  using: composite
+  steps:
+    - name: Record Start Time
+      id: start-time
+      shell: bash
+      run: |
+        echo "START_TIME=$(date +'%Y-%m-%dT%H:%M:%S')" | tee -a "$GITHUB_OUTPUT"
+
+    - name: Extract Test and Instance Configuration
+      env:
+        TEST_PARAMETER: ${{ inputs.test-parameter }}
+        TEST_PLATFORM: ${{ inputs.test-platform }}
+        TEST_RUN: ${{ inputs.test-run }}
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        #####################################
+        # Extract Kubernetes-related Values #
+        #####################################
+
+        export KUBERNETES_DISTRIBUTION=$(echo "$TEST_PLATFORM" | cut -d - -f 1)
+        export KUBERNETES_VERSION=$(echo "$TEST_PLATFORM" | cut -d - -f 2)
+        export KUBERNETES_ARCHITECTURE=$(echo "$TEST_PLATFORM" | cut -d - -f 3)
+
+        echo "KUBERNETES_DISTRIBUTION=$KUBERNETES_DISTRIBUTION" | tee -a "$GITHUB_ENV"
+        echo "KUBERNETES_VERSION=$KUBERNETES_VERSION" | tee -a "$GITHUB_ENV"
+        echo "KUBERNETES_ARCHITECTURE=$KUBERNETES_ARCHITECTURE" | tee -a "$GITHUB_ENV"
+
+        ##################################
+        # Extract Instance Configuration #
+        ##################################
+
+        export INSTANCE_SIZE=$(yq '.instance-size' -e ./tests/infrastructure.yaml)
+        INSTANCE_TYPE=$(yq '.[env(KUBERNETES_DISTRIBUTION)].[env(KUBERNETES_ARCHITECTURE)].[env(INSTANCE_SIZE)]' -e "$GITHUB_ACTION_PATH/instances.yml")
+
+        echo "INSTANCE_TYPE=$INSTANCE_TYPE" | tee -a "$GITHUB_ENV"
+
+        ############################
+        # Validate Test Parameters #
+        ############################
+
+        if [ -z "$TEST_RUN" ]; then
+          echo "TEST_RUN must be defined and not empty"
+          exit 1
+        fi
+
+        if [ "$TEST_RUN" != "all" ]; then
+          if [ -z "$TEST_PARAMETER" ]; then
+            echo "TEST_PARAMETER must be defined and not empty"
+            exit 1
+          fi
+
+          if [ "$TEST_RUN" == "test-suite" ]; then
+            yq '.suites[] | select(.name == env(TEST_PARAMETER))' -e ./tests/test-definition.yaml
+          elif [ "$TEST_RUN" == "test" ]; then
+            yq '.tests[] | select(.name == env(TEST_PARAMETER))' -e ./tests/test-definition.yaml
+          fi
+        fi
+
+        echo "TEST_PARAMETER=$TEST_PARAMETER" | tee -a "$GITHUB_ENV"
+        echo "TEST_RUN=$TEST_RUN" | tee -a "$GITHUB_ENV"
+
+    - name: Prepare Replicated Cluster
+      if: env.KUBERNETES_DISTRIBUTION != 'ionos'
+      id: prepare-replicated-cluster
+      uses: replicatedhq/replicated-actions/create-cluster@v1 # todo, hash
+      with:
+        # See: https://github.com/replicatedhq/replicated-actions/tree/main/create-cluster#inputs
+        api-token: ${{ inputs.replicated-api-token }}
+        cluster-name: integration-test-${{ github.repository }}-${{ github.run_id }}
+        instance-type: ${{ env.INSTANCE_TYPE }}
+        kubernetes-distribution: ${{ env.KUBERNETES_DISTRIBUTION }}
+        kubernetes-version: ${{ env.KUBERNETES_VERSION }}
+        ttl: 4h # todo: allow this to be configurable
+        disk: 50 # todo: allow this to be configurable
+        nodes: 1 # todo: allow this to be configurable
+        tags: |
+          - key: node-architecture
+            value: ${{ env.KUBERNETES_ARCHITECTURE }}
+          - key: kubernetes-distribution
+            value: ${{ env.KUBERNETES_DISTRIBUTION }}
+          - key: triggered-by
+            value: ${{ github.triggering_actor }}
+
+    - name: Set Replicated kubeconfig
+      if: env.KUBERNETES_DISTRIBUTION != 'ionos'
+      env:
+        KUBECONFIG: ${{ steps.prepare-replicated-cluster.outputs.cluster-kubeconfig }}
+      shell: bash
+      run: |
+        set -euo pipefail
+        mkdir ~/.kube
+        echo "$KUBECONFIG" > ~/.kube/config
+
+    - name: Extract Operator Name
+      env:
+        REPOSITORY: ${{ github.repository }}
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        OPERATOR_NAME=$(echo "$REPOSITORY" | cut -d / -f 2 | sed 's/-operator//g')
+        echo "OPERATOR_NAME=$OPERATOR_NAME" | tee -a "$GITHUB_ENV"
+
+    - name: Setup Tool Directory
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        TOOL_DIRECTORY="$HOME/.local/bin"
+        mkdir -p "$TOOL_DIRECTORY"
+
+        echo "$TOOL_DIRECTORY" | tee -a "$GITHUB_PATH"
+        echo "TOOL_DIRECTORY=$TOOL_DIRECTORY" | tee -a "$GITHUB_ENV"
+
+    # We don't need to install kubectl, kind or helm because it is already part of the installed
+    # tools of the runner image.
+    # See https://github.com/actions/runner-images/blob/main/images/ubuntu/scripts/build/install-kubernetes-tools.sh
+    - name: Install kubectl-kuttl
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        curl -L -o "$TOOL_DIRECTORY/kubectl-kuttl" https://github.com/kudobuilder/kuttl/releases/download/v0.19.0/kubectl-kuttl_0.19.0_linux_x86_64
+        chmod +x "$TOOL_DIRECTORY/kubectl-kuttl"
+
+    # Python3 is already installed, if we ever need to specify the version, we can use the
+    # setup-python action.
+    # See https://github.com/actions/runner-images/blob/main/images/ubuntu/scripts/build/install-python.sh
+    - name: Install beku
+      shell: bash
+      run: |
+        set -euo pipefail
+        pip install beku-stackabletech
+
+    # mikefarah/yq is already installed on the runner
+    # See https://github.com/actions/runner-images/blob/main/images/ubuntu/scripts/build/install-yq.sh
+
+    - name: Install stackablectl
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        curl -L -o "$TOOL_DIRECTORY/stackablectl" https://github.com/stackabletech/stackable-cockpit/releases/latest/download/stackablectl-x86_64-unknown-linux-gnu
+        chmod +x "$TOOL_DIRECTORY/stackablectl"
+
+    - name: Install apt packages
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        sudo apt update
+        sudo apt install -y \
+          gettext-base
+
+    - name: Run Integration Test (${{ inputs.test-run }}=${{ inputs.test-parameter }})
+      env:
+        REF_NAME: ${{ github.ref_name }}
+        GH_TOKEN: ${{ github.token }}
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        OPERATOR_VERSION=$("$GITHUB_ACTION_PATH/../.scripts/get_operator_version.sh" "$REF_NAME")
+        python ./scripts/run-tests --skip-tests --operator "$OPERATOR_NAME=$OPERATOR_VERSION"
+
+        if [ "$TEST_RUN" == "all" ]; then
+          python ./scripts/run-tests --skip-release --log-level debug
+        else
+          python ./scripts/run-tests --skip-release --log-level debug "--$TEST_RUN" "$TEST_PARAMETER"
+        fi
+
+    - name: Destroy Replicated Cluster
+      if: env.KUBERNETES_DISTRIBUTION != 'ionos' && always()
+      # If the creation of the cluster failed, we don't want to error and abort
+      continue-on-error: true
+      uses: replicatedhq/replicated-actions/remove-cluster@v1 # todo, hash
+      with:
+        # See: https://github.com/replicatedhq/replicated-actions/tree/main/remove-cluster#inputs
+        api-token: ${{ inputs.replicated-api-token }}
+        cluster-id: ${{ steps.prepare-replicated-cluster.outputs.cluster-id }}
+
+    - name: Record End Time
+      id: end-time
+      if: always()
+      shell: bash
+      run: |
+        echo "END_TIME=$(date +'%Y-%m-%dT%H:%M:%S')" | tee -a "$GITHUB_OUTPUT"
diff --git a/run-integration-test/instances.yml b/run-integration-test/instances.yml
@@ -0,0 +1,35 @@
+eks:
+  arm64:
+    small: m7g.large
+    medium: m7g.2xlarge
+    large: m7g.4xlarge
+  amd64:
+    small: m6i.large
+    medium: m6i.2xlarge
+    large: m6i.4xlarge
+
+gke:
+  arm64:
+    small: t2a-standard-2
+    medium: t2a-standard-8
+    large: t2a-standard-16
+  amd64:
+    small: e2-standard-2
+    medium: e2-standard-8
+    large: e2-standard-16
+
+aks:
+  arm64:
+    small: Standard_D2ps_v5
+    medium: Standard_D8ps_v5
+    large: Standard_D16ps_v5
+  amd64:
+    small: Standard_DS1_v2
+    medium: Standard_DS3_v2
+    large: Standard_DS5_v2
+
+kind:
+  amd64:
+    small: r1.small
+    medium: r1.medium
+    large: r1.large
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/usr/bin/env bash

		uname -m \| sed -e "s#x86_64#amd64#" \| sed -e "s#aarch64#arm64#"