e2e.yaml

name: e2e
on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - "main"
      - "release/**"

permissions:
  contents: read # for actions/checkout to fetch code

jobs:
  kind:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Setup QEMU
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
      - name: Setup Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
        with:
          buildkitd-flags: "--debug"
      - name: Cache Docker layers
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        id: cache
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-ghcache-
      - name: Setup Go
        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:
          go-version-file: 'go.mod'
          cache-dependency-path: |
            **/go.sum
            **/go.mod
      - name: Setup Kubernetes
        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
        with:
          version: v0.20.0
          cluster_name: kind
          node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
      - name: Setup Helm
        uses: fluxcd/pkg/actions/helm@main
      - name: Setup Kustomize
        uses: fluxcd/pkg/actions/kustomize@main
      - name: Run tests
        run: make test
      - name: Check if working tree is dirty
        run: |
          if [[ $(git diff --stat) != '' ]]; then
            git --no-pager diff
            echo 'run make test and commit changes'
            exit 1
          fi
      - name: Build container image
        run: |
          make docker-build IMG=test/helm-controller:latest \
            BUILD_PLATFORMS=linux/amd64 \
            BUILD_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache \
              --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \
              --load"
      - # Temp fix
        # https://github.com/docker/build-push-action/issues/252
        # https://github.com/moby/buildkit/issues/1896
        name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
      - name: Load test image
        run: kind load docker-image test/helm-controller:latest
      - name: Install CRDs
        run: make install
      - name: Run default status test
        run: |
          kubectl apply -f config/testdata/status-defaults
          RESULT=$(kubectl get helmrelease status-defaults -o go-template={{.status}})
          EXPECTED='map[observedGeneration:-1]'
          if [ "${RESULT}" != "${EXPECTED}" ] ; then
            echo -e "${RESULT}\n\ndoes not equal\n\n${EXPECTED}"
            exit 1
          fi
      - name: Deploy controllers
        run: |
          make dev-deploy IMG=test/helm-controller:latest
          kubectl -n helm-system rollout status deploy/source-controller --timeout=1m
          kubectl -n helm-system rollout status deploy/helm-controller --timeout=1m
        env:
          KUBEBUILDER_ASSETS: ${{ github.workspace }}/kubebuilder/bin
      - name: Test samples
        run: |
          kubectl create ns samples
          kubectl -n samples apply -f config/samples
          kubectl -n samples wait hr/podinfo-ocirepository --for=condition=ready --timeout=4m
          kubectl -n samples wait hr/podinfo-gitrepository --for=condition=ready --timeout=4m
          kubectl -n samples wait hr/podinfo-helmrepository --for=condition=ready --timeout=4m
          kubectl delete ns samples
      - name: Install sources
        run: |
          kubectl -n helm-system apply -f config/testdata/sources
      - name: Run smoke test
        run: |
          kubectl -n helm-system apply -f config/testdata/podinfo
          kubectl -n helm-system wait helmreleases/podinfo --for=condition=ready --timeout=4m
          kubectl -n helm-system wait helmreleases/podinfo-git --for=condition=ready --timeout=4m
          kubectl -n helm-system wait helmreleases/podinfo-oci --for=condition=ready --timeout=4m
          kubectl -n helm-system delete -f config/testdata/podinfo
      - name: Run dependency tests
        run: |
          kubectl -n helm-system apply -f config/testdata/dependencies
          kubectl -n helm-system wait helmreleases/backend --for=condition=ready --timeout=4m
          kubectl -n helm-system wait helmreleases/frontend --for=condition=ready --timeout=4m
          kubectl -n helm-system delete -f config/testdata/dependencies
      - name: Run values test
        run: |
          kubectl -n helm-system apply -f config/testdata/valuesfrom
          kubectl -n helm-system wait helmreleases/valuesfrom --for=condition=ready --timeout=4m

          RESULT=$(helm -n helm-system get values valuesfrom)
          EXPECTED=$(cat config/testdata/valuesfrom/golden.txt)
          if [ "$RESULT" != "$EXPECTED" ]; then
            echo -e "$RESULT\n\ndoes not equal\n\n$EXPECTED"
            exit 1
          fi

           kubectl -n helm-system delete -f config/testdata/valuesfrom
      - name: Run target namespace test
        run: |
          kubectl -n helm-system apply -f config/testdata/targetnamespace
          kubectl -n helm-system wait helmreleases/targetnamespace --for=condition=ready --timeout=4m

          # Confirm release in "default" namespace
          kubectl -n default get deployment default-targetnamespace-podinfo

          kubectl -n helm-system delete -f config/testdata/targetnamespace
      - name: Run install create target namespace test
        run: |
          kubectl -n helm-system apply -f config/testdata/install-create-target-ns
          kubectl -n helm-system wait helmreleases/install-create-target-ns --for=condition=ready --timeout=4m

          # Confirm release in "install-create-target-ns" namespace
          kubectl -n install-create-target-ns get deployment install-create-target-ns-install-create-target-ns-podinfo

          kubectl -n helm-system delete -f config/testdata/install-create-target-ns
      - name: Run install from helmChart test
        run: |
          kubectl -n helm-system apply -f config/testdata/install-from-hc-source
          kubectl -n helm-system wait helmreleases/podinfo-from-hc --for=condition=ready --timeout=4m
          kubectl -n helm-system delete -f config/testdata/install-from-hc-source
      - name: Run install from ocirepo test
        run: |
          kubectl -n helm-system apply -f config/testdata/install-from-ocirepo-source
          kubectl -n helm-system wait helmreleases/podinfo-from-ocirepo --for=condition=ready --timeout=4m
          kubectl -n helm-system delete -f config/testdata/install-from-ocirepo-source
      - name: Run install fail test
        run: |
          test_name=install-fail
          kubectl -n helm-system apply -f config/testdata/$test_name
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was installed and not uninstalled.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl -n helm-system delete -f config/testdata/$test_name
      - name: Run install test fail test
        run: |
          test_name=install-test-fail
          kubectl -n helm-system apply -f config/testdata/$test_name
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .TestSuccess=="False" and .Ready=="False"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was installed and not uninstalled.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl -n helm-system delete -f config/testdata/$test_name
      - name: Run install test fail ignore test
        run: |
          test_name=install-test-fail-ignore
          kubectl -n helm-system apply -f config/testdata/$test_name
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .TestSuccess=="False" and .Ready=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was installed and not uninstalled.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl -n helm-system delete -f config/testdata/$test_name
      - name: Run install fail with remediation test
        run: |
          test_name=install-fail-remediate
          kubectl -n helm-system apply -f config/testdata/$test_name
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Remediated=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Ensure release was uninstalled.
          RELEASE_STATUS=$(helm -n helm-system history $test_name -o json | jq -r 'if length == 1 then .[0].status else empty end')
          if [ "$RELEASE_STATUS" != "uninstalled" ]; then
            echo -e "Unexpected release status: $RELEASE_STATUS"
            exit 1
          fi

          kubectl -n helm-system delete -f config/testdata/$test_name
          helm -n helm-system delete $test_name
      - name: Run install fail with retry test
        run: |
          test_name=install-fail-retry
          kubectl -n helm-system apply -f config/testdata/$test_name
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.installFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Stalled=="True" )' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was installed.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl -n helm-system delete -f config/testdata/$test_name
      - name: Run upgrade fail test
        run: |
          test_name=upgrade-fail
          kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was installed.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Stalled=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was upgraded and not rolled back.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 2 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl delete -n helm-system -f config/testdata/$test_name/install.yaml
      - name: Run upgrade test fail test
        run: |
          test_name=upgrade-test-fail
          kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was installed.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .TestSuccess=="False" and .Ready=="False" and .Stalled=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was upgraded and not rolled back.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 2 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl delete -n helm-system -f config/testdata/$test_name/install.yaml
      - name: Run upgrade fail with remediation test
        run: |
          test_name=upgrade-fail-remediate
          kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was installed.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Remediated=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was upgraded and then rolled back.
          HISTORY=$(helm -n helm-system history -o json $test_name)
          REVISION_COUNT=$(echo "$HISTORY" | jq 'length')
          if [ "$REVISION_COUNT" != 3 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
          fi
          LAST_REVISION_DESCRIPTION=$(echo "$HISTORY" | jq -r 'last | .description')
          if [ "$LAST_REVISION_DESCRIPTION" != "Rollback to 1" ]; then
            echo -e "Unexpected last revision description: $LAST_REVISION_DESCRIPTION"
            exit 1
          fi

          kubectl delete -n helm-system -f config/testdata/$test_name/install.yaml
      - name: Run upgrade fail retry test
        run: |
          test_name=upgrade-fail-retry
          kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was installed.
          REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi

          kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" )' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was upgraded and rolled back twice.
          HISTORY=$(helm -n helm-system history -o json $test_name)
          REVISION_COUNT=$(echo "$HISTORY" | jq 'length')
          if [ "$REVISION_COUNT" != 5 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
          fi
          LAST_REVISION_DESCRIPTION=$(echo "$HISTORY" | jq -r 'last | .description')
          if [ "$LAST_REVISION_DESCRIPTION" != "Rollback to 3" ]; then
            echo -e "Unexpected last revision description: $LAST_REVISION_DESCRIPTION"
            exit 1
          fi

          kubectl delete -n helm-system -f config/testdata/$test_name/install.yaml
      - name: Run upgrade from ocirepo source
        run: |
            test_name=upgrade-from-ocirepo-source
            kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
            echo -n ">>> Waiting for expected conditions"
            count=0
            until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
              echo -n '.'
              sleep 5
              count=$((count + 1))
              if [[ ${count} -eq 24 ]]; then
                echo ' No more retries left!'
                exit 1
              fi
            done
            echo ' done'
  
            # Validate release was installed.
            REVISION_COUNT=$(helm -n helm-system history -o json $test_name | jq 'length')
            if [ "$REVISION_COUNT" != 1 ]; then
              echo -e "Unexpected revision count: $REVISION_COUNT"
              exit 1
            fi
  
            kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
            echo -n ">>> Waiting for expected conditions"
            count=0
            until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
              echo -n '.'
              sleep 5
              count=$((count + 1))
              if [[ ${count} -eq 24 ]]; then
                echo ' No more retries left!'
                exit 1
              fi
            done
            echo ' done'
  
            kubectl delete -n helm-system -f config/testdata/$test_name/install.yaml
      - name: Run upgrade fail with uninstall remediation strategy test
        run: |
          test_name=upgrade-fail-remediate-uninstall
          kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was installed.
          HISTORY=$(helm -n helm-system history -o json $test_name)
          REVISION_COUNT=$(echo "$HISTORY" | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi
          LAST_REVISION_STATUS=$(echo "$HISTORY" | jq -r 'last | .status')
          if [ "$LAST_REVISION_STATUS" != "deployed" ]; then
            echo -e "Unexpected last revision status: $LAST_REVISION_STATUS"
            exit 1
          fi

          kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 1 and .status.installFailures == 1 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" )' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'

          # Validate release was uninstalled/reinstalled.
          HISTORY=$(helm -n helm-system history -o json $test_name)
          REVISION_COUNT=$(echo "$HISTORY" | jq 'length')
          if [ "$REVISION_COUNT" != 1 ]; then
            echo -e "Unexpected revision count: $REVISION_COUNT"
            exit 1
          fi
          LAST_REVISION_STATUS=$(echo "$HISTORY" | jq -r 'last | .status')
          if [ "$LAST_REVISION_STATUS" != "failed" ]; then
            echo -e "Unexpected last revision status: $LAST_REVISION_STATUS"
            exit 1
          fi

          kubectl delete -n helm-system -f config/testdata/$test_name/install.yaml
      - name: Run impersonation tests
        run: |
          kubectl apply -f config/testdata/impersonation
          kubectl -n impersonation wait helmreleases/podinfo --for=condition=ready --timeout=2m
          echo -n ">>> Waiting for expected conditions"
          count=0
          until [ 'true' == "$( kubectl -n impersonation get helmrelease/podinfo-fail -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False"' )" ]; do
            echo -n '.'
            sleep 5
            count=$((count + 1))
            if [[ ${count} -eq 24 ]]; then
              echo ' No more retries left!'
              exit 1
            fi
          done
          echo ' done'
      - name: Run delete-ns tests
        run: |
          kubectl apply -f config/testdata/delete-ns
          kubectl -n delete-ns wait helmreleases/podinfo --for=condition=ready --timeout=2m
          kubectl delete ns delete-ns 1>/dev/null 2>&1 &
          echo -n ">>> Waiting for namespace to be deleted"
          if kubectl wait --for=delete namespace delete-ns --timeout=5m; then
            echo ' Namespace deleted successfully'
          else
            echo ' Timed out waiting for namespace to be deleted'
            kubectl get all -n delete-ns
            exit 1
          fi
      - name: Run post-renderer-kustomize test
        run: |
          kubectl -n helm-system apply -f config/testdata/post-renderer-kustomize
          kubectl -n helm-system wait helmreleases/post-renderer-kustomize --for=condition=ready --timeout=4m
          RESULT=$(kubectl get deployment -n helm-system mypodinfo -o jsonpath='{.metadata.labels.xxxx}')
          if [ "$RESULT" != "yyyy" ]; then
            echo -e "$RESULT\n\ndoes not equal\n\nyyyy"
            exit 1
          fi
          RESULT=$(kubectl get deployment -n helm-system mypodinfo -o jsonpath='{.metadata.labels.yyyy}')
          if [ "$RESULT" != "xxxx" ]; then
            echo -e "$RESULT\n\ndoes not equal\n\nxxxx"
            exit 1
          fi
          kubectl -n helm-system delete -f config/testdata/post-renderer-kustomize
      - name: Bootstrap CRDs Upgrade Tests
        if: ${{ startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/') }}
        run: |
          REF=${{ github.ref }}
          if echo "$REF" | grep 'refs/tags/'; then 
            TYPE=tag
            REF=${REF#refs/tags/}
          else
            TYPE=branch
            if echo "$REF" | grep 'refs/pull/'; then 
              REF=${REF#refs/pull/}
            else
              REF=${REF#refs/heads/}
            fi
          fi
          echo "$HEAD_REF,$CURR_REF -> $REF of type $TYPE"
          echo "helm install --namespace default --set $TYPE=$REF --set url=https://github.com/${{ github.repository }} this config/testdata/charts/crds/bootstrap"
          helm install --namespace default --set $TYPE=$REF --set url=https://github.com/${{ github.repository }} this config/testdata/charts/crds/bootstrap
          kubectl -n default apply -f config/testdata/crds-upgrade/init
          kubectl -n default wait helmreleases/crds-upgrade-test --for=condition=ready --timeout=2m
      - name: CRDs Upgrade Test Create
        if: ${{ startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/') }}
        run: |
          kubectl -n default apply -f config/testdata/crds-upgrade/create
          kubectl -n default wait helmreleases/crds-upgrade-test --for=condition=ready --timeout=2m
      - name: CRDs Upgrade Test CreateReplace
        if: ${{ startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/') }}
        run: |
          kubectl -n default apply -f config/testdata/crds-upgrade/create-replace
          kubectl -n default wait helmreleases/crds-upgrade-test --for=condition=ready --timeout=2m
      - name: Logs
        run: |
          kubectl -n helm-system logs deploy/source-controller
          kubectl -n helm-system logs deploy/helm-controller
      - name: Debug failure
        if: failure()
        run: |
          kubectl -n helm-system get helmrepositories -oyaml || true
          kubectl -n helm-system get helmcharts -oyaml || true
          kubectl -n helm-system get helmreleases -oyaml || true
          kubectl -n helm-system get all
          helm ls -n helm-system --all
          kubectl -n helm-system logs deploy/source-controller || true
          kubectl -n helm-system logs deploy/helm-controller || true