diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
index 433fbbd..09bf3f9 100644
--- a/.github/actions/test/action.yml
+++ b/.github/actions/test/action.yml
@@ -20,6 +20,7 @@ runs:
       with:
         cli-version: latest
         balena-token: ${{ fromJSON(inputs.secrets).BALENA_API_KEY }}
+        skip-cache: 'true'
     - name: Print balena CLI version
       shell: bash
       run: |
diff --git a/action.yml b/action.yml
index 8229ac6..029e5d0 100644
--- a/action.yml
+++ b/action.yml
@@ -11,20 +11,16 @@ inputs:
   balena-token:
     description: "balenaCloud API token to login automatically"
     required: false
+  skip-cache:
+    description: "Skip using the tool cache and always re-download"
+    required: false
 
 # Builds and then runs as separate steps as default GitHub method does not allow passing build args
 runs:
   using: "composite"
   steps:
-    - name: Restore balena CLI cache
-      id: cache
-      uses: actions/cache/restore@v4
-      with:
-        path: ${{ runner.tool_cache }}/balena-cli
-        key: balena-cli-${{ runner.os }}-${{ runner.arch }}
-
     - name: Get latest release
-      if: ${{ inputs.cli-version }} != 'latest' || ${{ inputs.cli-version }} != ''
+      if: inputs.cli-version != 'latest' || inputs.cli-version != ''
       id: latest
       shell: bash
       working-directory: ${{ runner.temp }}
@@ -32,6 +28,14 @@ runs:
         release="$(curl -s https://api.github.com/repos/balena-io/balena-cli/releases/latest | jq -r '.tag_name')"
         echo "release=${release}" >> "${GITHUB_OUTPUT}"
 
+    - name: Restore balena CLI cache
+      if: inputs.skip-cache == 'true'
+      id: cache
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ runner.tool_cache }}/balena-cli
+        key: balena-cli-${{ runner.os }}-${{ runner.arch }}-${{ steps.latest.outputs.release || inputs.cli-version }}
+
     - name: Download balena CLI (Linux X64)
       if: runner.os == 'Linux' && runner.arch == 'x64' && ! steps.cache.outputs.cache-hit
       working-directory: ${{ runner.temp }}
@@ -42,7 +46,7 @@ runs:
       run: |
         set -x
         curl -fsSL "${REPO}/releases/download/${VERSION}/balena-cli-${VERSION}-linux-x64-standalone.zip" -o balena-cli.zip
-        unzip balena-cli.zip -d balena-cli
+        unzip balena-cli.zip -d .
 
     - name: Download balena CLI (Linux ARM64)
       if: runner.os == 'Linux' && runner.arch == 'ARM64' && ! steps.cache.outputs.cache-hit
@@ -54,7 +58,7 @@ runs:
       run: |
         set -x
         curl -fsSL "${REPO}/releases/download/${VERSION}/balena-cli-${VERSION}-linux-arm64-standalone.zip" -o balena-cli.zip
-        unzip balena-cli.zip -d balena-cli
+        unzip balena-cli.zip -d .
 
     - name: Download balena CLI (Windows X64)
       if: runner.os == 'Windows' && runner.arch == 'x64' && ! steps.cache.outputs.cache-hit
@@ -66,7 +70,7 @@ runs:
       run: |
         set -x
         curl -fsSL "${REPO}/releases/download/${VERSION}/balena-cli-${VERSION}-windows-x64-standalone.zip" -o balena-cli.zip
-        unzip balena-cli.zip -d balena-cli
+        unzip balena-cli.zip -d .
 
     - name: Download balena CLI (macOS X64)
       if: runner.os == 'macOS' && runner.arch == 'x64' && ! steps.cache.outputs.cache-hit
@@ -78,7 +82,7 @@ runs:
       run: |
         set -x
         curl -fsSL "${REPO}/releases/download/${VERSION}/balena-cli-${VERSION}-macOS-x64-standalone.zip" -o balena-cli.zip
-        unzip balena-cli.zip -d balena-cli
+        unzip balena-cli.zip -d .
 
     - name: Download balena CLI (macOS ARM64)
       if: runner.os == 'macOS' && runner.arch == 'ARM64' && ! steps.cache.outputs.cache-hit
@@ -90,7 +94,7 @@ runs:
       run: |
         set -x
         curl -fsSL "${REPO}/releases/download/${VERSION}/balena-cli-${VERSION}-macOS-arm64-standalone.zip" -o balena-cli.zip
-        unzip balena-cli.zip -d balena-cli
+        unzip balena-cli.zip -d .
 
     - name: Install balena CLI to tool cache
       uses: AnimMouse/tool-cache@v1
@@ -99,11 +103,11 @@ runs:
         cache_hit: ${{ steps.cache.outputs.cache-hit }}
 
     - name: Save balena CLI cache
-      if: '! steps.cache.outputs.cache-hit'
+      if: "! steps.cache.outputs.cache-hit"
       uses: actions/cache/save@v4
       with:
         path: ${{ runner.tool_cache }}/balena-cli
-        key: balena-cli-${{ runner.os }}-${{ runner.arch}}
+        key: balena-cli-${{ runner.os }}-${{ runner.arch }}-${{ steps.latest.outputs.release || inputs.cli-version }}
 
     - name: Print balena CLI version
       shell: bash