diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml
index 522b94dca3..e3e91f0419 100644
--- a/.github/actions/setup-go/action.yml
+++ b/.github/actions/setup-go/action.yml
@@ -7,6 +7,12 @@ inputs:
   key-prefix:
     description: "A prefix to use for the cache key, to separate cache entries from other workflows"
     required: false
+  use-build-cache: 
+    description: "Whether to use the build cache"
+    required: false
+    # Boolean values aren't supported in the workflow syntax, so we use a
+    # string. To not confuse the value with true/false, we use 'yes' and 'no'.
+    default: 'yes'
 
 runs:
   using: "composite"
@@ -17,7 +23,8 @@ runs:
       with:
         go-version: '${{ inputs.go-version }}'
 
-    - name: go cache
+    - name: go module and build cache
+      if: ${{ inputs.use-build-cache == 'yes' }}
       uses: actions/cache@v3
       with:
         # In order:
@@ -35,6 +42,18 @@ runs:
           ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ github.job }}-
           ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-
 
+    - name: go module cache
+      if: ${{ inputs.use-build-cache == 'no' }}
+      uses: actions/cache@v3
+      with:
+        # Just the module download cache.
+        path: |
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ github.job }}-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ github.job }}-
+          ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-
+
     - name: set GOPATH
       shell: bash
       run: |
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index dd8b5dbfe3..290a3b8377 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -148,6 +148,7 @@ jobs:
         with:
           go-version: '${{ env.GO_VERSION }}'
           key-prefix: cross-compile
+          use-build-cache: 'no'
 
       - name: build release for all architectures
         run: make release