diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
index efb0c5a8bd..ad8e2711a2 100644
--- a/.github/workflows/actions.yml
+++ b/.github/workflows/actions.yml
@@ -27,15 +27,7 @@ jobs:
           key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
           restore-keys: |
             ${{ runner.os }}-pip-
-      - name: Setup SSH Keys and known_hosts
-        env:
-            SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-        run: |
-            ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-            ssh-add - <<< "${{ secrets.KERAS_CORE }}"
       - name: Install dependencies
-        env:
-            SSH_AUTH_SOCK: /tmp/ssh_agent.sock
         run: |
           pip install -r requirements.txt --progress-bar off
           pip install --no-deps -e "." --progress-bar off
@@ -70,15 +62,7 @@ jobs:
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
         restore-keys: |
           ${{ runner.os }}-pip-
-    - name: Setup SSH Keys and known_hosts
-      env:
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-      run: |
-          ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-          ssh-add - <<< "${{ secrets.KERAS_CORE }}"
     - name: Install dependencies
-      env:
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
       run: |
           pip install -r requirements.txt --progress-bar off
           pip install --no-deps -e "." --progress-bar off
diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml
index 0e2a8a1713..536134bcdd 100644
--- a/.github/workflows/publish-to-pypi.yml
+++ b/.github/workflows/publish-to-pypi.yml
@@ -7,15 +7,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@master
-    - name: Setup SSH Keys and known_hosts
-      env:
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-      run: |
-          ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-          ssh-add - <<< "${{ secrets.KERAS_CORE }}"
-    - name: Install dependencies
-      env:
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
     - name: Install dependencies
       run: |
           pip install -r requirements.txt --progress-bar off