MRG: Merge pull request #18 from octue/upgrade-deployment-workflow

Upgrade deployment workflow
octue · Oct 25, 2024 · c75f705 · c75f705
2 parents b35fddf + d60b58e
commit c75f705
Show file tree

Hide file tree

Showing 6 changed files with 137 additions and 32 deletions.
diff --git a/.github/workflows/check-semantic-version.yml b/.github/workflows/check-semantic-version.yml
@@ -17,7 +17,7 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'skipci')"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: octue/[email protected]

diff --git a/.github/workflows/deploy-cloud-run-django-server.yml b/.github/workflows/deploy-cloud-run-django-server.yml
@@ -20,13 +20,6 @@ on:
       server_name:
         type: string
         required: true
-      cloud_run_base_url:
-        type: string
-        required: true
-      gcp_environment:
-        type: string
-        required: false
-        default: 'main'
       dockerfile:
         type: string
         required: false
@@ -46,6 +39,16 @@ on:
         required: false
         default: ""
         description: Defaults to "<server_name>.settings"
+      env_vars:
+        type: string
+        required: false
+        default: ''
+        description: 'Additional environment variables to use with the `collectstatic` and `migrate` steps. Provide in comma-separated form e.g. "VAR1=SOMETHING,VAR2=20".'
+      gcp_secrets:
+        type: string
+        required: false
+        default: ''
+        description: 'Secrets from GCP Secret Manager to use with the `collectstatic` and `migrate` steps. Provide in comma-separated form with each secret in the form "ENVIRONMENT_VARIABLE:secret-path" e.g. "FIRST_SECRET:my-project/first-secret,SECOND_SECRET:my-project/second-secret"'
 
     secrets:
       build_secrets:
@@ -79,21 +82,20 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install poetry
         uses: snok/install-poetry@v1
 
       - name: Get deployment info
         id: get-deployment-info
-        uses: octue/get-deployment-info@0.2.1
+        uses: octue/get-deployment-info@0.3.3
         with:
           gcp_project_name: ${{ inputs.gcp_project_name }}
           gcp_project_number: ${{ inputs.gcp_project_number }}
           gcp_region: ${{ inputs.gcp_region }}
           gcp_resource_affix: ${{ inputs.gcp_resource_affix }}
           gcp_service_name: ${{ inputs.server_name }}
-          gcp_environment: ${{ inputs.gcp_environment }}
 
   build:
     runs-on: ubuntu-latest
@@ -106,7 +108,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -169,7 +171,7 @@ jobs:
     steps:
       - name: Checkout
         # Shouldn't be necessary since we pull code in an image, but it's required by google-github-actions/auth
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -189,23 +191,22 @@ jobs:
       - name: Setup GCloud CLI
         uses: "google-github-actions/setup-gcloud@v2"
 
-      - name: Get SecretManager prefix
+      - name: Get and show Secret Manager prefix
         id: secrets_prefix
         run: |
-          echo "prefix=${{ needs.info.outputs.gcp_project_name }}/${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}" >> $GITHUB_OUTPUT
-
-      - name: Show SecretManager prefix
-        run: |
-          echo "Secretmanager prefix: ${{ steps.secrets_prefix.outputs.prefix }}"
+          export prefix=${{ needs.info.outputs.gcp_project_name }}/${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}
+          echo "prefix=$prefix" >> $GITHUB_OUTPUT
+          echo "Secret Manager prefix: $prefix"
 
-      - name: Access GCP SecretManager
+      - name: Access GCP Secret Manager
         id: secrets
         uses: "google-github-actions/get-secretmanager-secrets@v2"
         with:
           secrets: |-
             DATABASE_URL:${{ steps.secrets_prefix.outputs.prefix }}-db-proxy-uri
             DJANGO_SECRET_KEY:${{ steps.secrets_prefix.outputs.prefix }}-django-secret-key
             GOOGLE_APPLICATION_CREDENTIALS:${{ steps.secrets_prefix.outputs.prefix }}-google-application-credentials
+            ${{ inputs.gcp_secrets }}
 
       - name: Connect to Cloud SQL
         # Note the DATABASE_URL should point to the proxy, i.e. localhost:5432, not to the actual DB
@@ -237,7 +238,7 @@ jobs:
             echo "django_settings_module=${{ inputs.django_settings_module }}" >> $GITHUB_OUTPUT
           fi
 
-      - name: Set dotenv
+      - name: Set environment variables in .env file
         # Set here rather than piped directly to the container, to allow us to usefully debug later steps of this action via reverse shell
         run: |
           touch .env
@@ -248,6 +249,43 @@ jobs:
           echo "GCP_TASKS_RESOURCE_AFFIX=${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}" >> .env
           echo "GCP_TASKS_DEFAULT_QUEUE_NAME=${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}" >> .env
           echo "GCP_TASKS_DOMAIN=https://doesnt-matter-because-tasks-should-get-mocked.com" >> .env
+          echo "GCP_REGION=${{ needs.info.outputs.gcp_region }}" >> .env
+          echo "GCP_RESOURCE_AFFIX=${{ needs.info.outputs.gcp_resource_affix }}" >> .env
+          echo "GCP_ENVIRONMENT=${{ needs.info.outputs.branch_tag_kebab }}" >> .env
+
+      - name: Add additional envvars to .env file
+        if: ${{ inputs.env_vars != '' }}
+        run: |
+          split_env_vars=($(echo ${{ inputs.env_vars }} | tr "," "\n"))
+
+          for var in "${split_env_vars[@]}";
+          do
+            echo "$var" >> .env
+          done
+
+      - name: Export secrets to JSON
+        run: echo '${{ toJSON(steps.secrets.outputs) }}' >> secrets.json
+
+      - name: Remove GOOGLE_APPLICATION_CREDENTIALS value from secrets.json
+        run: jq 'del(.GOOGLE_APPLICATION_CREDENTIALS)' secrets.json > secrets2.json
+
+      - name: Add secrets to .env file
+        run: |
+          function append_json_to_env () {
+            INPUT_FILE="${1}"
+            while IFS=$'\t\n' read -r LINE; do
+                echo "${LINE}" >> .env
+            done < <(
+                <"${INPUT_FILE}" jq \
+                    --compact-output \
+                    --raw-output \
+                    --monochrome-output \
+                    --from-file \
+                    <(echo 'to_entries | map("\(.key)=\(.value)") | .[]')
+            )
+          }
+          
+          append_json_to_env secrets2.json
 
       - name: Configure Docker for GCP
         run: gcloud auth configure-docker ${{ needs.info.outputs.gcp_region }}-docker.pkg.dev
@@ -258,13 +296,13 @@ jobs:
           echo "docker run \
             --network="host" \
             --env-file .env \
-            --volume ${{ steps.authoverride.outputs.credentials_file_path }}:/workspace/${{ steps.authoverride.outputs.credentials_file_name }} \
+            --volume ${{ steps.store-gcp-credentials-in-file.outputs.credentials_file_path }}:/workspace/${{ steps.store-gcp-credentials-in-file.outputs.credentials_file_name }} \
             -it \
             --rm \
-            ${{ github.event.inputs.image_version_artifact }} \
+            ${{ needs.info.outputs.image_version_artifact }} \
             bash
           " >> start.sh
-          echo "Exec into the container by running `sh start.sh`."
+          echo "Exec into the container by running "sh start.sh"."
 
       - name: Setup tmate session [DEBUG]
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true'}}
@@ -307,7 +345,7 @@ jobs:
       #     env_vars: |
       #       ATOMIC_REQUESTS=0
       #       GCP_TASKS_RESOURCE_AFFIX=${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}
-      #       GCP_TASKS_DEFAULT_QUEUE_NAME=${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}-primary
+      #       GCP_TASKS_DEFAULT_QUEUE_NAME=${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}-default
       #       GCP_TASKS_DOMAIN=https://sha${{ needs.info.outputs.short_sha }}---${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}-worker-fhngvhbkyq-ew.a.run.app/
       #     image: ${{ needs.info.outputs.image_version_artifact }}
       #     region: ${{ needs.info.outputs.gcp_region }}
@@ -325,16 +363,16 @@ jobs:
           env_vars: |
             DJANGO_SETTINGS_MODULE=${{ steps.django-settings.outputs.django_settings_module }}
             GCP_TASKS_RESOURCE_AFFIX=${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}
-            GCP_TASKS_DEFAULT_QUEUE_NAME=${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}-primary
-            GCP_TASKS_DOMAIN=https://sha${{ needs.info.outputs.short_sha }}---${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}-${{ needs.info.outputs.server_name }}-${{ inputs.cloud_run_base_url }}
+            GCP_TASKS_DEFAULT_QUEUE_NAME=${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}-default
+            GCP_TASKS_DOMAIN=https://sha${{ needs.info.outputs.short_sha }}---${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}-${{ needs.info.outputs.server_name }}-${{ needs.info.outputs.gcp_project_number }}.${{ needs.info.outputs.gcp_region }}.run.app
           image: ${{ needs.info.outputs.image_version_artifact }}
           region: ${{ needs.info.outputs.gcp_region }}
           service: ${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.branch_tag_kebab }}-${{ needs.info.outputs.server_name }}
           tag: sha${{ needs.info.outputs.short_sha }}
           flags: ${{ inputs.cloud_run_flags }}
 
       # TODO unapply the migration if it or the deployment(s) failed
-      # - name: Show Cloud Run Deployment URLs
-      #   run: |
+      - name: Show Cloud Run Deployment URLs
+        run: |
+          echo "${{ steps.deploy-server.outputs.url }}"
       #     echo "${{ steps.deploy-worker.outputs.url }}"
-      #     echo "${{ steps.deploy-server.outputs.url }}"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,48 @@
+# This workflow releases a new version of workflows repository.
+
+name: Release
+
+# Only trigger when a pull request into main branch is merged.
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Branch or commit SHA to tag"
+        type: string
+        required: true
+      tag:
+        description: "Tag for release"
+        type: string
+        required: true
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Get package version
+        id: get-package-version
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        run: echo "package_version=$(cat VERSION.txt)" >> $GITHUB_OUTPUT
+
+      - name: Create Release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, no need to create your own.
+        with:
+          commitish: ${{ inputs.ref }}
+          tag_name: ${{ inputs.tag || steps.get-package-version.outputs.package_version }}
+          release_name: ${{ github.event.pull_request.title }}
+          body: ${{ github.event.pull_request.body }}
+          draft: false
+          prerelease: false
diff --git a/.github/workflows/shell-django-server.yml b/.github/workflows/shell-django-server.yml
@@ -49,7 +49,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install poetry
         uses: snok/install-poetry@v1
@@ -76,7 +76,7 @@ jobs:
     steps:
       - name: Checkout
         # Shouldn't be necessary since we pull code in an image, but it's required by google-github-actions/auth
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Authenticate with GCP Workload Identity
         id: auth

diff --git a/.github/workflows/update-pull-request.yml b/.github/workflows/update-pull-request.yml
@@ -0,0 +1,18 @@
+# This workflow updates the pull request description with an auto-generated section containing the categorised commit
+# message headers of the pull request's commits. The auto generated section is enveloped between two comments:
+# "<!--- START AUTOGENERATED NOTES --->" and "<!--- END AUTOGENERATED NOTES --->". Anything outside these in the
+# description is left untouched. Auto-generated updates can be skipped for a commit if
+# "<!--- SKIP AUTOGENERATED NOTES --->" is added to the pull request description.
+
+name: update-pull-request
+
+on: [pull_request]
+
+jobs:
+  description:
+    uses: octue/workflows/.github/workflows/generate-pull-request-description.yml@main
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+    permissions:
+      contents: read
+      pull-requests: write
diff --git a/VERSION.txt b/VERSION.txt
@@ -0,0 +1 @@
+0.1.0