Deploy Delay Notifications #23219
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Delay Notifications | |
| on: | |
| schedule: | |
| - cron: "*/10 * * * *" # Runs every 10 minutes | |
| jobs: | |
| check-deployment: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| dev_summary: ${{ steps.check-dev-status.outputs.dev_summary }} | |
| staging_summary: ${{ steps.check-staging-status.outputs.staging_summary }} | |
| steps: | |
| - name: Get latest commit SHA and time from master branch | |
| id: git-info | |
| run: | | |
| latest_commit_info=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/repos/${{ github.repository }}/commits/master") | |
| latest_sha=$(echo "${latest_commit_info}" | jq -r '.sha') | |
| echo "latest_sha=${latest_sha}" >> $GITHUB_ENV | |
| echo "latest_sha: ${latest_sha}" | |
| commit_time=$(echo "${latest_commit_info}" | jq -r '.commit.committer.date') | |
| echo "commit_time=${commit_time}" >> $GITHUB_ENV | |
| echo "commit_time: ${commit_time}" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Get deployed SHA for development | |
| id: dev-deploy-sha | |
| run: | | |
| deployed_sha=$(curl -s https://dev-api.va.gov/v0/status | jq -r .git_revision) | |
| echo "dev_deployed_sha=${deployed_sha}" >> $GITHUB_ENV | |
| - name: Get deployed SHA for staging | |
| id: staging-deploy-sha | |
| run: | | |
| deployed_sha=$(curl -s https://staging-api.va.gov/v0/status | jq -r .git_revision) | |
| echo "staging_deployed_sha=${deployed_sha}" >> $GITHUB_ENV | |
| - name: Check deployment status for development | |
| if: ${{ env.latest_sha != '' && env.dev_deployed_sha != '' }} | |
| id: check-dev-status | |
| run: | | |
| latest_sha=${{ env.latest_sha }} | |
| commit_time=${{ env.commit_time }} | |
| deployed_sha=${{ env.dev_deployed_sha }} | |
| info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to development" | |
| action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-dev|ArgoCD dev>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>" | |
| if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then | |
| echo "${info_message} has been deployed." | |
| echo "dev_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT | |
| elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '45 minutes ago' +%s)" ]; then | |
| echo "${info_message} has been delayed for more than 45 minutes. Skipping notification." | |
| echo "dev_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT | |
| elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '30 minutes ago' +%s)" ]; then | |
| echo "${info_message} has been delayed for more than 30 minutes." | |
| echo "Current commit on development is ${deployed_sha:0:8}." | |
| echo "dev_summary=${info_message} has been delayed for more than 30 minutes. Current commit on development is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT | |
| exit 1 | |
| else | |
| echo "Awaiting deployment of ${info_message}." | |
| echo "dev_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT | |
| fi | |
| - name: Check deployment status for staging | |
| if: ${{ always() && env.latest_sha != '' && env.staging_deployed_sha != '' }} | |
| id: check-staging-status | |
| run: | | |
| latest_sha=${{ env.latest_sha }} | |
| commit_time=${{ env.commit_time }} | |
| deployed_sha=${{ env.staging_deployed_sha }} | |
| info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to staging" | |
| action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-staging|ArgoCD staging>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>" | |
| if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then | |
| echo "${info_message} has been deployed." | |
| echo "staging_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT | |
| elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '45 minutes ago' +%s)" ]; then | |
| echo "${info_message} has been delayed for more than 45 minutes. Skipping notification." | |
| echo "staging_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT | |
| elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '30 minutes ago' +%s)" ]; then | |
| echo "${info_message} has been delayed for more than 30 minutes." | |
| echo "Current commit on staging is ${deployed_sha:0:8}." | |
| echo "staging_summary=${info_message} has been delayed for more than 30 minutes. Current commit on staging is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT | |
| exit 1 | |
| else | |
| echo "Awaiting deployment of ${info_message}." | |
| echo "staging_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT | |
| fi | |
| notify-on-failure: | |
| runs-on: ubuntu-latest | |
| needs: [check-deployment] | |
| env: | |
| dev_summary: ${{ needs.check-deployment.outputs.dev_summary }} | |
| staging_summary: ${{ needs.check-deployment.outputs.staging_summary }} | |
| if: ${{ failure() }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: "us-gov-west-1" | |
| - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb | |
| with: | |
| ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN | |
| env_variable_name: VA_VSP_BOT_GITHUB_TOKEN | |
| - name: Checkout VSP actions | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: department-of-veterans-affairs/vsp-github-actions | |
| ref: refs/heads/main | |
| token: ${{ env.VA_VSP_BOT_GITHUB_TOKEN }} | |
| persist-credentials: false | |
| path: ./.github/actions/vsp-github-actions | |
| - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb | |
| with: | |
| ssm_parameter: /devops/github_actions_slack_socket_token | |
| env_variable_name: SLACK_APP_TOKEN | |
| - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb | |
| with: | |
| ssm_parameter: /devops/github_actions_slack_bot_user_token | |
| env_variable_name: SLACK_BOT_TOKEN | |
| - name: Notify for deployment failure | |
| if: ${{ env.dev_summary != '' || env.staging_summary != '' }} | |
| uses: ./.github/actions/vsp-github-actions/slack-socket | |
| with: | |
| slack_app_token: ${{ env.SLACK_APP_TOKEN }} | |
| slack_bot_token: ${{ env.SLACK_BOT_TOKEN }} | |
| message: "Vets API Deployment Delay:" | |
| blocks: | | |
| [ | |
| { "type": "divider" }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>\n\n *Development Summary:*\n${{ env.dev_summary }}\n\n *Staging Summary:*\n${{ env.staging_summary }}" | |
| } | |
| }, | |
| { "type": "divider" } | |
| ] | |
| channel_id: "C039HRTHXDH" | |
| - name: Notify for other failure | |
| if: ${{ env.dev_summary == '' && env.staging_summary == '' }} | |
| uses: ./.github/actions/vsp-github-actions/slack-socket | |
| with: | |
| slack_app_token: ${{ env.SLACK_APP_TOKEN }} | |
| slack_bot_token: ${{ env.SLACK_BOT_TOKEN }} | |
| message: "Vets API Deployment Delay:" | |
| blocks: | | |
| [ | |
| { "type": "divider" }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n\n Unknown error occured. See logs:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>" | |
| } | |
| }, | |
| { "type": "divider" } | |
| ] | |
| channel_id: "C039HRTHXDH" |