Deploy: backend on lambda and frontend on ec2-ecs

canopas · Sep 18, 2023 · db42a7e · db42a7e
1 parent b387090
commit db42a7e
Show file tree

Hide file tree

Showing 20 changed files with 604 additions and 327 deletions.
diff --git a/.github/workflows/deploy-backend-dev.yml b/.github/workflows/deploy-backend-dev.yml
@@ -0,0 +1,70 @@
+name: DeployBackendDev
+
+on:
+  push:
+    branches:
+      - "master"
+
+jobs:
+  deploy-backend-dev:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "^1.20"
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-backend-access
+          aws-region: ap-south-1
+
+      - name: Build backend and deploy to S3
+        run: |
+          apt-get update && apt-get install -y zip
+          GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o main main.go
+          zip canopas_serverless_dev_${{ github.sha }}.zip main
+          aws s3 cp canopas_serverless_dev_${{ github.sha }}.zip s3://canopas-lambda-handlers
+
+      - name: Deploy cloudformation stack
+        id: canopas-website-dev-lambda-stack
+        uses: aws-actions/aws-cloudformation-github-deploy@v1
+        with:
+          name: canopas-website-dev-lambda-stack
+          template: infrastructure/backend.yml
+          capabilities: CAPABILITY_IAM,CAPABILITY_NAMED_IAM
+          timeout-in-minutes: "10"
+          no-fail-on-empty-changeset: "1"
+          parameter-overrides: >-
+            EnvName=dev,
+            Sender=${{ secrets.SENDER }},
+            ContactSender=${{ secrets.CONTACT_SENDER }},
+            JobsReceiver=${{ secrets.JOBS_RECEIVER }},
+            ContactReceiver=${{ secrets.CONTACT_RECEIVER }},
+            AccessKeyId=${{ secrets.ACCESS_KEY_ID }},
+            SecretAccessKey=${{ secrets.SECRET_ACCESS_KEY }},
+            Region=${{ secrets.AWS_REGION }},
+            DbUsername=${{ secrets.DB_USERNAME }},
+            DbPassword=${{ secrets.DB_PASSWORD }},
+            DbHost=${{ secrets.DB_HOST }},
+            DbPort=${{ secrets.DB_PORT }},
+            DbName=${{ secrets.DB_NAME }},
+            RecaptchaConfigJSONBase64=${{ secrets.RECAPTCHA_CONFIG_JSON_BASE64 }},
+            RecaptchaSiteKey=${{ secrets.RECAPTCHA_SITE_KEY }},
+            RecaptchaProjectId=${{ secrets.RECAPTCHA_PROJECT_ID }},
+            JobsSpreadsheetId=${{ secrets.JOBS_SPREADSHEET_ID }},
+            GithubAccessToken=${{ secrets.PERSONAL_ACCESS_TOKEN_GITHUB }},
+            ResourcesURL=${{secrets.RESOURCES_URL}},
+            LambdaBucket=canopas-lambda-handlers, 
+            LambdaUrl=canopas_serverless_dev_${{ github.sha }}.zip,
+            LambdaTimeout=10, 
+            LambdaRoleName=canopas-dev-lambda-role,
+            LambdaName=canopas-dev-lambda-function, 
+            ApiGatewayName=canopas-dev-lambda-api,
+            CustomDomainName=${{ secrets.DEV_SERVERLESS_DOMAIN }}
diff --git a/.github/workflows/deploy-backend-prod.yml b/.github/workflows/deploy-backend-prod.yml
@@ -0,0 +1,70 @@
+name: DeployBackendProd
+
+on:
+  push:
+    branches:
+      - "master"
+
+jobs:
+  deploy-backend-prod:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "^1.20"
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-backend-access
+          aws-region: ap-south-1
+
+      - name: Build backend and deploy to S3
+        run: |
+          apt-get update && apt-get install -y zip
+          GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o main main.go
+          zip canopas_serverless_prod_${{ github.sha }}.zip main
+          aws s3 cp canopas_serverless_prod_${{ github.sha }}.zip s3://canopas-lambda-handlers
+
+      - name: Deploy cloudformation stack
+        id: canopas-website-prod-lambda-stack
+        uses: aws-actions/aws-cloudformation-github-deploy@v1
+        with:
+          name: canopas-website-prod-lambda-stack
+          template: infrastructure/backend.yml
+          capabilities: CAPABILITY_IAM,CAPABILITY_NAMED_IAM
+          timeout-in-minutes: "10"
+          no-fail-on-empty-changeset: "1"
+          parameter-overrides: >-
+            EnvName=prod,
+            Sender=${{ secrets.SENDER }},
+            ContactSender=${{ secrets.CONTACT_SENDER }},
+            JobsReceiver=${{ secrets.JOBS_RECEIVER }},
+            ContactReceiver=${{ secrets.CONTACT_RECEIVER }},
+            AccessKeyId=${{ secrets.ACCESS_KEY_ID }},
+            SecretAccessKey=${{ secrets.SECRET_ACCESS_KEY }},
+            Region=${{ secrets.AWS_REGION }},
+            DbUsername=${{ secrets.DB_USERNAME }},
+            DbPassword=${{ secrets.DB_PASSWORD }},
+            DbHost=${{ secrets.DB_HOST }},
+            DbPort=${{ secrets.DB_PORT }},
+            DbName=${{ secrets.DB_NAME }},
+            RecaptchaConfigJSONBase64=${{ secrets.RECAPTCHA_CONFIG_JSON_BASE64 }},
+            RecaptchaSiteKey=${{ secrets.RECAPTCHA_SITE_KEY }},
+            RecaptchaProjectId=${{ secrets.RECAPTCHA_PROJECT_ID }},
+            JobsSpreadsheetId=${{ secrets.JOBS_SPREADSHEET_ID }},
+            GithubAccessToken=${{ secrets.PERSONAL_ACCESS_TOKEN_GITHUB }},
+            ResourcesURL=${{secrets.RESOURCES_URL}},
+            LambdaBucket=canopas-lambda-handlers, 
+            LambdaUrl=canopas_serverless_prod_${{ github.sha }}.zip,
+            LambdaTimeout=10, 
+            LambdaRoleName=canopas-prod-lambda-role,
+            LambdaName=canopas-prod-lambda-function, 
+            ApiGatewayName=canopas-prod-lambda-api,
+            CustomDomainName=${{ secrets.PROD_SERVERLESS_DOMAIN }}
diff --git a/.github/workflows/deploy-dev.yml → .github/workflows/deploy-frontend-dev.yml b/.github/workflows/deploy-dev.yml → .github/workflows/deploy-frontend-dev.yml
@@ -1,12 +1,14 @@
-name: DeployDev
+name: DeployFrontendDev
 
 on:
+  repository_dispatch:
+    types: [deploy-blog-dev]
   push:
     branches:
       - "master"
 
 jobs:
-  deploy-dev:
+  deploy-frontend-dev:
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -17,7 +19,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: "18"
+          node-version: "20"
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -30,7 +32,6 @@ jobs:
           cd nginx 
           sed -i "s|WEBSITE_URL|dev-stack.canopas.com|g" conf.d/default.conf
           sed -i "s|API_URL|dev-stack-api.canopas.com|g" conf.d/default.conf
-          sed -i "s|ARTICLES_URL|development.dzkjozipsd9ra.amplifyapp.com|g" conf.d/default.conf
           bash ./../deploy/deploy-ecr-image.sh dev-nginx ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/canopas-website-nginx
 
       - name: Build frontend and push on ECR
@@ -40,48 +41,24 @@ jobs:
           sh ./../deploy/generate-sitemap.sh https://dev-stack.canopas.com https://dev-stack-api.canopas.com
           bash ./../deploy/deploy-ecr-image.sh dev-frontend ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/canopas-website-frontend
 
-      - name: Checkout
-        uses: actions/[email protected]
-
-      - uses: actions/setup-go@v2
-        with:
-          go-version: "^1.17.5"
-
-      - name: Build backend and push on ECR
+      - name: Get latest blog image tag
+        id: get_blog_image_tag
         run: |
-          bash ./deploy/deploy-ecr-image.sh dev-backend ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/canopas-website-backend
+          aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.ap-south-1.amazonaws.com
+          echo "::set-output name=blogImageTag::$(aws ecr describe-images --repository-name canopas-blog --output text --query 'sort_by(imageDetails,& imagePushedAt)[*].imageTags[*]' | grep -w 'dev' | tr '\t' '\n' | tail -1)"
 
       - name: Deploy cloudformation stack
         id: canopas-website-dev-ECS-EC2-stack
         uses: aws-actions/aws-cloudformation-github-deploy@v1
         with:
           name: canopas-website-dev-ECS-EC2-stack
-          template: infrastructure/template.yml
+          template: infrastructure/frontend.yml
           capabilities: CAPABILITY_IAM,CAPABILITY_NAMED_IAM
           timeout-in-minutes: "10"
           no-fail-on-empty-changeset: "1"
           parameter-overrides: >-
             EnvName=dev,
-            Sender=${{ secrets.SENDER }},
-            ContactSender=${{ secrets.CONTACT_SENDER }},
-            JobsReceiver=${{ secrets.JOBS_RECEIVER }},
-            ContactReceiver=${{ secrets.CONTACT_RECEIVER }},
-            AccessKeyId=${{ secrets.ACCESS_KEY_ID }},
-            SecretAccessKey=${{ secrets.SECRET_ACCESS_KEY }},
-            Region=${{ secrets.AWS_REGION }},
-            DbUsername=${{ secrets.DB_USERNAME }},
-            DbPassword=${{ secrets.DB_PASSWORD }},
-            DbHost=${{ secrets.DB_HOST }},
-            DbPort=${{ secrets.DB_PORT }},
-            DbName=${{ secrets.DB_NAME }},
-            RecaptchaConfigJSONBase64=${{ secrets.RECAPTCHA_CONFIG_JSON_BASE64 }},
-            RecaptchaSiteKey=${{ secrets.RECAPTCHA_SITE_KEY }},
-            RecaptchaProjectId=${{ secrets.RECAPTCHA_PROJECT_ID }},
-            JobsSpreadsheetId=${{ secrets.JOBS_SPREADSHEET_ID }},
-            JobsSheetId=${{ secrets.JOBS_SHEET_ID }},
-            GithubAccessToken=${{ secrets.PERSONAL_ACCESS_TOKEN_GITHUB }},
-            ResourcesURL=${{secrets.RESOURCES_URL}},
             ClusterName=canopas-website-dev,
             ImageTag=${{ github.sha }}-${{ github.run_attempt }}-dev-frontend,
-            BackendImageTag=${{ github.sha }}-${{ github.run_attempt }}-dev-backend,
-            NginxImageTag=${{ github.sha }}-${{ github.run_attempt }}-dev-nginx
+            NginxImageTag=${{ github.sha }}-${{ github.run_attempt }}-dev-nginx,
+            BlogImageTag=${{steps.get_blog_image_tag.outputs.blogImageTag}}
diff --git a/.github/workflows/deploy-prod.yml → .github/workflows/deploy-frontend-prod.yml b/.github/workflows/deploy-prod.yml → .github/workflows/deploy-frontend-prod.yml
@@ -1,11 +1,14 @@
-name: DeployProd
+name: DeployFrontendProd
+
 on:
+  repository_dispatch:
+    types: [deploy-blog-prod]
   push:
     branches:
       - "master"
-      
+
 jobs:
-  deploy-prod:
+  deploy-frontend-prod:
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -16,7 +19,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: "18"
+          node-version: "20"
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -29,7 +32,6 @@ jobs:
           cd nginx 
           sed -i "s|WEBSITE_URL|canopas.com|g" conf.d/default.conf
           sed -i "s|API_URL|prod-stack-api.canopas.com|g" conf.d/default.conf
-          sed -i "s|ARTICLES_URL|articles.canopas.com|g" conf.d/default.conf
           bash ./../deploy/deploy-ecr-image.sh prod-nginx ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/canopas-website-nginx
 
       - name: Build frontend and push on ECR
@@ -39,49 +41,25 @@ jobs:
           mv src/config.prod.js src/config.js
           sh ./../deploy/generate-sitemap.sh https://canopas.com https://prod-stack-api.canopas.com
           bash ./../deploy/deploy-ecr-image.sh prod-frontend ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/canopas-website-frontend
-  
-      - name: Checkout
-        uses: actions/[email protected]
-
-      - uses: actions/setup-go@v2
-        with:
-          go-version: "^1.17.5"
 
-      - name: Build backend and push on ECR
+      - name: Get latest blog image tag
+        id: get_blog_image_tag
         run: |
-          bash ./deploy/deploy-ecr-image.sh prod-backend ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.$AWS_REGION.amazonaws.com/canopas-website-backend
+          aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.ap-south-1.amazonaws.com
+          echo "::set-output name=blogImageTag::$(aws ecr describe-images --repository-name canopas-blog --output text --query 'sort_by(imageDetails,& imagePushedAt)[*].imageTags[*]' | grep -w 'prod' | tr '\t' '\n' | tail -1)"
 
       - name: Deploy cloudformation stack
         id: canopas-website-prod-ECS-EC2-stack
         uses: aws-actions/aws-cloudformation-github-deploy@v1
         with:
           name: canopas-website-prod-ECS-EC2-stack
-          template: infrastructure/template.yml
+          template: infrastructure/frontend.yml
           capabilities: CAPABILITY_IAM,CAPABILITY_NAMED_IAM
           timeout-in-minutes: "10"
           no-fail-on-empty-changeset: "1"
           parameter-overrides: >-
             EnvName=prod,
-            Sender=${{ secrets.SENDER }},
-            ContactSender=${{ secrets.CONTACT_SENDER }},
-            JobsReceiver=${{ secrets.JOBS_RECEIVER }},
-            ContactReceiver=${{ secrets.CONTACT_RECEIVER }},
-            AccessKeyId=${{ secrets.ACCESS_KEY_ID }},
-            SecretAccessKey=${{ secrets.SECRET_ACCESS_KEY }},
-            Region=${{ secrets.AWS_REGION }},
-            DbUsername=${{ secrets.DB_USERNAME }},
-            DbPassword=${{ secrets.DB_PASSWORD }},
-            DbHost=${{ secrets.DB_HOST }},
-            DbPort=${{ secrets.DB_PORT }},
-            DbName=${{ secrets.DB_NAME }},
-            RecaptchaConfigJSONBase64=${{ secrets.RECAPTCHA_CONFIG_JSON_BASE64 }},
-            RecaptchaSiteKey=${{ secrets.RECAPTCHA_SITE_KEY }},
-            RecaptchaProjectId=${{ secrets.RECAPTCHA_PROJECT_ID }},
-            JobsSpreadsheetId=${{ secrets.JOBS_SPREADSHEET_ID }},
-            JobsSheetId=${{ secrets.JOBS_SHEET_ID }},
-            GithubAccessToken=${{ secrets.PERSONAL_ACCESS_TOKEN_GITHUB }},
-            ResourcesURL=${{secrets.RESOURCES_URL}},
             ClusterName=canopas-website-prod,
             ImageTag=${{ github.sha }}-${{ github.run_attempt }}-prod-frontend,
-            BackendImageTag=${{ github.sha }}-${{ github.run_attempt }}-prod-backend,
-            NginxImageTag=${{ github.sha }}-${{ github.run_attempt }}-prod-nginx
+            NginxImageTag=${{ github.sha }}-${{ github.run_attempt }}-prod-nginx,
+            BlogImageTag=${{steps.get_blog_image_tag.outputs.blogImageTag}}
diff --git a/blogs/blog_test.go b/blogs/blog_test.go
@@ -7,7 +7,7 @@ import (
 	"utils"
 
 	"github.com/gin-gonic/gin"
-	"github.com/tj/assert"
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_Unique_Int_Success(t *testing.T) {

diff --git a/blogs/go.mod b/blogs/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/aws/aws-sdk-go v1.44.83
 	github.com/gin-gonic/gin v1.9.0
 	github.com/sirupsen/logrus v1.9.0
-	github.com/tj/assert v0.0.3
+	github.com/stretchr/testify v1.8.4
 	utils v0.0.0-00010101000000-000000000000
 )
 
@@ -44,7 +44,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/stretchr/testify v1.8.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.9 // indirect
 	go.opencensus.io v0.23.0 // indirect

diff --git a/blogs/go.sum b/blogs/go.sum
@@ -263,8 +263,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
-github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU=
@@ -734,7 +734,6 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

diff --git a/deploy/deploy-ecr-image.sh b/deploy/deploy-ecr-image.sh
@@ -13,11 +13,3 @@ docker build -t canopas-website-ssr-app:$IMAGE_TAG-$PLATFORM .
 docker tag canopas-website-ssr-app:$IMAGE_TAG-$PLATFORM $IMAGE_ARN:$IMAGE_TAG-$PLATFORM
 
 docker push $IMAGE_ARN:$IMAGE_TAG-$PLATFORM
-
-# delete untagged images
-aws ecr describe-repositories --output text | awk '{print $5}' | egrep -v '^$' | while read line; do
-    repo=$(echo $line | sed -e "s/arn:aws:ecr.*\///g")
-    aws ecr list-images --repository-name $repo --filter tagStatus=UNTAGGED --query 'imageIds[*]' --output text | while read imageId; do
-        aws ecr batch-delete-image --repository-name $repo --image-ids imageDigest=$imageId
-    done
-done