fix: Fix API deployment issues on a small ECS task (#349)

packages/backend/Dockerfile

@@ -17,7 +17,7 @@ ENV PIP_NO_CACHE_DIR off
 RUN apt-get update && apt-get install -y gcc postgresql-client ca-certificates jq \
   && update-ca-certificates \
   && pip install --upgrade pip \
-  && pip install --no-cache-dir setuptools pdm~=2.5.2 gunicorn awscli
+  && pip install --no-cache-dir setuptools pdm~=2.5.2 awscli
 
 
 COPY --from=chamber /chamber /bin/chamber

packages/backend/Makefile

@@ -57,5 +57,5 @@ secrets:
 	$(MAKE) -C $(PROJECT_ROOT_DIR) secrets-editor SERVICE_NAME=backend
 
 remote-shell:
-	./scripts/execute_remote.sh
+	chamber exec $(ENV_STAGE) -- ./scripts/execute_remote.sh
 

packages/backend/gunicorn.py

@@ -2,8 +2,19 @@
 import logging
 from multiprocessing import cpu_count
 
+import environ
+
+env = environ.Env(
+    # set casting, default value
+    DJANGO_DEBUG=(bool, False)
+)
+
+DEBUG = env("DJANGO_DEBUG")
+
 
 def max_workers():
+    if DEBUG:
+        return 2
     return cpu_count() * 2 + 1
 
 

packages/backend/infra/stacks/api/stack.ts

@@ -1,4 +1,4 @@
-import { App, Fn, Stack, StackProps } from 'aws-cdk-lib';
+import {App, Duration, Fn, Stack, StackProps} from 'aws-cdk-lib';
 import * as ecs from 'aws-cdk-lib/aws-ecs';
 import * as iam from 'aws-cdk-lib/aws-iam';
 import * as sm from 'aws-cdk-lib/aws-secretsmanager';
@@ -94,14 +94,20 @@ export class ApiStack extends Stack {
       {
         securityGroup: resources.fargateContainerSecurityGroup,
         serviceName: getApiServiceName(props.envSettings),
+        healthCheckGracePeriod: Duration.minutes(2),
         cluster: resources.mainCluster,
         cpu: 512,
-        memoryLimitMiB: 2048,
+        memoryLimitMiB: 1024,
         desiredCount: 1,
         taskRole,
         taskImageOptions: [
           {
             containerName: 'backend',
+            command:  [
+              "sh",
+              "-c",
+              "/bin/chamber exec $CHAMBER_SERVICE_NAME -- ./scripts/run.sh",
+            ],
             image: ecs.ContainerImage.fromEcrRepository(
               resources.backendRepository,
               envSettings.version

packages/backend/project.json

@@ -50,30 +50,38 @@
       },
       "dependsOn": ["compose-build-image"]
     },
+    "deploy:api": {
+      "executor": "nx:run-commands",
+      "options": {
+        "cwd": "packages/backend",
+        "color": true,
+        "commands": [
+          "pnpm nx cdk:deploy:api",
+          "pnpm nx run tools:upload-service-version api \"url=https://${SB_DOMAIN_API}\""
+        ],
+        "parallel": false
+      }
+    },
+    "deploy:migrations": {
+      "executor": "nx:run-commands",
+      "options": {
+        "cwd": "packages/backend",
+        "color": true,
+        "commands": [
+          "pnpm nx cdk:deploy:migrations",
+          "pnpm nx run trigger-migrations-job",
+          "pnpm nx run tools:upload-service-version migrations"
+        ],
+        "parallel": false
+      }
+    },
     "deploy": {
       "executor": "nx:run-commands",
       "options": {
         "cwd": "packages/backend",
         "color": true,
         "commands": ["nx cdk:deploy:api", "nx cdk:deploy:migrations"],
         "parallel": false
-      },
-      "configurations": {
-        "api": {
-          "commands": [
-            "pnpm nx cdk:deploy:api",
-            "pnpm nx run tools:upload-service-version api \"url=https://${SB_DOMAIN_API}\""
-          ],
-          "parallel": false
-        },
-        "migrations": {
-          "commands": [
-            "pnpm nx cdk:deploy:migrations",
-            "pnpm nx run trigger-migrations-job",
-            "pnpm nx run tools:upload-service-version migrations"
-          ],
-          "parallel": false
-        }
       }
     }
   },

packages/backend/pyproject.toml

@@ -113,7 +113,8 @@ dependencies = [
     "pyotp>=2.8.0",
     "openai>=0.27.2",
     "pydantic>=1.10.7",
-    "pyyaml>=6.0.0"
+    "pyyaml>=6.0.0",
+    "gunicorn==21.2.0",
 ]
 requires-python = "~=3.11.0"
 license = {text = "MIT"}

packages/backend/scripts/execute_remote.sh

@@ -13,20 +13,6 @@ then
    exit 1
 fi
 
-ENV_FILE="../../.env"
-ENV_STAGE_FILE="${ENV_FILE}.${ENV_STAGE}"
-
-if [ ! -f "$ENV_FILE" ]; then
-  echo "$ENV_FILE cannot be found"
-  exit 1
-elif [ ! -f "$ENV_STAGE_FILE" ]; then
-  echo "$ENV_STAGE_FILE cannot be found"
-  exit 1
-else
-  export $(echo $(cat "$ENV_FILE" | sed 's/#.*//g'| xargs) | envsubst)
-  export $(echo $(cat "$ENV_STAGE_FILE" | sed 's/#.*//g'| xargs) | envsubst)
-fi
-
 PROJECT_ENV_NAME=${PROJECT_NAME}-${ENV_STAGE}
 
 CLUSTER_NAME="${PROJECT_ENV_NAME}-main"
@@ -41,4 +27,4 @@ aws ecs execute-command \
   --region "${AWS_DEFAULT_REGION//\"/}" \
   --task "$TASK_ARN" \
   --container backend \
-  --command "/bin/chamber exec ${CHAMBER_SERVICE_NAME} -- /bin/bash" --interactive
+  --command "/bin/bash" --interactive

packages/backend/scripts/run.sh

@@ -3,4 +3,4 @@ set -e
 
 echo Starting app server...
 
-gunicorn -c gunicorn.py config.wsgi:application
+pdm run gunicorn -c gunicorn.py config.wsgi:application

packages/infra/infra-core/src/lib/patterns/applicationMultipleTargetGroupsFargateService.ts

@@ -3,18 +3,18 @@ import {
   FargateService,
   FargateTaskDefinition,
   AwsLogDriver,
-} from "aws-cdk-lib/aws-ecs";
-import { ApplicationTargetGroup } from "aws-cdk-lib/aws-elasticloadbalancingv2";
-import { FeatureFlags } from "aws-cdk-lib";
-import * as cxapi from "aws-cdk-lib/cx-api";
-import { Construct } from "constructs";
+} from 'aws-cdk-lib/aws-ecs';
+import { ApplicationTargetGroup } from 'aws-cdk-lib/aws-elasticloadbalancingv2';
+import { FeatureFlags } from 'aws-cdk-lib';
+import * as cxapi from 'aws-cdk-lib/cx-api';
+import { Construct } from 'constructs';
 import {
   ApplicationMultipleTargetGroupsServiceBase,
   ApplicationMultipleTargetGroupsServiceBaseProps,
-} from "./applicationMultipleTargetGroupsFargateServiceBase";
-import { ISecurityGroup, SubnetType } from "aws-cdk-lib/aws-ec2";
-import { IRole } from "aws-cdk-lib/aws-iam";
-import { ILogGroup, LogGroup } from "aws-cdk-lib/aws-logs";
+} from './applicationMultipleTargetGroupsFargateServiceBase';
+import { ISecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2';
+import { IRole } from 'aws-cdk-lib/aws-iam';
+import { ILogGroup, LogGroup } from 'aws-cdk-lib/aws-logs';
 
 /**
  * The properties for the ApplicationMultipleTargetGroupsFargateService service.
@@ -158,13 +158,13 @@ export class ApplicationMultipleTargetGroupsFargateService extends ApplicationMu
 
     if (props.taskDefinition && props.taskImageOptions) {
       throw new Error(
-        "You must specify only one of TaskDefinition or TaskImageOptions."
+        'You must specify only one of TaskDefinition or TaskImageOptions.'
       );
     } else if (props.taskDefinition) {
       this.taskDefinition = props.taskDefinition;
     } else if (props.taskImageOptions) {
       const taskImageOptions = props.taskImageOptions;
-      this.taskDefinition = new FargateTaskDefinition(this, "TaskDef", {
+      this.taskDefinition = new FargateTaskDefinition(this, 'TaskDef', {
         memoryLimitMiB: props.memoryLimitMiB,
         cpu: props.cpu,
         executionRole: props.executionRole,
@@ -173,22 +173,18 @@ export class ApplicationMultipleTargetGroupsFargateService extends ApplicationMu
       });
 
       for (const taskImageOptionsProps of taskImageOptions) {
-        const containerName = taskImageOptionsProps.containerName ?? "web";
+        const containerName = taskImageOptionsProps.containerName ?? 'web';
         const container = this.taskDefinition.addContainer(containerName, {
           image: taskImageOptionsProps.image,
           logging:
             taskImageOptionsProps.enableLogging === false
               ? undefined
               : taskImageOptionsProps.logDriver ||
-                this.createAWSLogDriver(this.node.id),
+                this.createAWSLogDriver(`${this.node.id}-${containerName}`),
           environment: taskImageOptionsProps.environment,
           secrets: taskImageOptionsProps.secrets,
           dockerLabels: taskImageOptionsProps.dockerLabels,
-          command: [
-            "sh",
-            "-c",
-            "/bin/chamber exec $CHAMBER_SERVICE_NAME -- ./scripts/run.sh",
-          ],
+          command: taskImageOptionsProps.command,
         });
         if (taskImageOptionsProps.containerPorts) {
           for (const containerPort of taskImageOptionsProps.containerPorts) {
@@ -199,10 +195,10 @@ export class ApplicationMultipleTargetGroupsFargateService extends ApplicationMu
         }
       }
     } else {
-      throw new Error("You must specify one of: taskDefinition or image");
+      throw new Error('You must specify one of: taskDefinition or image');
     }
     if (!this.taskDefinition.defaultContainer) {
-      throw new Error("At least one essential container must be specified");
+      throw new Error('At least one essential container must be specified');
     }
     if (this.taskDefinition.defaultContainer.portMappings.length === 0) {
       this.taskDefinition.defaultContainer.addPortMappings({
@@ -225,7 +221,7 @@ export class ApplicationMultipleTargetGroupsFargateService extends ApplicationMu
   }
 
   protected createAWSLogDriver(prefix: string): AwsLogDriver {
-    const logGroup = new LogGroup(this, "LogGroup");
+    const logGroup = new LogGroup(this, `${prefix}-LogGroup`);
     this.logGroups.push(logGroup);
     return new AwsLogDriver({
       streamPrefix: prefix,
@@ -242,7 +238,7 @@ export class ApplicationMultipleTargetGroupsFargateService extends ApplicationMu
       ? this.internalDesiredCount
       : this.desiredCount;
 
-    return new FargateService(this, "Service", {
+    return new FargateService(this, 'Service', {
       cluster: this.cluster,
       desiredCount: desiredCount,
       taskDefinition: this.taskDefinition,

packages/infra/infra-core/src/lib/patterns/applicationMultipleTargetGroupsFargateServiceBase.ts

@@ -192,6 +192,8 @@ export interface ApplicationLoadBalancedTaskImageProps {
    * @default - No labels.
    */
   readonly dockerLabels?: { [key: string]: string };
+
+  readonly command?: string[];
 }
 
 /**