Configure ECS syslog

buildspecs/dalmatian-logspout.yml

@@ -0,0 +1,24 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - echo "Build started on $(date)"
+      - echo "Logging in to Amazon ECR..."
+      - aws ecr get-login-password --region "$AWS_DEFAULT_REGION" | docker login --username AWS --password-stdin "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com"
+      - |
+        if [ -n "$DOCKERHUB_USERNAME" ] && [ -n "DOCKERHUB_TOKEN" ];
+        then
+          echo "Logging into Dockerhub ...";
+          echo "$DOCKERHUB_TOKEN" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin;
+        fi;
+      - echo "Pulling Logspout docker image ..."
+      - docker pull gliderlabs/logspout:v3.2.14
+  build:
+    commands:
+      - echo "Adding ECR repo tag..."
+      - docker tag gliderlabs/logspout:v3.2.14 "$REPOSITORY_URI:latest"
+  post_build:
+    commands:
+      - echo "Pushing the Docker image..."
+      - docker push "$REPOSITORY_URI:latest"

container-definitions/app.json.tpl

@@ -1,7 +1,5 @@
 [
   {
-    "essential": true,
-    "memoryReservation": 16,
     "image": "${image}",
     "name": "${container_name}",
     %{ if cloudwatch_log_group != "" }
@@ -52,6 +50,13 @@
     %{ if linux_parameters != "{}" }
     "linuxParameters": ${linux_parameters},
     %{ endif }
-    "entrypoint": ${entrypoint}
+    %{if entrypoint != "[]"}
+    "entrypoint": ${entrypoint},
+    %{ endif }
+    %{if command != "[]"}
+    "command": ${command},
+    %{ endif }
+    "memoryReservation": 16,
+    "essential": true
   }
 ]

ec2-userdata/ecs-instance.tpl

@@ -34,6 +34,25 @@ fi
 sudo yum install -y \
   jq \
   rsync
+%{~ if syslog_endpoint != "" }
+# Configure Syslog
+sudo yum install -y \
+  rsyslog-gnutls
+
+{
+  echo "\$DefaultNetstreamDriverCAFile /etc/ssl/certs/ca-bundle.crt"
+  echo "\$ActionSendStreamDriver gtls # use gtls netstream driver"
+  echo "\$ActionSendStreamDriverMode 1 # require TLS"
+  echo "\$ActionSendStreamDriverAuthMode x509/name # authenticate by hostname"
+%{~ if syslog_permitted_peer != ""}
+  echo "\$ActionSendStreamDriverPermittedPeer ${syslog_permitted_peer}"
+%{~ endif }
+  echo "*.*     @@${syslog_endpoint}"
+} > /etc/rsyslog.d/syslog-remote.conf
+
+service rsyslog restart
+%{ endif }
+
 %{~ if efs_id != ""}
 # EFS
 sudo mkdir -p /mnt/efs

ecs-cluster-infrastructure-logspout-ecr.tf

@@ -0,0 +1,17 @@
+resource "aws_ecr_repository" "infrastructure_ecs_cluster_logspout" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  name = "${local.resource_prefix}-ecs-cluster-logspout"
+
+  #tfsec:ignore:aws-ecr-enforce-immutable-repository
+  image_tag_mutability = "MUTABLE"
+
+  encryption_configuration {
+    encryption_type = local.infrastructure_kms_encryption ? "KMS" : "AES256"
+    kms_key         = local.infrastructure_kms_encryption ? aws_kms_key.infrastructure[0].arn : null
+  }
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+}

ecs-cluster-infrastructure-logspout-image-codebuild.tf

@@ -0,0 +1,146 @@
+resource "aws_iam_role" "infrastructure_ecs_cluster_logspout_image_codebuild" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  name        = "${local.resource_prefix}-${substr(sha512("ecs-cluster-logspout-image-codebuild"), 0, 6)}"
+  description = "${local.resource_prefix}-ecs-cluster-logspout-image-codebuild"
+  assume_role_policy = templatefile(
+    "${path.root}/policies/assume-roles/service-principle-standard.json.tpl",
+    { services = jsonencode(["codebuild.amazonaws.com", "events.amazonaws.com"]) }
+  )
+}
+
+resource "aws_iam_policy" "infrastructure_ecs_cluster_logspout_image_codebuild_cloudwatch_rw" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  name        = "${local.resource_prefix}-${substr(sha512("ecs-cluster-logspout-image-codebuild-cloudwatch-rw"), 0, 6)}"
+  description = "${local.resource_prefix}-ecs-cluster-logspout-image-codebuild-cloudwatch-rw"
+  policy      = templatefile("${path.root}/policies/cloudwatch-logs-rw.json.tpl", {})
+}
+
+resource "aws_iam_role_policy_attachment" "infrastructure_ecs_cluster_logspout_image_codebuild_cloudwatch_rw" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  role       = aws_iam_role.infrastructure_ecs_cluster_logspout_image_codebuild[0].name
+  policy_arn = aws_iam_policy.infrastructure_ecs_cluster_logspout_image_codebuild_cloudwatch_rw[0].arn
+}
+
+resource "aws_iam_policy" "infrastructure_ecs_cluster_logspout_image_codebuild_allow_builds" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  name        = "${local.resource_prefix}-${substr(sha512("ecs-cluster-logspout-image-codebuild-allow-builds"), 0, 6)}"
+  description = "${local.resource_prefix}-ecs-cluster-logspout-image-codebuild-allow-builds"
+  policy      = templatefile("${path.root}/policies/codebuild-allow-builds.json.tpl", {})
+}
+
+resource "aws_iam_role_policy_attachment" "infrastructure_ecs_cluster_logspout_image_codebuild_allow_builds" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  role       = aws_iam_role.infrastructure_ecs_cluster_logspout_image_codebuild[0].name
+  policy_arn = aws_iam_policy.infrastructure_ecs_cluster_logspout_image_codebuild_allow_builds[0].arn
+}
+
+resource "aws_iam_policy" "infrastructure_ecs_cluster_logspout_image_codebuild_ecr_push" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  name        = "${local.resource_prefix}-${substr(sha512("ecs-cluster-logspout-image-codebuild-ecr-push"), 0, 6)}"
+  description = "${local.resource_prefix}-ecs-cluster-logspout-image-codebuild-ecr-push"
+  policy = templatefile(
+    "${path.root}/policies/ecr-push.json.tpl",
+    { ecr_repository_arn = aws_ecr_repository.infrastructure_ecs_cluster_logspout[0].arn }
+  )
+}
+
+resource "aws_iam_role_policy_attachment" "infrastructure_ecs_cluster_logspout_image_codebuild_ecr_push" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  role       = aws_iam_role.infrastructure_ecs_cluster_logspout_image_codebuild[0].name
+  policy_arn = aws_iam_policy.infrastructure_ecs_cluster_logspout_image_codebuild_ecr_push[0].arn
+}
+
+resource "aws_codebuild_project" "infrastructure_ecs_cluster_logspout_image_build" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  name          = "${local.resource_prefix}-ecs-cluster-logspout-image-build"
+  description   = "${local.resource_prefix} ECS Cluster Logspout Image Build"
+  build_timeout = "20"
+  service_role  = aws_iam_role.infrastructure_ecs_cluster_logspout_image_codebuild[0].arn
+
+  artifacts {
+    type = "NO_ARTIFACTS"
+  }
+
+  environment {
+    compute_type    = "BUILD_GENERAL1_SMALL"
+    image           = "aws/codebuild/standard:7.0"
+    type            = "LINUX_CONTAINER"
+    privileged_mode = true
+
+    environment_variable {
+      name  = "AWS_ACCOUNT_ID"
+      value = local.aws_account_id
+    }
+
+    environment_variable {
+      name  = "REPOSITORY_URI"
+      value = aws_ecr_repository.infrastructure_ecs_cluster_logspout[0].repository_url
+    }
+
+    environment_variable {
+      name  = "DOCKERHUB_USERNAME"
+      value = local.infrastructure_dockerhub_username
+    }
+
+    environment_variable {
+      name  = "DOCKERHUB_TOKEN"
+      value = local.infrastructure_dockerhub_token
+    }
+  }
+
+  source {
+    type      = "NO_SOURCE"
+    buildspec = templatefile("${path.root}/buildspecs/dalmatian-logspout.yml", {})
+  }
+
+  depends_on = [
+    aws_iam_role_policy_attachment.infrastructure_ecs_cluster_logspout_image_codebuild_cloudwatch_rw,
+    aws_iam_role_policy_attachment.infrastructure_ecs_cluster_logspout_image_codebuild_allow_builds,
+    aws_iam_role_policy_attachment.infrastructure_ecs_cluster_logspout_image_codebuild_ecr_push,
+  ]
+}
+
+resource "terraform_data" "infrastructure_ecs_cluster_logspout_image_build_trigger_codebuild" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  triggers_replace = [
+    md5(templatefile("${path.root}/buildspecs/dalmatian-logspout.yml", {})),
+  ]
+
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+      ${path.root}/local-exec-scripts/trigger-codedeploy-project.sh \
+      -n "${aws_codebuild_project.infrastructure_ecs_cluster_logspout_image_build[0].name}"
+    EOF
+  }
+
+  depends_on = [
+
+  ]
+}
+
+resource "aws_cloudwatch_event_rule" "infrastructure_ecs_cluster_logspout_image_build_trigger_codebuild" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  name                = "${local.resource_prefix_hash}-ecs-cluster-logspout-image-build-trigger-codebuild"
+  description         = "${local.resource_prefix} ECS Cluster Logspout Image Build Trigger CodeBuild"
+  schedule_expression = "rate(24 hours)"
+}
+
+resource "aws_cloudwatch_event_target" "infrastructure_ecs_cluster_logspout_image_build_trigger_codebuild" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  target_id = "${local.resource_prefix_hash}-ecs-cluster-image-build-trigger-codebuild"
+  rule      = aws_cloudwatch_event_rule.infrastructure_ecs_cluster_logspout_image_build_trigger_codebuild[0].name
+  arn       = aws_codebuild_project.infrastructure_ecs_cluster_logspout_image_build[0].id
+  role_arn  = aws_iam_role.infrastructure_ecs_cluster_logspout_image_codebuild[0].arn
+}

ecs-cluster-infrastructure-logspout-service.tf

@@ -0,0 +1,45 @@
+resource "aws_ecs_task_definition" "infrastructure_ecs_cluster_logspout" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  family = "${local.resource_prefix}-ecs-infrastructure-logsput"
+  container_definitions = templatefile(
+    "./container-definitions/app.json.tpl",
+    {
+      container_name      = "logspout"
+      image               = aws_ecr_repository.infrastructure_ecs_cluster_logspout[0].repository_url
+      entrypoint          = jsonencode([])
+      command             = jsonencode([local.infrastructure_ecs_cluster_syslog_endpoint])
+      environment_file_s3 = ""
+      environment         = jsonencode([])
+      secrets             = jsonencode([])
+      container_port      = 0
+      extra_hosts         = jsonencode([])
+      volumes = jsonencode([
+        {
+          sourceVolume  = "dockersock"
+          containerPath = "/var/run/docker.sock"
+        }
+      ])
+      linux_parameters      = jsonencode({})
+      cloudwatch_log_group  = ""
+      awslogs_stream_prefix = ""
+      region                = local.aws_region
+    }
+  )
+
+  volume {
+    name      = "dockersock"
+    host_path = "/var/run/docker.sock"
+  }
+  network_mode             = "bridge"
+  requires_compatibilities = ["EC2"]
+}
+
+resource "aws_ecs_service" "infrastructure_ecs_cluster_logspout" {
+  count = local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  name                = "logsput"
+  cluster             = aws_ecs_cluster.infrastructure[0].name
+  task_definition     = aws_ecs_task_definition.infrastructure_ecs_cluster_logspout[0].arn
+  scheduling_strategy = "DAEMON"
+}

ecs-cluster-infrastructure-security-group.tf

@@ -126,3 +126,16 @@ resource "aws_security_group_rule" "infrastructure_ecs_cluster_container_instanc
   cidr_blocks              = length(each.value["cidr_blocks"]) > 0 ? each.value["cidr_blocks"] : null
   security_group_id        = aws_security_group.infrastructure_ecs_cluster_container_instances[0].id
 }
+
+resource "aws_security_group_rule" "infrastructure_ecs_cluster_container_instances_egress_logspout_tcp" {
+  count = local.enable_infrastructure_ecs_cluster && local.infrastrucutre_ecs_cluster_logspout_enabled ? 1 : 0
+
+  description = "Allow Logspout tcp outbound"
+  type        = "egress"
+  from_port   = local.infrastructure_ecs_cluster_syslog_port
+  to_port     = local.infrastructure_ecs_cluster_syslog_port
+  protocol    = "tcp"
+  # tfsec:ignore:aws-ec2-no-public-egress-sgr
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = aws_security_group.infrastructure_ecs_cluster_container_instances[0].id
+}

ecs-cluster-infrastructure-service-scheduled-task.tf

@@ -20,6 +20,7 @@ resource "aws_ecs_task_definition" "infrastructure_ecs_cluster_service_scheduled
       container_name      = each.value["container_name"]
       image               = aws_ecr_repository.infrastructure_ecs_cluster_service[each.value["container_name"]].repository_url
       entrypoint          = each.value["entrypoint"] != null ? jsonencode(each.value["entrypoint"]) : "[]"
+      command             = jsonencode([])
       environment_file_s3 = "${aws_s3_bucket.infrastructure_ecs_cluster_service_environment_files[0].arn}/${each.value["container_name"]}.env"
       environment         = jsonencode([])
       secrets             = jsonencode([])

ecs-cluster-infrastructure-service.tf

@@ -222,6 +222,7 @@ resource "aws_ecs_task_definition" "infrastructure_ecs_cluster_service" {
       container_name      = each.key
       image               = aws_ecr_repository.infrastructure_ecs_cluster_service[each.key].repository_url
       entrypoint          = each.value["container_entrypoint"] != null ? jsonencode(each.value["container_entrypoint"]) : "[]"
+      command             = jsonencode([])
       environment_file_s3 = "${aws_s3_bucket.infrastructure_ecs_cluster_service_environment_files[0].arn}/${each.key}.env"
       environment         = jsonencode([])
       secrets             = jsonencode([])

locals.tf

@@ -175,6 +175,10 @@ locals {
   infrastructure_ecs_cluster_enable_execute_command_logging           = var.infrastructure_ecs_cluster_enable_execute_command_logging
   infrastructure_ecs_cluster_wafs                                     = var.infrastructure_ecs_cluster_wafs
   infrastructure_ecs_cluster_enable_ssm_dhmc                          = local.enable_infrastructure_ecs_cluster ? data.external.ssm_dhmc_setting[0].result.setting_value != "$None" : false
+  infrastructure_ecs_cluster_syslog_endpoint                          = var.infrastructure_ecs_cluster_syslog_endpoint
+  infrastructure_ecs_cluster_syslog_port                              = local.infrastructure_ecs_cluster_syslog_endpoint != "" ? split(":", local.infrastructure_ecs_cluster_syslog_endpoint)[2] : null
+  infrastructure_ecs_cluster_syslog_permitted_peer                    = var.infrastructure_ecs_cluster_syslog_permitted_peer
+  infrastrucutre_ecs_cluster_logspout_enabled                         = local.enable_infrastructure_ecs_cluster && local.infrastructure_ecs_cluster_syslog_endpoint != ""
   infrastructure_ecs_cluster_user_data = base64encode(
     templatefile("ec2-userdata/ecs-instance.tpl", {
       docker_storage_volume_device_name = local.infrastructure_ecs_cluster_ebs_docker_storage_volume_device_name,
@@ -185,9 +189,11 @@ locals {
       efs_id = local.enable_infrastructure_ecs_cluster_efs && (
         local.infrastructure_vpc_network_enable_private || local.infrastructure_vpc_network_enable_public
       ) ? aws_efs_file_system.infrastructure_ecs_cluster[0].id : "",
-      region         = local.aws_region,
-      efs_dirs       = join(" ", local.ecs_cluster_efs_directories),
-      log_debug_mode = local.infrastructure_ecs_cluster_enable_debug_mode
+      region                = local.aws_region,
+      efs_dirs              = join(" ", local.ecs_cluster_efs_directories),
+      syslog_endpoint       = local.infrastructure_ecs_cluster_syslog_endpoint
+      syslog_permitted_peer = local.infrastructure_ecs_cluster_syslog_permitted_peer
+      log_debug_mode        = local.infrastructure_ecs_cluster_enable_debug_mode
     })
   )
 

rds-infrastructure-s3-backups-task-definition.tf

@@ -140,6 +140,7 @@ resource "aws_ecs_task_definition" "infrastructure_rds_s3_backups_scheduled_task
           s3_bucket_name = aws_s3_bucket.infrastructure_rds_s3_backups[0].bucket
         }
       )])
+      command             = jsonencode([])
       environment_file_s3 = ""
       environment = jsonencode([
         {

variables.tf

@@ -423,6 +423,16 @@ variable "infrastructure_ecs_cluster_enable_execute_command_logging" {
   type        = bool
 }
 
+variable "infrastructure_ecs_cluster_syslog_endpoint" {
+  description = "ECS Infrastructure Syslog endpoint. If specified, rsyslog will be installed on the ECS container instances and configured to send logs to this endpoint. Logspout containers will also be launched to gather and send Docker logs (Application logs from the running ECS services). The port must be included in the URI, eg. 'syslog+tls://example.com:1234'"
+  type        = string
+}
+
+variable "infrastructure_ecs_cluster_syslog_permitted_peer" {
+  description = "Specify the certificate common name (CN) of the remote to ensure syslog communication is restricted to permitted endpoints (eg. '*.example.com')"
+  type        = string
+}
+
 variable "infrastructure_ecs_cluster_wafs" {
   description = "Map of WAF ACLs to craete, which can be used with service CloudFront distributions"
   type = map(object({