E3S deploy by terraform

.gitignore

@@ -0,0 +1,5 @@
+terraform.tfvars
+terraform.tfstate
+terraform.tfstate.backup
+*.terraform
+*.plan

deploy/autoscaling.tf

@@ -0,0 +1,141 @@
+resource "aws_autoscaling_group" "linux" {
+  name = local.e3s_linux_autoscaling_name
+  mixed_instances_policy {
+    launch_template {
+      launch_template_specification {
+        launch_template_id = aws_launch_template.e3s_linux.id
+        version            = aws_launch_template.e3s_linux.latest_version
+      }
+
+      dynamic "override" {
+        for_each = var.instance_types
+        content {
+          weighted_capacity = override.value.weight
+          instance_type     = override.value.instance_type
+        }
+      }
+    }
+
+    instances_distribution {
+      // as of now, there is no support of usual if/else blocks
+      // if var.linux_linux_spot_price == 0 use only on-demand instances, else will be used only on-spot
+      on_demand_percentage_above_base_capacity = var.spot_price.linux == "" ? 100 : 0
+      spot_max_price                           = var.spot_price.linux
+      spot_allocation_strategy                 = "capacity-optimized-prioritized"
+      on_demand_allocation_strategy            = "prioritized"
+    }
+  }
+
+  desired_capacity = 0
+  min_size         = 0
+  max_size         = 50
+
+  default_cooldown = 10
+
+  health_check_type         = "EC2"
+  health_check_grace_period = 10
+
+  vpc_zone_identifier = [for s in aws_subnet.private_per_zone : s.id]
+
+  termination_policies  = ["AllocationStrategy"]
+  protect_from_scale_in = true
+
+  force_delete            = true
+  service_linked_role_arn = format("arn:aws:iam::%s:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", data.aws_caller_identity.current.account_id)
+
+  lifecycle {
+    ignore_changes = [desired_capacity, min_size, max_size, tag]
+  }
+}
+
+resource "aws_autoscaling_group" "windows" {
+  name = local.e3s_windows_autoscaling_name
+  mixed_instances_policy {
+    launch_template {
+      launch_template_specification {
+        launch_template_id = aws_launch_template.e3s_windows.id
+        version            = aws_launch_template.e3s_windows.latest_version
+      }
+
+      dynamic "override" {
+        for_each = var.instance_types
+        content {
+          weighted_capacity = override.value.weight
+          instance_type     = override.value.instance_type
+        }
+      }
+    }
+
+    instances_distribution {
+      // as of now, there is no support of usual if/else blocks
+      // if var.windows_spot_price == 0 use only on-demand instances, else will be used only on-spot
+      on_demand_percentage_above_base_capacity = var.spot_price.windows == "" ? 100 : 0
+      spot_max_price                           = var.spot_price.windows
+      spot_allocation_strategy                 = "capacity-optimized-prioritized"
+      on_demand_allocation_strategy            = "prioritized"
+    }
+  }
+
+  desired_capacity = 0
+  min_size         = 0
+  max_size         = 50
+
+  default_cooldown = 10
+
+  health_check_type         = "EC2"
+  health_check_grace_period = 10
+
+  vpc_zone_identifier = [for s in aws_subnet.private_per_zone : s.id]
+
+  termination_policies  = ["AllocationStrategy"]
+  protect_from_scale_in = true
+
+  force_delete            = true
+  service_linked_role_arn = format("arn:aws:iam::%s:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", data.aws_caller_identity.current.account_id)
+
+  lifecycle {
+    ignore_changes = [desired_capacity, min_size, max_size, tag]
+  }
+}
+
+resource "aws_autoscaling_policy" "linux_forecast" {
+  autoscaling_group_name = aws_autoscaling_group.linux.name
+  name                   = "predictive"
+  policy_type            = "PredictiveScaling"
+  predictive_scaling_configuration {
+    metric_specification {
+      target_value = 100
+      predefined_metric_pair_specification {
+        predefined_metric_type = "ASGCPUUtilization"
+      }
+    }
+    mode                         = "ForecastAndScale"
+    scheduling_buffer_time       = "120"
+    max_capacity_breach_behavior = "HonorMaxCapacity"
+  }
+
+  lifecycle {
+    ignore_changes = [predictive_scaling_configuration]
+  }
+}
+
+resource "aws_autoscaling_policy" "windows_forecast" {
+  autoscaling_group_name = aws_autoscaling_group.windows.name
+  name                   = "predictive"
+  policy_type            = "PredictiveScaling"
+  predictive_scaling_configuration {
+    metric_specification {
+      target_value = 100
+      predefined_metric_pair_specification {
+        predefined_metric_type = "ASGCPUUtilization"
+      }
+    }
+    mode                         = "ForecastAndScale"
+    scheduling_buffer_time       = "300"
+    max_capacity_breach_behavior = "HonorMaxCapacity"
+  }
+
+  lifecycle {
+    ignore_changes = [predictive_scaling_configuration]
+  }
+}

deploy/cloudwatch.tf

@@ -0,0 +1,20 @@
+resource "aws_vpc_endpoint" "cloudwatch" {
+  count = var.enable_cloudwatch ? 1 : 0
+
+  vpc_id     = aws_vpc.main.id
+  subnet_ids = [for s in aws_subnet.private_per_zone : s.id]
+
+  service_name       = format("com.amazonaws.%s.logs", var.region)
+  vpc_endpoint_type  = "Interface"
+  policy             = file("./iam_data/cloudwatch-endpoint-policy.json")
+  security_group_ids = [aws_security_group.cloudwatch[0].id]
+
+  private_dns_enabled = true
+}
+
+resource "aws_cloudwatch_log_group" "e3s_tasks" {
+  count             = var.enable_cloudwatch ? 1 : 0
+  name              = local.e3s_log_group_name
+  log_group_class   = "STANDARD"
+  retention_in_days = 3
+}

deploy/e3s.tf

@@ -0,0 +1,125 @@
+locals {
+  zone_subnet_map = { for subnet in aws_subnet.public_per_zone : subnet.availability_zone => subnet.id }
+}
+
+resource "random_shuffle" "e3s_subnet_location" {
+  input        = sort([for location in data.aws_ec2_instance_type_offerings.supported_server_zones.locations : location])
+  result_count = 1
+}
+
+data "aws_ec2_instance_type_offerings" "supported_server_zones" {
+  filter {
+    name   = "instance-type"
+    values = [var.e3s_server_instance_type]
+  }
+
+  location_type = "availability-zone"
+}
+
+resource "tls_private_key" "pri_key" {
+  count     = var.allow_agent_ssh ? 1 : 0
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "aws_key_pair" "agent" {
+  count      = var.allow_agent_ssh ? 1 : 0
+  key_name   = local.e3s_agent_key_name
+  public_key = tls_private_key.pri_key[0].public_key_openssh
+}
+
+data "aws_ami" "ubuntu_22_04" {
+  most_recent = true
+
+  # Amazon
+  owners = ["099720109477"]
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["x86_64"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "image-type"
+    values = ["machine"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+resource "aws_instance" "e3s_server" {
+  ami           = data.aws_ami.ubuntu_22_04.id
+  instance_type = var.e3s_server_instance_type
+
+  subnet_id = local.zone_subnet_map[random_shuffle.e3s_subnet_location.result[0]]
+
+  key_name = var.e3s_key_name
+
+  vpc_security_group_ids = [aws_security_group.e3s_server.id]
+
+  iam_instance_profile = aws_iam_instance_profile.e3s.name
+
+  cpu_options {
+    core_count       = 1
+    threads_per_core = 2
+  }
+
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_tokens                 = "required"
+    http_put_response_hop_limit = 2
+  }
+
+  tags = {
+    Name = local.e3s_server_instance_name
+  }
+
+  user_data = templatefile("./ec2_data/e3s_user_data.sh", {
+    region                   = var.region
+    cluster_name             = local.e3s_cluster_name
+    task_role                = local.e3s_task_role_name
+    env                      = var.environment
+    linux_capacityprovider   = local.e3s_linux_capacityprovider
+    windows_capacityprovider = local.e3s_windows_capacityprovider
+    target_group             = local.e3s_tg_name
+    bucket_name              = var.bucket.name
+    bucket_region            = length(aws_s3_bucket.main) > 0 ? var.region : var.bucket.region
+    log_group                = length(aws_cloudwatch_log_group.e3s_tasks) > 0 ? local.e3s_log_group_name : ""
+
+    zbr_host = var.zebrunner.host
+    zbr_user = var.zebrunner.user
+    zbr_pass = var.zebrunner.pass
+
+    agent_key      = length(tls_private_key.pri_key) > 0 ? tls_private_key.pri_key[0].private_key_pem : ""
+    agent_key_name = local.e3s_agent_key_name
+
+    # db_dns      = aws_rds_cluster.aurora.endpoint
+    remote_data   = var.data_layer_remote
+    db_username   = var.remote_db.username
+    db_pass       = var.remote_db.pass
+    db_name       = var.data_layer_remote ? aws_db_instance.postgres[0].db_name : ""
+    db_dns        = var.data_layer_remote ? aws_db_instance.postgres[0].endpoint : ""
+    cache_address = var.data_layer_remote ? aws_elasticache_serverless_cache.redis[0].endpoint[0].address : ""
+    cache_port    = var.data_layer_remote ? aws_elasticache_serverless_cache.redis[0].endpoint[0].port : ""
+  })
+
+  # depends_on = [aws_ecs_cluster.e3s, aws_lb_listener.main, aws_rds_cluster_instance.aurora_instance]
+  depends_on = [aws_ecs_cluster.e3s, aws_lb_listener.main]
+
+  lifecycle {
+    ignore_changes = [user_data]
+  }
+}