From baec3f45f770b138283a75075d8978badb90c416 Mon Sep 17 00:00:00 2001 From: dmtgrinevich Date: Wed, 17 Jul 2024 14:43:10 +0000 Subject: [PATCH] E3S deploy by terraform --- .gitignore | 5 + deploy/.terraform.lock.hcl | 64 ++++++ deploy/autoscaling.tf | 141 ++++++++++++ deploy/cloudwatch.tf | 20 ++ deploy/e3s.tf | 125 +++++++++++ deploy/ec2.tf | 118 ++++++++++ deploy/ec2_data/e3s_user_data.sh | 95 +++++++++ deploy/ec2_data/linux_user_data.sh | 21 ++ deploy/ec2_data/windows_user_data.ps1 | 16 ++ deploy/ecs.tf | 56 +++++ deploy/elbv2.tf | 58 +++++ deploy/iam.tf | 77 +++++++ .../iam_data/cloudwatch-endpoint-policy.json | 15 ++ deploy/iam_data/e3s-agent-policy.json | 35 +++ deploy/iam_data/e3s-policy.json | 82 +++++++ deploy/iam_data/e3s-task-policy.json | 14 ++ deploy/iam_data/s3-bucket-policy.json | 22 ++ deploy/iam_data/s3-endpoint-policy.json | 29 +++ deploy/outputs.tf | 44 ++++ deploy/provider.tf | 31 +++ deploy/rds.tf | 64 ++++++ deploy/redis.tf | 45 ++++ deploy/s3.tf | 26 +++ deploy/security_groups.tf | 201 ++++++++++++++++++ deploy/vars.tf | 142 +++++++++++++ deploy/vpc.tf | 77 +++++++ 26 files changed, 1623 insertions(+) create mode 100644 .gitignore create mode 100644 deploy/.terraform.lock.hcl create mode 100644 deploy/autoscaling.tf create mode 100644 deploy/cloudwatch.tf create mode 100644 deploy/e3s.tf create mode 100644 deploy/ec2.tf create mode 100644 deploy/ec2_data/e3s_user_data.sh create mode 100644 deploy/ec2_data/linux_user_data.sh create mode 100644 deploy/ec2_data/windows_user_data.ps1 create mode 100644 deploy/ecs.tf create mode 100644 deploy/elbv2.tf create mode 100644 deploy/iam.tf create mode 100644 deploy/iam_data/cloudwatch-endpoint-policy.json create mode 100644 deploy/iam_data/e3s-agent-policy.json create mode 100644 deploy/iam_data/e3s-policy.json create mode 100644 deploy/iam_data/e3s-task-policy.json create mode 100644 deploy/iam_data/s3-bucket-policy.json create mode 100644 deploy/iam_data/s3-endpoint-policy.json create mode 100644 deploy/outputs.tf create mode 100644 deploy/provider.tf create mode 100644 deploy/rds.tf create mode 100644 deploy/redis.tf create mode 100644 deploy/s3.tf create mode 100644 deploy/security_groups.tf create mode 100644 deploy/vars.tf create mode 100644 deploy/vpc.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..54b1ec3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +terraform.tfvars +terraform.tfstate +terraform.tfstate.backup +*.terraform +*.plan diff --git a/deploy/.terraform.lock.hcl b/deploy/.terraform.lock.hcl new file mode 100644 index 0000000..2e8d5ee --- /dev/null +++ b/deploy/.terraform.lock.hcl @@ -0,0 +1,64 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.54.1" + constraints = "5.54.1" + hashes = [ + "h1:SOdZNOAcBvbrkV6V1S7UiGh9K//O66qfyXpHgyXeBeI=", + "zh:37c09b9a0a0a2f7854fe52c6adb15f71593810b458a8283ed71d68036af7ba3a", + "zh:42fe11d87723d4e43b9c6224ae6bacdcb53faee8abc58f0fc625a161d1f71cb1", + "zh:57c6dfc46f28c9c2737559bd84acbc05aeae90431e731bb72a0024028a2d2412", + "zh:5ba9665a4ca0e182effd75575b19a4d47383ec02662024b9fe26f78286c36619", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b55980be0237644123a02a30b56d4cc03863ef29036c47d6e8ab5429ab45adf5", + "zh:b81e7664f10855a3a6fc234a18b4c4f1456273126a40c41516f2061696fb9870", + "zh:bd09736ffafd92af104c3c34b5add138ae8db4402eb687863ce472ca7e5ff2e2", + "zh:cc2eb1c62fba2a11d1f239e650cc2ae94bcab01c907384dcf2e213a6ee1bd5b2", + "zh:e5dc40205d9cf6f353c0ca532ae29afc6c83928bc9bcca47d74b640d3bb5a38c", + "zh:ebf1acdcd13f10db1b9c85050ddaadc70ab269c47c5a240753362446442d8371", + "zh:f2fc28a4ad94af5e6144a7309286505e3eb7a94d9dc106722b506c372ff7f591", + "zh:f49445e8435944df122aa89853260a2716ba8b73d6a6a70cae1661554926d5a2", + "zh:fc3b5046e60ae7cab20715be23de8436eb12736136fd6d0f0cc1549ebda6cc73", + "zh:fdb98a53500e245a3b5bec077b994da6959dba8fc4eb7534528658d820e06bd5", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.2" + constraints = "3.6.2" + hashes = [ + "h1:wmG0QFjQ2OfyPy6BB7mQ57WtoZZGGV07uAPQeDmIrAE=", + "zh:0ef01a4f81147b32c1bea3429974d4d104bbc4be2ba3cfa667031a8183ef88ec", + "zh:1bcd2d8161e89e39886119965ef0f37fcce2da9c1aca34263dd3002ba05fcb53", + "zh:37c75d15e9514556a5f4ed02e1548aaa95c0ecd6ff9af1119ac905144c70c114", + "zh:4210550a767226976bc7e57d988b9ce48f4411fa8a60cd74a6b246baf7589dad", + "zh:562007382520cd4baa7320f35e1370ffe84e46ed4e2071fdc7e4b1a9b1f8ae9b", + "zh:5efb9da90f665e43f22c2e13e0ce48e86cae2d960aaf1abf721b497f32025916", + "zh:6f71257a6b1218d02a573fc9bff0657410404fb2ef23bc66ae8cd968f98d5ff6", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9647e18f221380a85f2f0ab387c68fdafd58af6193a932417299cdcae4710150", + "zh:bb6297ce412c3c2fa9fec726114e5e0508dd2638cad6a0cb433194930c97a544", + "zh:f83e925ed73ff8a5ef6e3608ad9225baa5376446349572c2449c0c0b3cf184b7", + "zh:fbef0781cb64de76b1df1ca11078aecba7800d82fd4a956302734999cfd9a4af", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.5" + hashes = [ + "h1:e4LBdJoZJNOQXPWgOAG0UuPBVhCStu98PieNlqJTmeU=", + "zh:01cfb11cb74654c003f6d4e32bbef8f5969ee2856394a96d127da4949c65153e", + "zh:0472ea1574026aa1e8ca82bb6df2c40cd0478e9336b7a8a64e652119a2fa4f32", + "zh:1a8ddba2b1550c5d02003ea5d6cdda2eef6870ece86c5619f33edd699c9dc14b", + "zh:1e3bb505c000adb12cdf60af5b08f0ed68bc3955b0d4d4a126db5ca4d429eb4a", + "zh:6636401b2463c25e03e68a6b786acf91a311c78444b1dc4f97c539f9f78de22a", + "zh:76858f9d8b460e7b2a338c477671d07286b0d287fd2d2e3214030ae8f61dd56e", + "zh:a13b69fb43cb8746793b3069c4d897bb18f454290b496f19d03c3387d1c9a2dc", + "zh:a90ca81bb9bb509063b736842250ecff0f886a91baae8de65c8430168001dad9", + "zh:c4de401395936e41234f1956ebadbd2ed9f414e6908f27d578614aaa529870d4", + "zh:c657e121af8fde19964482997f0de2d5173217274f6997e16389e7707ed8ece8", + "zh:d68b07a67fbd604c38ec9733069fbf23441436fecf554de6c75c032f82e1ef19", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/deploy/autoscaling.tf b/deploy/autoscaling.tf new file mode 100644 index 0000000..bb89561 --- /dev/null +++ b/deploy/autoscaling.tf @@ -0,0 +1,141 @@ +resource "aws_autoscaling_group" "linux" { + name = local.e3s_linux_autoscaling_name + mixed_instances_policy { + launch_template { + launch_template_specification { + launch_template_id = aws_launch_template.e3s_linux.id + version = aws_launch_template.e3s_linux.latest_version + } + + dynamic "override" { + for_each = var.instance_types + content { + weighted_capacity = override.value.weight + instance_type = override.value.instance_type + } + } + } + + instances_distribution { + // as of now, there is no support of usual if/else blocks + // if var.linux_linux_spot_price == 0 use only on-demand instances, else will be used only on-spot + on_demand_percentage_above_base_capacity = var.spot_price.linux == "" ? 100 : 0 + spot_max_price = var.spot_price.linux + spot_allocation_strategy = "capacity-optimized-prioritized" + on_demand_allocation_strategy = "prioritized" + } + } + + desired_capacity = 0 + min_size = 0 + max_size = 50 + + default_cooldown = 10 + + health_check_type = "EC2" + health_check_grace_period = 10 + + vpc_zone_identifier = [for s in aws_subnet.private_per_zone : s.id] + + termination_policies = ["AllocationStrategy"] + protect_from_scale_in = true + + force_delete = true + service_linked_role_arn = format("arn:aws:iam::%s:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", data.aws_caller_identity.current.account_id) + + lifecycle { + ignore_changes = [desired_capacity, min_size, max_size, tag] + } +} + +resource "aws_autoscaling_group" "windows" { + name = local.e3s_windows_autoscaling_name + mixed_instances_policy { + launch_template { + launch_template_specification { + launch_template_id = aws_launch_template.e3s_windows.id + version = aws_launch_template.e3s_windows.latest_version + } + + dynamic "override" { + for_each = var.instance_types + content { + weighted_capacity = override.value.weight + instance_type = override.value.instance_type + } + } + } + + instances_distribution { + // as of now, there is no support of usual if/else blocks + // if var.windows_spot_price == 0 use only on-demand instances, else will be used only on-spot + on_demand_percentage_above_base_capacity = var.spot_price.windows == "" ? 100 : 0 + spot_max_price = var.spot_price.windows + spot_allocation_strategy = "capacity-optimized-prioritized" + on_demand_allocation_strategy = "prioritized" + } + } + + desired_capacity = 0 + min_size = 0 + max_size = 50 + + default_cooldown = 10 + + health_check_type = "EC2" + health_check_grace_period = 10 + + vpc_zone_identifier = [for s in aws_subnet.private_per_zone : s.id] + + termination_policies = ["AllocationStrategy"] + protect_from_scale_in = true + + force_delete = true + service_linked_role_arn = format("arn:aws:iam::%s:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", data.aws_caller_identity.current.account_id) + + lifecycle { + ignore_changes = [desired_capacity, min_size, max_size, tag] + } +} + +resource "aws_autoscaling_policy" "linux_forecast" { + autoscaling_group_name = aws_autoscaling_group.linux.name + name = "predictive" + policy_type = "PredictiveScaling" + predictive_scaling_configuration { + metric_specification { + target_value = 100 + predefined_metric_pair_specification { + predefined_metric_type = "ASGCPUUtilization" + } + } + mode = "ForecastAndScale" + scheduling_buffer_time = "120" + max_capacity_breach_behavior = "HonorMaxCapacity" + } + + lifecycle { + ignore_changes = [predictive_scaling_configuration] + } +} + +resource "aws_autoscaling_policy" "windows_forecast" { + autoscaling_group_name = aws_autoscaling_group.windows.name + name = "predictive" + policy_type = "PredictiveScaling" + predictive_scaling_configuration { + metric_specification { + target_value = 100 + predefined_metric_pair_specification { + predefined_metric_type = "ASGCPUUtilization" + } + } + mode = "ForecastAndScale" + scheduling_buffer_time = "300" + max_capacity_breach_behavior = "HonorMaxCapacity" + } + + lifecycle { + ignore_changes = [predictive_scaling_configuration] + } +} diff --git a/deploy/cloudwatch.tf b/deploy/cloudwatch.tf new file mode 100644 index 0000000..e432c59 --- /dev/null +++ b/deploy/cloudwatch.tf @@ -0,0 +1,20 @@ +resource "aws_vpc_endpoint" "cloudwatch" { + count = var.enable_cloudwatch ? 1 : 0 + + vpc_id = aws_vpc.main.id + subnet_ids = [for s in aws_subnet.private_per_zone : s.id] + + service_name = format("com.amazonaws.%s.logs", var.region) + vpc_endpoint_type = "Interface" + policy = file("./iam_data/cloudwatch-endpoint-policy.json") + security_group_ids = [aws_security_group.cloudwatch[0].id] + + private_dns_enabled = true +} + +resource "aws_cloudwatch_log_group" "e3s_tasks" { + count = var.enable_cloudwatch ? 1 : 0 + name = local.e3s_log_group_name + log_group_class = "STANDARD" + retention_in_days = 3 +} diff --git a/deploy/e3s.tf b/deploy/e3s.tf new file mode 100644 index 0000000..c984d8a --- /dev/null +++ b/deploy/e3s.tf @@ -0,0 +1,125 @@ +locals { + zone_subnet_map = { for subnet in aws_subnet.public_per_zone : subnet.availability_zone => subnet.id } +} + +resource "random_shuffle" "e3s_subnet_location" { + input = sort([for location in data.aws_ec2_instance_type_offerings.supported_server_zones.locations : location]) + result_count = 1 +} + +data "aws_ec2_instance_type_offerings" "supported_server_zones" { + filter { + name = "instance-type" + values = [var.e3s_server_instance_type] + } + + location_type = "availability-zone" +} + +resource "tls_private_key" "pri_key" { + count = var.allow_agent_ssh ? 1 : 0 + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "aws_key_pair" "agent" { + count = var.allow_agent_ssh ? 1 : 0 + key_name = local.e3s_agent_key_name + public_key = tls_private_key.pri_key[0].public_key_openssh +} + +data "aws_ami" "ubuntu_22_04" { + most_recent = true + + # Amazon + owners = ["099720109477"] + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "image-type" + values = ["machine"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } +} + +resource "aws_instance" "e3s_server" { + ami = data.aws_ami.ubuntu_22_04.id + instance_type = var.e3s_server_instance_type + + subnet_id = local.zone_subnet_map[random_shuffle.e3s_subnet_location.result[0]] + + key_name = var.e3s_key_name + + vpc_security_group_ids = [aws_security_group.e3s_server.id] + + iam_instance_profile = aws_iam_instance_profile.e3s.name + + cpu_options { + core_count = 1 + threads_per_core = 2 + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } + + tags = { + Name = local.e3s_server_instance_name + } + + user_data = templatefile("./ec2_data/e3s_user_data.sh", { + region = var.region + cluster_name = local.e3s_cluster_name + task_role = local.e3s_task_role_name + env = var.environment + linux_capacityprovider = local.e3s_linux_capacityprovider + windows_capacityprovider = local.e3s_windows_capacityprovider + target_group = local.e3s_tg_name + bucket_name = var.bucket.name + bucket_region = length(aws_s3_bucket.main) > 0 ? var.region : var.bucket.region + log_group = length(aws_cloudwatch_log_group.e3s_tasks) > 0 ? local.e3s_log_group_name : "" + + zbr_host = var.zebrunner.host + zbr_user = var.zebrunner.user + zbr_pass = var.zebrunner.pass + + agent_key = length(tls_private_key.pri_key) > 0 ? tls_private_key.pri_key[0].private_key_pem : "" + agent_key_name = local.e3s_agent_key_name + + # db_dns = aws_rds_cluster.aurora.endpoint + remote_data = var.data_layer_remote + db_username = var.remote_db.username + db_pass = var.remote_db.pass + db_name = var.data_layer_remote ? aws_db_instance.postgres[0].db_name : "" + db_dns = var.data_layer_remote ? aws_db_instance.postgres[0].endpoint : "" + cache_address = var.data_layer_remote ? aws_elasticache_serverless_cache.redis[0].endpoint[0].address : "" + cache_port = var.data_layer_remote ? aws_elasticache_serverless_cache.redis[0].endpoint[0].port : "" + }) + + # depends_on = [aws_ecs_cluster.e3s, aws_lb_listener.main, aws_rds_cluster_instance.aurora_instance] + depends_on = [aws_ecs_cluster.e3s, aws_lb_listener.main] + + lifecycle { + ignore_changes = [user_data] + } +} diff --git a/deploy/ec2.tf b/deploy/ec2.tf new file mode 100644 index 0000000..5b700b1 --- /dev/null +++ b/deploy/ec2.tf @@ -0,0 +1,118 @@ +data "aws_ami" "zbr_linux" { + most_recent = true + owners = ["aws-marketplace"] + filter { + name = "name" + values = ["Zebrunner ESG Agent *"] + } + + filter { + name = "block-device-mapping.device-name" + values = ["/dev/xvda"] + } +} + +data "aws_ami" "zbr_windows" { + most_recent = true + owners = ["aws-marketplace"] + filter { + name = "name" + values = ["Zebrunner ESG Agent *"] + } + + filter { + name = "platform" + values = ["windows"] + } +} + +resource "aws_launch_template" "e3s_linux" { + name = local.e3s_linux_launch_template_name + image_id = data.aws_ami.zbr_linux.id + vpc_security_group_ids = [aws_security_group.e3s_agent.id] + ebs_optimized = true + key_name = var.allow_agent_ssh ? aws_key_pair.agent[0].key_name : "" + + instance_initiated_shutdown_behavior = "terminate" + + block_device_mappings { + device_name = "/dev/xvda" + ebs { + volume_size = 70 + volume_type = "gp3" + delete_on_termination = true + encrypted = true + } + } + + monitoring { + enabled = true + } + + iam_instance_profile { + name = aws_iam_instance_profile.e3s_agent.name + } + + hibernation_options { + configured = false + } + + enclave_options { + enabled = false + } + + metadata_options { + http_tokens = "required" + http_endpoint = "enabled" + http_put_response_hop_limit = 1 + } + + disable_api_termination = false + + user_data = base64encode(templatefile("./ec2_data/linux_user_data.sh", { cluster_name = local.e3s_cluster_name, cidr_block = aws_vpc.main.cidr_block })) + + depends_on = [aws_iam_instance_profile.e3s_agent] +} + +resource "aws_launch_template" "e3s_windows" { + name = local.e3s_windows_launch_template_name + image_id = data.aws_ami.zbr_windows.id + vpc_security_group_ids = var.allow_agent_ssh ? [aws_security_group.e3s_agent.id, aws_security_group.windows_rdp[0].id] : [aws_security_group.e3s_agent.id] + key_name = var.allow_agent_ssh ? aws_key_pair.agent[0].key_name : "" + + ebs_optimized = true + block_device_mappings { + device_name = "/dev/sda1" + ebs { + volume_size = 100 + volume_type = "gp3" + delete_on_termination = true + encrypted = true + } + } + monitoring { + enabled = true + } + disable_api_termination = false + + instance_initiated_shutdown_behavior = "terminate" + hibernation_options { + configured = false + } + metadata_options { + http_tokens = "required" + http_endpoint = "enabled" + http_put_response_hop_limit = 1 + } + enclave_options { + enabled = false + } + + iam_instance_profile { + name = aws_iam_instance_profile.e3s_agent.name + } + + user_data = base64encode(templatefile("./ec2_data/windows_user_data.ps1", { cluster_name = local.e3s_cluster_name, cidr_block = aws_vpc.main.cidr_block })) + + depends_on = [aws_iam_instance_profile.e3s_agent] +} diff --git a/deploy/ec2_data/e3s_user_data.sh b/deploy/ec2_data/e3s_user_data.sh new file mode 100644 index 0000000..14a419d --- /dev/null +++ b/deploy/ec2_data/e3s_user_data.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +user="ubuntu" +e3s_path="/home/$user/tools/e3s" + +replace() { + param=$1 + value=$2 + file=$3 + + sed -i -e "s^$param.*^$param=$value^" "$file" +} + +sudo apt-get update && sudo apt-get upgrade + +# install jq +sudo apt-get -y install jq unzip + +# insall aws cli +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip awscliv2.zip +sudo ./aws/install + +# Add Docker's official GPG key: +sudo apt-get -y install ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc +# Add the repository to Apt sources: +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# Add user to docker group +sudo usermod -aG docker "$user" +# Grant admin rights +sudo usermod -aG sudo "$user" + +git config --system --add safe.directory '*' + +mkdir -p "$e3s_path" +git clone https://github.com/zebrunner/e3s.git "$e3s_path" + +if [ -n "${agent_key}" ]; then + echo "${agent_key}" > "$e3s_path"/${agent_key_name}.pem + chmod 400 "$e3s_path"/${agent_key_name}.pem +fi + +cd "$e3s_path" +case ${remote_data} in + (true) + git checkout "terraform-remote" + + # data.env + replace "POSTGRES_PASSWORD" ${db_pass} "./properties/data.env" + replace "DATABASE" "postgres://${db_username}:${db_pass}@${db_dns}/${db_name}" "./properties/data.env" + replace "ELASTIC_CACHE" "${cache_address}:${cache_port}" "./properties/data.env" + replace "CACHE_REMOTE" "true" "./properties/data.env" + ;; + (false) + git checkout "terraform-local" + ;; + (*) + echo "remote_data is not a bool value" + ;; +esac + +# config.env +replace "AWS_REGION" ${region} "./properties/config.env" +replace "AWS_CLUSTER" ${cluster_name} "./properties/config.env" +replace "AWS_TASK_ROLE" ${task_role} "./properties/config.env" +replace "AWS_LOGS_GROUP" ${log_group} "./properties/config.env" +replace "S3_BUCKET" ${bucket_name} "./properties/config.env" +replace "S3_REGION" ${bucket_region} "./properties/config.env" +replace "ZEBRUNNER_HOST" ${zbr_host} "./properties/config.env" +replace "ZEBRUNNER_INTEGRATION_USER" ${zbr_user} "./properties/config.env" +replace "ZEBRUNNER_INTEGRATION_PASSWORD" ${zbr_pass} "./properties/config.env" +replace "ZEBRUNNER_ENV" ${env} "./properties/config.env" + +# router.env +replace "AWS_LINUX_CAPACITY_PROVIDER" ${linux_capacityprovider} "./properties/router.env" +replace "AWS_WIN_CAPACITY_PROVIDER" ${windows_capacityprovider} "./properties/router.env" +replace "AWS_TARGET_GROUP" ${target_group} "./properties/router.env" + +# scaler.env + +# task-definitions.env + +# start server +./zebrunner.sh start + +sudo chown -R "$user" "$e3s_path" \ No newline at end of file diff --git a/deploy/ec2_data/linux_user_data.sh b/deploy/ec2_data/linux_user_data.sh new file mode 100644 index 0000000..d3faf4f --- /dev/null +++ b/deploy/ec2_data/linux_user_data.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +#Amazon ECS stores logs in the /var/log/ecs folder of your container instances +echo ECS_CLUSTER=${cluster_name} >> /etc/ecs/ecs.config +echo ECS_LOGLEVEL=info >> /etc/ecs/ecs.config +echo ECS_LOGLEVEL_ON_INSTANCE=info >> /etc/ecs/ecs.config +echo ECS_AVAILABLE_LOGGING_DRIVERS="[\"json-file\",\"awslogs\"]" >> /etc/ecs/ecs.config +echo ECS_AWSVPC_ADDITIONAL_LOCAL_ROUTES=[\"${cidr_block}\"] >> /etc/ecs/ecs.config +echo ECS_ENABLE_TASK_ENI=true >> /etc/ecs/ecs.config + +#https://aws.amazon.com/blogs/containers/graceful-shutdowns-with-ecs/ +echo ECS_CONTAINER_STOP_TIMEOUT=15s >> /etc/ecs/ecs.config +echo ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=5m >> /etc/ecs/ecs.config +echo ECS_ENABLE_TASK_IAM_ROLE=true >> /etc/ecs/ecs.config +echo ECS_ENABLE_CONTAINER_METADATA=true >> /etc/ecs/ecs.config +echo ECS_AWSVPC_BLOCK_IMDS=true >> /etc/ecs/ecs.config +echo ECS_ENABLE_SPOT_INSTANCE_DRAINING=true >> /etc/ecs/ecs.config +echo ECS_PULL_DEPENDENT_CONTAINERS_UPFRONT=true >> /etc/ecs/ecs.config + +#ECS_IMAGE_PULL_BEHAVIOR +echo ECS_IMAGE_PULL_BEHAVIOR=prefer-cached >> /etc/ecs/ecs.config diff --git a/deploy/ec2_data/windows_user_data.ps1 b/deploy/ec2_data/windows_user_data.ps1 new file mode 100644 index 0000000..8e54255 --- /dev/null +++ b/deploy/ec2_data/windows_user_data.ps1 @@ -0,0 +1,16 @@ + + Import-Module ECSTools + [Environment]::SetEnvironmentVariable("ECS_LOGLEVEL", "info", "Machine") + [Environment]::SetEnvironmentVariable("ECS_LOGLEVEL_ON_INSTANCE", "info", "Machine") + [Environment]::SetEnvironmentVariable("ECS_CONTAINER_STOP_TIMEOUT", "15s", "Machine") + [Environment]::SetEnvironmentVariable("ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION", "5m", "Machine") + [Environment]::SetEnvironmentVariable("ECS_ENABLE_SPOT_INSTANCE_DRAINING", $TRUE, "Machine") + [Environment]::SetEnvironmentVariable("ECS_PULL_DEPENDENT_CONTAINERS_UPFRONT", $TRUE, "Machine") + [Environment]::SetEnvironmentVariable("ECS_ENABLE_CONTAINER_METADATA", "$TRUE", "Machine") + [Environment]::SetEnvironmentVariable("ECS_IMAGE_PULL_BEHAVIOR", "prefer-cached", "Machine") + + Initialize-ECSAgent -Cluster '${cluster_name}' -EnableTaskIAMRole -EnableTaskENI -AwsvpcBlockIMDS -LoggingDrivers '["json-file","awslogs"]' -AwsvpcAdditionalLocalRoutes '["${cidr_block}"]' + + Set-TimeZone -Name "Pacific Standard Time" + +true diff --git a/deploy/ecs.tf b/deploy/ecs.tf new file mode 100644 index 0000000..a22d5ca --- /dev/null +++ b/deploy/ecs.tf @@ -0,0 +1,56 @@ +resource "aws_ecs_capacity_provider" "e3s_linux" { + name = local.e3s_linux_capacityprovider + auto_scaling_group_provider { + auto_scaling_group_arn = aws_autoscaling_group.linux.arn + managed_termination_protection = "DISABLED" + managed_scaling { + status = "ENABLED" + target_capacity = 100 + minimum_scaling_step_size = 1 + maximum_scaling_step_size = 100 + instance_warmup_period = 300 + } + } +} + +resource "aws_ecs_capacity_provider" "e3s_windows" { + name = local.e3s_windows_capacityprovider + auto_scaling_group_provider { + auto_scaling_group_arn = aws_autoscaling_group.windows.arn + managed_termination_protection = "DISABLED" + managed_scaling { + status = "ENABLED" + target_capacity = 100 + minimum_scaling_step_size = 1 + maximum_scaling_step_size = 100 + instance_warmup_period = 300 + } + } +} + +resource "aws_ecs_cluster" "e3s" { + name = local.e3s_cluster_name +} + +resource "aws_ecs_cluster_capacity_providers" "attachment" { + cluster_name = aws_ecs_cluster.e3s.name + capacity_providers = [aws_ecs_capacity_provider.e3s_linux.name, aws_ecs_capacity_provider.e3s_windows.name] + default_capacity_provider_strategy { + weight = 1 + capacity_provider = aws_ecs_capacity_provider.e3s_linux.name + } + + provisioner "local-exec" { + command = <<-EOF +ecsPolicy=`aws ecs describe-clusters --region ${var.region} --cluster ${aws_ecs_cluster.e3s.name} --include ATTACHMENTS --query 'clusters[0].attachments[].details[]' | grep "${aws_ecs_capacity_provider.e3s_windows.name}" -A 4 | grep "ECSManagedAutoScalingPolicy" | cut -d ':' -f 2 | cut -d '"' -f 2` +aws autoscaling put-scaling-policy --region ${var.region} --auto-scaling-group-name ${aws_autoscaling_group.windows.name} --policy-name "$ecsPolicy" --policy-type TargetTrackingScaling --target-tracking-configuration "{ \"CustomizedMetricSpecification\": { \"MetricName\": \"CapacityProviderReservation\", \"Namespace\": \"AWS/ECS/ManagedScaling\", \"Dimensions\": [{ \"Name\": \"CapacityProviderName\", \"Value\": \"${aws_ecs_capacity_provider.e3s_windows.name}\" }, { \"Name\": \"ClusterName\", \"Value\": \"${aws_ecs_cluster.e3s.name}\"}], \"Statistic\": \"Average\"}, \"TargetValue\": 100.0, \"DisableScaleIn\": false }" --no-enabled +EOF + } + + provisioner "local-exec" { + command = <<-EOF +ecsPolicy2=`aws ecs describe-clusters --region ${var.region} --cluster ${aws_ecs_cluster.e3s.name} --include ATTACHMENTS --query 'clusters[0].attachments[].details[]' | grep "${aws_ecs_capacity_provider.e3s_linux.name}" -A 4 | grep "ECSManagedAutoScalingPolicy" | cut -d ':' -f 2 | cut -d '"' -f 2` +aws autoscaling put-scaling-policy --region ${var.region} --auto-scaling-group-name ${aws_autoscaling_group.linux.name} --policy-name "$ecsPolicy2" --policy-type TargetTrackingScaling --target-tracking-configuration "{ \"CustomizedMetricSpecification\": { \"MetricName\": \"CapacityProviderReservation\", \"Namespace\": \"AWS/ECS/ManagedScaling\", \"Dimensions\": [{ \"Name\": \"CapacityProviderName\", \"Value\": \"${aws_ecs_capacity_provider.e3s_linux.name}\" }, { \"Name\": \"ClusterName\", \"Value\": \"${aws_ecs_cluster.e3s.name}\"}], \"Statistic\": \"Average\"}, \"TargetValue\": 100.0, \"DisableScaleIn\": false }" --no-enabled +EOF + } +} diff --git a/deploy/elbv2.tf b/deploy/elbv2.tf new file mode 100644 index 0000000..606f437 --- /dev/null +++ b/deploy/elbv2.tf @@ -0,0 +1,58 @@ +resource "aws_lb_target_group" "main" { + name = local.e3s_tg_name + vpc_id = aws_vpc.main.id + protocol = "HTTP" + protocol_version = "HTTP1" + port = 4444 + target_type = "instance" + + health_check { + protocol = "HTTP" + port = "traffic-port" + enabled = "true" + path = "/" + interval = 30 + timeout = 5 + healthy_threshold = 5 + unhealthy_threshold = 5 + matcher = 200 + } + + deregistration_delay = 660 +} + +resource "aws_lb" "main" { + name = local.e3s_alb_name + subnets = [for subnet in aws_subnet.public_per_zone : subnet.id] + security_groups = [aws_security_group.e3s_server.id] + load_balancer_type = "application" + ip_address_type = "ipv4" + internal = false + idle_timeout = 660 +} + +resource "aws_lb_listener" "main" { + load_balancer_arn = aws_lb.main.arn + + # Be aware of bug (terrafom is unable to flush ssl_policy field on http switch): + # https://github.com/hashicorp/terraform-provider-aws/issues/1851 + port = var.cert == "" ? 80 : 443 + protocol = var.cert == "" ? "HTTP" : "HTTPS" + ssl_policy = var.cert == "" ? "" : "ELBSecurityPolicy-2016-08" + certificate_arn = var.cert + + default_action { + type = "forward" + order = 1 + forward { + target_group { + arn = aws_lb_target_group.main.arn + weight = 1 + } + stickiness { + enabled = false + duration = 3600 + } + } + } +} diff --git a/deploy/iam.tf b/deploy/iam.tf new file mode 100644 index 0000000..7b8293f --- /dev/null +++ b/deploy/iam.tf @@ -0,0 +1,77 @@ +data "aws_caller_identity" "current" {} + +resource "aws_iam_policy" "e3s" { + name = local.e3s_policy_name + policy = templatefile("./iam_data/e3s-policy.json", { + bucket_name = var.bucket.name + env = var.environment + account = data.aws_caller_identity.current.account_id + region = var.region + }) +} + +resource "aws_iam_policy" "e3s_agent" { + name = local.e3s_agent_policy_name + policy = templatefile("./iam_data/e3s-agent-policy.json", { + env = var.environment + accout = data.aws_caller_identity.current.account_id + region = var.region + }) +} + +resource "aws_iam_policy" "e3s_task" { + name = local.e3s_task_policy_name + policy = templatefile("./iam_data/e3s-task-policy.json", { + bucket_name = var.bucket.name + }) +} + +data "aws_iam_policy_document" "ecs_assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "instance_assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "e3s_task" { + name = local.e3s_task_role_name + managed_policy_arns = [aws_iam_policy.e3s_task.arn] + assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json +} + +resource "aws_iam_role" "e3s" { + name = local.e3s_role_name + managed_policy_arns = [aws_iam_policy.e3s.arn] + assume_role_policy = data.aws_iam_policy_document.instance_assume_role_policy.json +} + +resource "aws_iam_role" "e3s_agent" { + name = local.e3s_agent_role_name + managed_policy_arns = [aws_iam_policy.e3s_agent.arn] + assume_role_policy = data.aws_iam_policy_document.instance_assume_role_policy.json +} + +resource "aws_iam_instance_profile" "e3s" { + name = local.e3s_role_name + role = aws_iam_role.e3s.name +} + +resource "aws_iam_instance_profile" "e3s_agent" { + name = local.e3s_agent_role_name + role = aws_iam_role.e3s_agent.name +} diff --git a/deploy/iam_data/cloudwatch-endpoint-policy.json b/deploy/iam_data/cloudwatch-endpoint-policy.json new file mode 100644 index 0000000..d771a83 --- /dev/null +++ b/deploy/iam_data/cloudwatch-endpoint-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "PutOnly", + "Principal": "*", + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": "*" + } + ] +} diff --git a/deploy/iam_data/e3s-agent-policy.json b/deploy/iam_data/e3s-agent-policy.json new file mode 100644 index 0000000..a97c5d2 --- /dev/null +++ b/deploy/iam_data/e3s-agent-policy.json @@ -0,0 +1,35 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "WithoutConstraints", + "Effect": "Allow", + "Action": [ + "ecs:DiscoverPollEndpoint", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage" + ], + "Resource": "*" + }, + { + "Sid": "ECS", + "Effect": "Allow", + "Action": [ + "ecs:DeregisterContainerInstance", + "ecs:RegisterContainerInstance", + "ecs:Submit*", + "ecs:StartTelemetrySession", + "ecs:UpdateContainerInstancesState", + "ecs:Poll" + ], + "Resource": [ + "arn:aws:ecs:${region}:${accout}:cluster/e3s-${env}", + "arn:aws:ecs:${region}:${accout}:container-instance/e3s-${env}/*" + ] + } + ] +} diff --git a/deploy/iam_data/e3s-policy.json b/deploy/iam_data/e3s-policy.json new file mode 100644 index 0000000..112f436 --- /dev/null +++ b/deploy/iam_data/e3s-policy.json @@ -0,0 +1,82 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "WithoutConstraints", + "Effect": "Allow", + "Action": [ + "ecs:RegisterTaskDefinition", + "ecs:ListTasks", + "ec2:DescribeInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeInstanceTypes", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeListeners", + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances" + ], + "Resource": "*" + }, + { + "Sid": "ECS", + "Effect": "Allow", + "Action": [ + "ecs:DescribeContainerInstances", + "ecs:DescribeTasks", + "ecs:StopTask", + "ecs:DescribeClusters", + "ecs:ListContainerInstances", + "ecs:RunTask", + "ecs:DescribeCapacityProviders", + "ecs:UpdateContainerInstancesState" + ], + "Resource": [ + "arn:aws:ecs:${region}:${account}:container-instance/e3s-${env}/*", + "arn:aws:ecs:${region}:${account}:task/e3s-${env}/*", + "arn:aws:ecs:${region}:${account}:cluster/e3s-${env}", + "arn:aws:ecs:${region}:${account}:task-definition/${env}-*", + "arn:aws:ecs:${region}:${account}:capacity-provider/e3s-${env}-*" + ] + }, + { + "Sid": "Autoscaling", + "Effect": "Allow", + "Action": [ + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:SetInstanceProtection", + "autoscaling:SetDesiredCapacity" + ], + "Resource": "arn:aws:autoscaling:${region}:${account}:autoScalingGroup:*:autoScalingGroupName/e3s-${env}-*" + }, + { + "Sid": "ELB", + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets" + ], + "Resource": "arn:aws:elasticloadbalancing:${region}:${account}:targetgroup/e3s-${env}-*" + }, + { + "Sid": "S3", + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::${bucket_name}", + "arn:aws:s3:::${bucket_name}/*" + ] + }, + { + "Sid": "IAM", + "Effect": "Allow", + "Action": [ + "iam:passRole" + ], + "Resource": "arn:aws:iam::${account}:role/e3s-${env}-task-role" + } + ] +} diff --git a/deploy/iam_data/e3s-task-policy.json b/deploy/iam_data/e3s-task-policy.json new file mode 100644 index 0000000..fcee106 --- /dev/null +++ b/deploy/iam_data/e3s-task-policy.json @@ -0,0 +1,14 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::${bucket_name}/*" + ] + } + ] +} diff --git a/deploy/iam_data/s3-bucket-policy.json b/deploy/iam_data/s3-bucket-policy.json new file mode 100644 index 0000000..88194e2 --- /dev/null +++ b/deploy/iam_data/s3-bucket-policy.json @@ -0,0 +1,22 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Allow-put-only-for-specific-vpc_endoint", + "Principal": "*", + "Effect": "Deny", + "Action": [ + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::${bucket_name}", + "arn:aws:s3:::${bucket_name}/*" + ], + "Condition": { + "StringNotEquals": { + "aws:sourceVpce": "${vpc_endpoint_id}" + } + } + } + ] +} \ No newline at end of file diff --git a/deploy/iam_data/s3-endpoint-policy.json b/deploy/iam_data/s3-endpoint-policy.json new file mode 100644 index 0000000..df9b221 --- /dev/null +++ b/deploy/iam_data/s3-endpoint-policy.json @@ -0,0 +1,29 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::${bucket_name}", + "arn:aws:s3:::${bucket_name}/*" + ] + }, + { + "Sid": "yum's repo file stored at s3", + "Effect": "Allow", + "Principal": "*", + "Action": "*", + "Resource": [ + "arn:aws:s3:::repo.${region}.amazonaws.com", + "arn:aws:s3:::repo.${region}.amazonaws.com/*", + "arn:aws:s3:::amazonlinux-2-repos-${region}/*" + ] + } + ] +} \ No newline at end of file diff --git a/deploy/outputs.tf b/deploy/outputs.tf new file mode 100644 index 0000000..0001262 --- /dev/null +++ b/deploy/outputs.tf @@ -0,0 +1,44 @@ +output "e3s_ip" { + description = "public adress of e3s server" + value = aws_instance.e3s_server.public_ip +} + +output "nat_gw_ip" { + description = "adress of nat gateway" + value = aws_nat_gateway.nat-gw.public_ip +} + +output "lb_dns" { + description = "load balancer dns" + value = aws_lb.main.dns_name +} + +output "vpc_id" { + description = "new vpc" + value = aws_vpc.main.id +} + +# output "db_dns" { +# description = "aurora dns" +# value = aws_rds_cluster.aurora.endpoint +# } + +output "db_dns" { + description = "postgres dns" + value = length(aws_db_instance.postgres) != 0 ? aws_db_instance.postgres[0].endpoint : "remote db is not created" +} + +output "cache_address" { + description = "redis read/write host:port" + value = length(aws_elasticache_serverless_cache.redis) != 0 ? format("%s:%s", aws_elasticache_serverless_cache.redis[0].endpoint[0].address, aws_elasticache_serverless_cache.redis[0].endpoint[0].port) : "remote redis is not created" +} + +output "cloudwatch_vpc_endpoint_id" { + description = "vpc interface endpoint for cloudwatch logs upload" + value = length(aws_vpc_endpoint.cloudwatch) != 0 ? aws_vpc_endpoint.cloudwatch[0].id : "cloudwatch endpoint is not created" +} + +output "s3_vpc_gw_endpoint_id" { + description = "vpc gateway endpoint for s3 artifacts upload" + value = aws_vpc_endpoint.s3_gw.id +} diff --git a/deploy/provider.tf b/deploy/provider.tf new file mode 100644 index 0000000..6f29fcd --- /dev/null +++ b/deploy/provider.tf @@ -0,0 +1,31 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.54.1" + } + + random = { + source = "hashicorp/random" + version = "3.6.2" + } + + tls = { + source = "hashicorp/tls" + version = "4.0.5" + } + } + + required_version = "~> 1.8.5" +} + +provider "aws" { + region = var.region + + default_tags { + tags = { + Environment = var.environment + Name = local.service_name + } + } +} diff --git a/deploy/rds.tf b/deploy/rds.tf new file mode 100644 index 0000000..587c34e --- /dev/null +++ b/deploy/rds.tf @@ -0,0 +1,64 @@ +resource "aws_db_subnet_group" "rds" { + count = var.data_layer_remote ? 1 : 0 + name = local.e3s_rds_subnet_name + description = "RDS subnet group" + subnet_ids = [for subnet in aws_subnet.private_per_zone : subnet.id] +} + +resource "aws_db_instance" "postgres" { + count = var.data_layer_remote ? 1 : 0 + db_name = local.e3s_rds_db_name + allocated_storage = 10 + max_allocated_storage = 30 + instance_class = "db.t4g.small" + engine = "postgres" + engine_version = "13.13" + username = var.remote_db.username + password = var.remote_db.pass + auto_minor_version_upgrade = true + db_subnet_group_name = aws_db_subnet_group.rds[0].name + port = 5432 + + maintenance_window = "Mon:00:00-Mon:01:00" + backup_window = "01:01-02:00" + backup_retention_period = 2 + storage_encrypted = true + + deletion_protection = false + apply_immediately = false + skip_final_snapshot = true + delete_automated_backups = true + + vpc_security_group_ids = [aws_security_group.rds[0].id] +} + + +# resource "aws_rds_cluster" "aurora" { +# engine = "aurora-postgresql" +# storage_type = "aurora-iopt1" +# port = 5432 +# db_subnet_group_name = aws_db_subnet_group.rds.name +# master_username = var.remote_db.username +# master_password = var.remote_db.pass +# preferred_maintenance_window = "Mon:00:00-Mon:01:00" +# preferred_backup_window = "01:01-02:00" +# backup_retention_period = 2 +# storage_encrypted = true + +# deletion_protection = false +# apply_immediately = true +# skip_final_snapshot = true +# delete_automated_backups = true + +# vpc_security_group_ids = [aws_security_group.rds.id] +# } + +# resource "aws_rds_cluster_instance" "aurora_instance" { +# cluster_identifier = aws_rds_cluster.aurora.id +# // aurora doesn't support db.t4g.small +# instance_class = "db.t4g.medium" +# engine = aws_rds_cluster.aurora.engine +# engine_version = aws_rds_cluster.aurora.engine_version +# db_subnet_group_name = aws_db_subnet_group.rds.name +# auto_minor_version_upgrade = true +# } diff --git a/deploy/redis.tf b/deploy/redis.tf new file mode 100644 index 0000000..fd74908 --- /dev/null +++ b/deploy/redis.tf @@ -0,0 +1,45 @@ +locals { + az_names = sort([for az_name in data.aws_availability_zones.available.names : az_name]) +} + +# restrict subnets to first 2 in lexicographical order regions (example: us-east-1a, us-east-1b) +data "aws_subnets" "redis" { + count = var.data_layer_remote ? 1 : 0 + filter { + name = "vpc-id" + values = [aws_vpc.main.id] + } + + filter { + name = "availability-zone" + values = [local.az_names[0], local.az_names[1]] + } + + filter { + name = "subnet-id" + values = [for subnet in aws_subnet.private_per_zone : subnet.id] + } + + depends_on = [aws_subnet.private_per_zone] +} + +resource "aws_elasticache_serverless_cache" "redis" { + count = var.data_layer_remote ? 1 : 0 + name = local.e3s_serverless_cache_name + engine = "redis" + major_engine_version = "7" + + cache_usage_limits { + data_storage { + maximum = 5 + unit = "GB" + } + ecpu_per_second { + maximum = 5000 + } + } + + subnet_ids = data.aws_subnets.redis[0].ids + security_group_ids = [aws_security_group.redis[0].id] +} + diff --git a/deploy/s3.tf b/deploy/s3.tf new file mode 100644 index 0000000..af14942 --- /dev/null +++ b/deploy/s3.tf @@ -0,0 +1,26 @@ +resource "aws_s3_bucket" "main" { + count = var.bucket.exists ? 0 : 1 + bucket = var.bucket.name + force_destroy = true +} + +resource "aws_s3_bucket_policy" "vpc_restrict_policy" { + count = var.bucket.exists ? 0 : 1 + bucket = aws_s3_bucket.main[0].id + policy = templatefile("./iam_data/s3-bucket-policy.json", { + bucket_name = var.bucket.name + vpc_endpoint_id = aws_vpc_endpoint.s3_gw.id + }) +} + +resource "aws_vpc_endpoint" "s3_gw" { + vpc_id = aws_vpc.main.id + + route_table_ids = [aws_route_table.internet-private.id] + service_name = format("com.amazonaws.%s.s3", var.region) + vpc_endpoint_type = "Gateway" + policy = templatefile("./iam_data/s3-endpoint-policy.json", { + bucket_name = var.bucket.name + region = var.region + }) +} diff --git a/deploy/security_groups.tf b/deploy/security_groups.tf new file mode 100644 index 0000000..e696e88 --- /dev/null +++ b/deploy/security_groups.tf @@ -0,0 +1,201 @@ +resource "aws_security_group" "e3s_server" { + vpc_id = aws_vpc.main.id + name = local.e3s_server_sg_name +} + +resource "aws_vpc_security_group_ingress_rule" "e3s_server_alb" { + security_group_id = aws_security_group.e3s_server.id + ip_protocol = "tcp" + cidr_ipv4 = "0.0.0.0/0" + from_port = var.cert == "" ? 80 : 443 + to_port = var.cert == "" ? 80 : 443 +} + +resource "aws_vpc_security_group_ingress_rule" "e3s_server_router_ports" { + security_group_id = aws_security_group.e3s_server.id + ip_protocol = "tcp" + cidr_ipv4 = "0.0.0.0/0" + from_port = 4444 + to_port = 4445 + description = "router_ports" +} + +resource "aws_vpc_security_group_ingress_rule" "e3s_server_ssh_ipv4" { + security_group_id = aws_security_group.e3s_server.id + ip_protocol = "tcp" + cidr_ipv4 = "0.0.0.0/0" + from_port = 22 + to_port = 22 + description = "ssh" +} + +resource "aws_vpc_security_group_ingress_rule" "e3s_server_ssh_ipv6" { + security_group_id = aws_security_group.e3s_server.id + ip_protocol = "tcp" + cidr_ipv6 = "::/0" + from_port = 22 + to_port = 22 + description = "ssh" +} + +resource "aws_vpc_security_group_egress_rule" "e3s_server_outbound_trafic_ipv4" { + security_group_id = aws_security_group.e3s_server.id + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_egress_rule" "e3s_server_outbound_trafic_ipv6" { + security_group_id = aws_security_group.e3s_server.id + ip_protocol = "-1" + cidr_ipv6 = "::/0" +} + +resource "aws_security_group" "e3s_agent" { + vpc_id = aws_vpc.main.id + name = local.e3s_agent_sg_name +} + +resource "aws_vpc_security_group_ingress_rule" "e3s_agent_inbound_trafic" { + security_group_id = aws_security_group.e3s_agent.id + ip_protocol = "tcp" + cidr_ipv4 = "${aws_instance.e3s_server.private_ip}/32" + description = "docker port range to access from e3s server" + from_port = 32768 + to_port = 64536 +} + +resource "aws_vpc_security_group_ingress_rule" "e3s_agent_ssh_ipv4" { + count = var.allow_agent_ssh ? 1 : 0 + security_group_id = aws_security_group.e3s_agent.id + ip_protocol = "tcp" + cidr_ipv4 = "${aws_instance.e3s_server.private_ip}/32" + from_port = 22 + to_port = 22 + description = "ssh" +} + +resource "aws_vpc_security_group_egress_rule" "e3s_agent_outbound_trafic_ipv4" { + security_group_id = aws_security_group.e3s_agent.id + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_egress_rule" "e3s_agent_outbound_trafic_ipv6" { + security_group_id = aws_security_group.e3s_agent.id + ip_protocol = "-1" + cidr_ipv6 = "::/0" +} + +resource "aws_security_group" "windows_rdp" { + count = var.allow_agent_ssh ? 1 : 0 + vpc_id = aws_vpc.main.id + name = local.e3s_rdp_sg_name +} + +resource "aws_vpc_security_group_ingress_rule" "e3s_rdp_ipv4" { + count = length(aws_security_group.windows_rdp) + security_group_id = aws_security_group.windows_rdp[count.index].id + ip_protocol = "tcp" + cidr_ipv4 = "${aws_instance.e3s_server.private_ip}/32" + from_port = 3389 + to_port = 3389 +} + +resource "aws_vpc_security_group_egress_rule" "e3s_rdp_outbound_trafic_ipv4" { + count = length(aws_security_group.windows_rdp) + security_group_id = aws_security_group.windows_rdp[count.index].id + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_egress_rule" "e3s_rdp_outbound_trafic_ipv6" { + count = length(aws_security_group.windows_rdp) + security_group_id = aws_security_group.windows_rdp[count.index].id + ip_protocol = "-1" + cidr_ipv6 = "::/0" +} + +resource "aws_security_group" "rds" { + count = var.data_layer_remote ? 1 : 0 + vpc_id = aws_vpc.main.id + name = local.e3s_rds_sg_name +} + +resource "aws_vpc_security_group_egress_rule" "e3s_rds_outbound_trafic_ipv4" { + count = var.data_layer_remote ? 1 : 0 + security_group_id = aws_security_group.rds[0].id + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_egress_rule" "e3s_rds_outbound_trafic_ipv6" { + count = var.data_layer_remote ? 1 : 0 + security_group_id = aws_security_group.rds[0].id + ip_protocol = "-1" + cidr_ipv6 = "::/0" +} + +resource "aws_vpc_security_group_ingress_rule" "e3s_rds_ipv4" { + count = var.data_layer_remote ? 1 : 0 + security_group_id = aws_security_group.rds[0].id + ip_protocol = "tcp" + cidr_ipv4 = "${aws_instance.e3s_server.private_ip}/32" + from_port = 5432 + to_port = 5432 +} + +resource "aws_security_group" "redis" { + count = var.data_layer_remote ? 1 : 0 + vpc_id = aws_vpc.main.id + name = local.e3s_cache_sg_name +} + +resource "aws_vpc_security_group_egress_rule" "e3s_redis_outbound_trafic_ipv4" { + count = var.data_layer_remote ? 1 : 0 + security_group_id = aws_security_group.redis[0].id + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_egress_rule" "e3s_redis_outbound_trafic_ipv6" { + count = var.data_layer_remote ? 1 : 0 + security_group_id = aws_security_group.redis[0].id + ip_protocol = "-1" + cidr_ipv6 = "::/0" +} + +resource "aws_vpc_security_group_ingress_rule" "e3s_redis_ipv4" { + count = var.data_layer_remote ? 1 : 0 + security_group_id = aws_security_group.redis[0].id + ip_protocol = "tcp" + cidr_ipv4 = "${aws_instance.e3s_server.private_ip}/32" + from_port = 6379 + to_port = 6380 +} + +resource "aws_security_group" "cloudwatch" { + count = var.enable_cloudwatch ? 1 : 0 + vpc_id = aws_vpc.main.id + name = local.e3s_cloudwatch_endpoint_sg_name +} + +resource "aws_vpc_security_group_ingress_rule" "cloudwatch" { + count = var.enable_cloudwatch ? 1 : 0 + security_group_id = aws_security_group.cloudwatch[0].id + ip_protocol = "-1" + referenced_security_group_id = aws_security_group.e3s_agent.id +} + +resource "aws_vpc_security_group_egress_rule" "cloudwatch_outbound_trafic_ipv4" { + count = var.enable_cloudwatch ? 1 : 0 + security_group_id = aws_security_group.cloudwatch[0].id + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_egress_rule" "cloudwatch_outbound_trafic_ipv6" { + count = var.enable_cloudwatch ? 1 : 0 + security_group_id = aws_security_group.cloudwatch[0].id + ip_protocol = "-1" + cidr_ipv6 = "::/0" +} diff --git a/deploy/vars.tf b/deploy/vars.tf new file mode 100644 index 0000000..b599c67 --- /dev/null +++ b/deploy/vars.tf @@ -0,0 +1,142 @@ +# TODO: Add custom condition for vars +# Mandatory +variable "environment" { + type = string + nullable = false +} + +variable "region" { + type = string + nullable = false +} + +variable "e3s_key_name" { + type = string + nullable = false +} + +variable "bucket" { + type = object({ + exists = bool + name = string + region = string + }) + nullable = false +} + +# Optional +variable "allow_agent_ssh" { + type = bool + default = false +} + +variable "cert" { + type = string + default = "" +} + +variable "enable_cloudwatch" { + type = bool + default = false +} + +variable "e3s_server_instance_type" { + type = string + default = "m5n.large" +} + +variable "data_layer_remote" { + type = bool + default = true +} + +variable "remote_db" { + type = object({ + username = string + pass = string + }) + default = { + username = "postgres" + pass = "postgres" + } +} + +variable "instance_types" { + type = list(object({ + weight = number + instance_type = string + })) + default = [ + { + weight = 1 + instance_type = "c5a.4xlarge" + }, + { + weight = 2 + instance_type = "c5a.8xlarge" + } + ] +} + +variable "spot_price" { + type = object({ + linux = string + windows = string + }) + default = { + linux = "" + windows = "" + } +} + +variable "zebrunner" { + type = object({ + host = string + user = string + pass = string + }) + default = { + host = "" + user = "" + pass = "" + } +} + +# consts +locals { + service_name = "e3s" + + e3s_server_instance_name = join("-", [local.service_name, var.environment]) + + e3s_agent_key_name = join("-", [local.service_name, var.environment, "agent"]) + + e3s_policy_name = join("-", [local.service_name, var.environment, "policy"]) + e3s_role_name = join("-", [local.service_name, var.environment, "role"]) + e3s_agent_policy_name = join("-", [local.service_name, var.environment, "agent", "policy"]) + e3s_agent_role_name = join("-", [local.service_name, var.environment, "agent", "role"]) + e3s_task_policy_name = join("-", [local.service_name, var.environment, "task", "policy"]) + e3s_task_role_name = join("-", [local.service_name, var.environment, "task", "role"]) + + e3s_server_sg_name = join("-", [local.service_name, var.environment, "sg"]) + e3s_agent_sg_name = join("-", [local.service_name, var.environment, "agent", "sg"]) + e3s_rdp_sg_name = join("-", [local.service_name, var.environment, "rdp", "sg"]) + e3s_rds_sg_name = join("-", [local.service_name, var.environment, "rds", "sg"]) + e3s_cache_sg_name = join("-", [local.service_name, var.environment, "cache", "sg"]) + e3s_cloudwatch_endpoint_sg_name = join("-", [local.service_name, var.environment, "cloudwatch", "sg"]) + + e3s_cluster_name = join("-", [local.service_name, var.environment]) + e3s_linux_launch_template_name = join("-", [local.service_name, var.environment, "linux", "launch", "template"]) + e3s_windows_launch_template_name = join("-", [local.service_name, var.environment, "windows", "launch", "template"]) + e3s_linux_autoscaling_name = join("-", [local.service_name, var.environment, "linux", "asg"]) + e3s_windows_autoscaling_name = join("-", [local.service_name, var.environment, "windows", "asg"]) + e3s_linux_capacityprovider = join("-", [local.service_name, var.environment, "linux", "capacityprovider"]) + e3s_windows_capacityprovider = join("-", [local.service_name, var.environment, "windows", "capacityprovider"]) + e3s_tg_name = join("-", [local.service_name, var.environment, "tg"]) + e3s_alb_name = join("-", [local.service_name, var.environment, "alb"]) + e3s_listener_name = join("-", [local.service_name, var.environment, "listener"]) + e3s_log_group_name = join("-", [local.service_name, var.environment, "log-group"]) + + e3s_rds_subnet_name = join("-", [local.service_name, var.environment, "rds", "subnet"]) + e3s_rds_db_name = join("_", [local.service_name, var.environment, "postgres"]) + e3s_serverless_cache_name = join("-", [local.service_name, var.environment, "redis"]) +} diff --git a/deploy/vpc.tf b/deploy/vpc.tf new file mode 100644 index 0000000..d48026a --- /dev/null +++ b/deploy/vpc.tf @@ -0,0 +1,77 @@ +resource "aws_vpc" "main" { + cidr_block = "10.0.0.0/16" + instance_tenancy = "default" + enable_dns_support = "true" + enable_dns_hostnames = "true" +} + +data "aws_availability_zones" "available" { + state = "available" +} + +resource "aws_subnet" "public_per_zone" { + for_each = { for id, az_name in data.aws_availability_zones.available.names : id => az_name } + + vpc_id = aws_vpc.main.id + map_public_ip_on_launch = true + availability_zone = each.value + cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 8, each.key) +} + +resource "aws_internet_gateway" "igw" { + vpc_id = aws_vpc.main.id +} + +resource "aws_route_table" "internet" { + vpc_id = aws_vpc.main.id + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.igw.id + } +} + +resource "aws_route_table_association" "internet-associations" { + for_each = (tomap(aws_subnet.public_per_zone)) + route_table_id = aws_route_table.internet.id + subnet_id = aws_subnet.public_per_zone[each.key].id +} + +resource "random_shuffle" "public_subnet" { + input = [for subnet in aws_subnet.public_per_zone : subnet.id] + result_count = 1 +} + +resource "aws_subnet" "private_per_zone" { + for_each = { for id, az_name in data.aws_availability_zones.available.names : id => az_name } + + vpc_id = aws_vpc.main.id + map_public_ip_on_launch = false + availability_zone = each.value + cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 8, length(aws_subnet.public_per_zone) + each.key) +} + +resource "aws_route_table" "internet-private" { + vpc_id = aws_vpc.main.id + route { + cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.nat-gw.id + } +} + +resource "aws_route_table_association" "nat-associations" { + for_each = (tomap(aws_subnet.private_per_zone)) + route_table_id = aws_route_table.internet-private.id + subnet_id = aws_subnet.private_per_zone[each.key].id +} + +resource "aws_eip" "nat" { + domain = "vpc" + network_border_group = var.region +} + +resource "aws_nat_gateway" "nat-gw" { + allocation_id = aws_eip.nat.id + subnet_id = random_shuffle.public_subnet.result[0] + + depends_on = [aws_internet_gateway.igw] +}