diff --git a/.gitignore b/.gitignore index b81d689..c2b4840 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -.terraform -.terraform.lock.hcl -terraform.tfstate.backup \ No newline at end of file +**/.terraform +**/.terraform.lock.hcl +**/terraform.tfstate.backup \ No newline at end of file diff --git a/ecr.tf b/cluster/ecr.tf similarity index 100% rename from ecr.tf rename to cluster/ecr.tf diff --git a/eip.tf b/cluster/eip.tf similarity index 100% rename from eip.tf rename to cluster/eip.tf diff --git a/eks.tf b/cluster/eks.tf similarity index 88% rename from eks.tf rename to cluster/eks.tf index 639c376..fb2b10d 100644 --- a/eks.tf +++ b/cluster/eks.tf @@ -38,8 +38,8 @@ module "external_dns_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "~> 5.0" - role_name = "${var.environment}-radar-base-external-dns-irsa" - attach_external_dns_policy = true + role_name = "${var.environment}-radar-base-external-dns-irsa" + attach_external_dns_policy = true external_dns_hosted_zone_arns = ["arn:aws:route53:::hostedzone/${aws_route53_zone.primary.id}"] oidc_providers = { @@ -56,8 +56,8 @@ module "cert_manager_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "~> 5.0" - role_name = "${var.environment}-radar-base-cert-manager-irsa" - attach_cert_manager_policy = true + role_name = "${var.environment}-radar-base-cert-manager-irsa" + attach_cert_manager_policy = true cert_manager_hosted_zone_arns = ["arn:aws:route53:::hostedzone/${aws_route53_zone.primary.id}"] oidc_providers = { @@ -70,20 +70,15 @@ module "cert_manager_irsa" { tags = merge(tomap({ "Name" : "${var.environment}-radar-base-cert-manager-irsa" }), var.common_tags) } -module "karpenter" { - source = "terraform-aws-modules/eks/aws//modules/karpenter" - version = "19.17.2" - - cluster_name = module.eks.cluster_name - - irsa_oidc_provider_arn = module.eks.oidc_provider_arn - irsa_namespace_service_accounts = ["karpenter:karpenter"] +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - iam_role_additional_policies = { - AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", var.AWS_REGION] + command = "aws" } - - tags = merge(tomap({ "Name" : "${var.environment}-radar-base-karpenter" }), var.common_tags) } module "eks" { @@ -231,31 +226,12 @@ module "eks" { username = module.eks_admins_iam_role.iam_role_name groups = ["system:masters"] }, - { - rolearn = module.karpenter.role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - }, ] tags = merge(tomap({ "Name" : "${var.environment}-${var.eks_cluster_base_name}" }), var.common_tags) } -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - command = "aws" - } -} - output "radar_base_eks_cluster_name" { value = module.eks.cluster_name } @@ -271,3 +247,15 @@ output "radar_base_eks_dmz_node_group_name" { output "radar_base_eks_worker_node_group_name" { value = element(split(":", module.eks.eks_managed_node_groups.worker.node_group_id), 1) } + +# output "radar_base_eks_karpenter_irsa_arn" { +# value = module.karpenter.irsa_arn +# } + +# output "radar_base_eks_karpenter_interruption_queue_name" { +# value = module.karpenter.queue_name +# } + +# output "radar_base_eks_karpenter_instance_profile" { +# value = module.karpenter.instance_profile_name +# } \ No newline at end of file diff --git a/iam.tf b/cluster/iam.tf similarity index 100% rename from iam.tf rename to cluster/iam.tf diff --git a/msk.tf b/cluster/msk.tf similarity index 100% rename from msk.tf rename to cluster/msk.tf diff --git a/cluster/provider.tf b/cluster/provider.tf new file mode 100644 index 0000000..001eedd --- /dev/null +++ b/cluster/provider.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "< 5.0.0" + } + postgresql = { + source = "cyrilgdn/postgresql" + version = ">= 1.19.0" + } + } + required_version = "~> 1.0" +} + +provider "aws" { + region = var.AWS_REGION + access_key = var.AWS_ACCESS_KEY_ID + secret_key = var.AWS_SECRET_ACCESS_KEY + token = var.AWS_SESSION_TOKEN +} diff --git a/rds.tf b/cluster/rds.tf similarity index 100% rename from rds.tf rename to cluster/rds.tf diff --git a/route53.tf b/cluster/route53.tf similarity index 100% rename from route53.tf rename to cluster/route53.tf diff --git a/s3.tf b/cluster/s3.tf similarity index 100% rename from s3.tf rename to cluster/s3.tf diff --git a/ses.tf b/cluster/ses.tf similarity index 100% rename from ses.tf rename to cluster/ses.tf diff --git a/variables.tf b/cluster/variables.tf similarity index 100% rename from variables.tf rename to cluster/variables.tf diff --git a/vpc.tf b/cluster/vpc.tf similarity index 100% rename from vpc.tf rename to cluster/vpc.tf diff --git a/config/karpenter.tf b/config/karpenter.tf new file mode 100644 index 0000000..6ce3ff9 --- /dev/null +++ b/config/karpenter.tf @@ -0,0 +1,160 @@ +data "aws_eks_cluster" "main" { + name = var.cluster_name +} + +data "aws_eks_cluster_auth" "main" { + name = var.cluster_name +} + +data "aws_autoscaling_groups" "main" { + filter { + name = "tag:eks:cluster-name" + values = [var.cluster_name] + } +} + +data "aws_eks_node_group" "worker" { + cluster_name = var.cluster_name + node_group_name = join("-", [ + element(split("-", [for asg in data.aws_autoscaling_groups.main.names : asg if startswith(asg, "eks-worker-")][0]), 1), + element(split("-", [for asg in data.aws_autoscaling_groups.main.names : asg if startswith(asg, "eks-worker-")][0]), 2) + ]) +} + +locals { + aws_account = element(split(":", data.aws_eks_cluster.main.arn), 4) + oidc_issuer = element(split("//", data.aws_eks_cluster.main.identity[0].oidc[0].issuer), 1) +} + +provider "kubernetes" { + host = data.aws_eks_cluster.main.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.main.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.main.token +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.main.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.main.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.main.token + } +} + +provider "kubectl" { + apply_retry_count = 5 + host = data.aws_eks_cluster.main.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.main.certificate_authority[0].data) + load_config_file = false + token = data.aws_eks_cluster_auth.main.token +} + +module "karpenter" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" + version = "19.17.2" + + cluster_name = data.aws_eks_cluster.main.id + + irsa_oidc_provider_arn = join("", ["arn:aws:iam::", local.aws_account, ":oidc-provider/", local.oidc_issuer]) + irsa_namespace_service_accounts = ["karpenter:karpenter"] + + create_iam_role = false + iam_role_arn = data.aws_eks_node_group.worker.node_role_arn + + tags = merge(tomap({ "Name" : "${var.environment}-radar-base-karpenter" }), var.common_tags) +} + +resource "helm_release" "karpenter" { + namespace = "karpenter" + create_namespace = true + + name = "karpenter" + repository = "oci://public.ecr.aws/karpenter" + chart = "karpenter" + version = "v0.29.0" + + set { + name = "settings.aws.clusterName" + value = data.aws_eks_cluster.main.id + } + + set { + name = "settings.aws.clusterEndpoint" + value = data.aws_eks_cluster.main.endpoint + } + + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.karpenter.irsa_arn + } + + set { + name = "settings.aws.defaultInstanceProfile" + value = module.karpenter.instance_profile_name + } + + set { + name = "settings.aws.interruptionQueueName" + value = module.karpenter.queue_name + } + +} + +resource "kubectl_manifest" "karpenter_provisioner" { + yaml_body = <<-YAML + apiVersion: karpenter.sh/v1alpha5 + kind: Provisioner + metadata: + name: default + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: kubernetes.io/os + operator: In + values: + - linux + - key: karpenter.sh/capacity-type + operator: In + values: + - "${lower(var.instance_capacity_type)}" + - key: topology.kubernetes.io/zone + operator: In + values: + - "${var.AWS_REGION}a" + # - "${var.AWS_REGION}b" + # - "${var.AWS_REGION}c" + ttlSecondsAfterEmpty: 30 + limits: + resources: + cpu: 64 + memory: 256Gi + providerRef: + name: default + YAML + + depends_on = [ + helm_release.karpenter + ] +} + +resource "kubectl_manifest" "karpenter_node_template" { + yaml_body = <<-YAML + apiVersion: karpenter.k8s.aws/v1alpha1 + kind: AWSNodeTemplate + metadata: + name: default + spec: + subnetSelector: + karpenter.sh/discovery: ${data.aws_eks_cluster.main.id} + securityGroupSelector: + karpenter.sh/discovery: ${data.aws_eks_cluster.main.id} + tags: + karpenter.sh/discovery: ${data.aws_eks_cluster.main.id} + YAML + + depends_on = [ + helm_release.karpenter + ] +} diff --git a/provider.tf b/config/provider.tf similarity index 83% rename from provider.tf rename to config/provider.tf index 9188a85..f29b86a 100644 --- a/provider.tf +++ b/config/provider.tf @@ -4,10 +4,6 @@ terraform { source = "hashicorp/aws" version = "< 5.0.0" } - postgresql = { - source = "cyrilgdn/postgresql" - version = ">= 1.19.0" - } helm = { source = "hashicorp/helm" version = ">= 2.11" diff --git a/config/variables.tf b/config/variables.tf new file mode 100644 index 0000000..a252628 --- /dev/null +++ b/config/variables.tf @@ -0,0 +1,59 @@ +variable "AWS_REGION" { + type = string + default = "eu-west-2" +} + +variable "AWS_ACCESS_KEY_ID" { + type = string + sensitive = true +} + +variable "AWS_SECRET_ACCESS_KEY" { + type = string + sensitive = true +} + +variable "AWS_SESSION_TOKEN" { + type = string + default = "" + sensitive = true +} + +variable "environment" { + type = string + default = "dev" +} + +variable "common_tags" { + type = map(string) + default = { + Project = "radar-base-development" + Environment = "dev" + } +} + +variable "cluster_name" { + type = string +} + +variable "instance_capacity_type" { + type = string + default = "SPOT" + + validation { + condition = var.instance_capacity_type == "ON_DEMAND" || var.instance_capacity_type == "SPOT" + error_message = "Invalid instance capacity type. Allowed values are 'ON_DEMAND' or 'SPOT'." + } +} + +# variable "karpenter_irsa_arn" { +# type = string +# } + +# variable "karpenter_instance_profile" { +# type = string +# } + +# variable "karpenter_interruption_queue_name" { +# type = string +# } \ No newline at end of file diff --git a/karpenter.tf b/karpenter.tf deleted file mode 100644 index bb42db7..0000000 --- a/karpenter.tf +++ /dev/null @@ -1,121 +0,0 @@ -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } - } -} - -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - -resource "helm_release" "karpenter" { - namespace = "karpenter" - create_namespace = true - - name = "karpenter" - repository = "oci://public.ecr.aws/karpenter" - chart = "karpenter" - version = "v0.29.0" - - set { - name = "settings.aws.clusterName" - value = module.eks.cluster_name - } - - set { - name = "settings.aws.clusterEndpoint" - value = module.eks.cluster_endpoint - } - - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter.irsa_arn - } - - set { - name = "settings.aws.defaultInstanceProfile" - value = module.karpenter.instance_profile_name - } - - set { - name = "settings.aws.interruptionQueueName" - value = module.karpenter.queue_name - } - -} - -resource "kubectl_manifest" "karpenter_provisioner" { - yaml_body = <<-YAML - apiVersion: karpenter.sh/v1alpha5 - kind: Provisioner - metadata: - name: default - spec: - requirements: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: kubernetes.io/os - operator: In - values: - - linux - - key: karpenter.sh/capacity-type - operator: In - values: - - "${lower(var.instance_capacity_type)}" - - key: topology.kubernetes.io/zone - operator: In - values: - - "${var.AWS_REGION}a" - - "${var.AWS_REGION}b" - - "${var.AWS_REGION}c" - ttlSecondsAfterEmpty: 30 - limits: - resources: - cpu: 64 - memory: 256Gi - providerRef: - name: default - YAML - - depends_on = [ - helm_release.karpenter - ] -} - -resource "kubectl_manifest" "karpenter_node_template" { - yaml_body = <<-YAML - apiVersion: karpenter.k8s.aws/v1alpha1 - kind: AWSNodeTemplate - metadata: - name: default - spec: - subnetSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - securityGroupSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} - YAML - - depends_on = [ - helm_release.karpenter - ] -} \ No newline at end of file