From ccd6006a94eeed5f962eb91f8a7c91382d80171f Mon Sep 17 00:00:00 2001 From: nimalank7 Date: Fri, 29 Nov 2024 17:47:23 +0000 Subject: [PATCH] Add AWS EFS CSI Driver Terraform Description: - Add a statically provisioned EFS CSI Driver as part of a series of PRs to move asset-manager to `nfs` volume type. This is because dynamic provisioning causes errors - see https://trello.com/c/2XkadNJ2/1011-resolve-asset-manager-pvc-related-errors - PR only setups the IAM roles for the EFS CSI Driver. Follow up PR will install the EFS CSI driver. - See https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/ - As part of https://github.com/alphagov/govuk-helm-charts/issues/1883 --- .../cluster-infrastructure/aws_efs_csi_iam.tf | 251 ++++++++++++++++++ .../cluster-infrastructure/outputs.tf | 10 + .../cluster-services/aws_efs_csi_driver.tf | 24 ++ .../outputs.tf | 3 - 4 files changed, 285 insertions(+), 3 deletions(-) create mode 100644 terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf create mode 100644 terraform/deployments/cluster-services/aws_efs_csi_driver.tf delete mode 100644 terraform/deployments/govuk-publishing-infrastructure/outputs.tf diff --git a/terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf b/terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf new file mode 100644 index 000000000..b6e5c3e9b --- /dev/null +++ b/terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf @@ -0,0 +1,251 @@ +locals { + efs_csi_driver_controller_service_account_name = "efs-csi-controller-sa" +} + +module "aws_efs_csi_driver_iam_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "~> 5.0" + create_role = true + role_name = "${local.efs_csi_driver_controller_service_account_name}-${var.cluster_name}" + role_description = "Role for the AWS EFS CSI driver controller. Corresponds to ${local.efs_csi_driver_controller_service_account_name} k8s ServiceAccount." + provider_url = module.eks.oidc_provider + role_policy_arns = [aws_iam_policy.aws_efs_csi_driver.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:${local.efs_csi_driver_controller_service_account_name}"] +} + +data "aws_iam_policy_document" "aws_efs_csi_driver" { + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:DescribeAccessPoints", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:DescribeMountTargets", + "ec2:DescribeAvailabilityZones" + ] + + resources = ["*"] + } + + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:CreateAccessPoint" + ] + + resources = [ + "arn:aws:ec2:*:*:volume/*", + "arn:aws:ec2:*:*:snapshot/*" + ] + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + values = ["CreateVolume", "CreateSnapshot"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteTags" + ] + + resources = [ + "arn:aws:ec2:*:*:volume/*", + "arn:aws:ec2:*:*:snapshot/*" + ] + } + + statement { + effect = "Allow" + + actions = [ + "ec2:CreateVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/ebs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:CreateVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/CSIVolumeName" + values = ["*"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:CreateVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/kubernetes.io/cluster/*" + values = ["owned"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "ec2:ResourceTag/ebs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "ec2:ResourceTag/CSIVolumeName" + values = ["*"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "ec2:ResourceTag/kubernetes.io/cluster/*" + values = ["owned"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteSnapshot" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "ec2:ResourceTag/CSIVolumeSnapshotName" + values = ["*"] + } + } + + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:CreateAccessPoint" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/efs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:TagResource" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/efs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:DeleteAccessPoint" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/efs.csi.aws.com/cluster" + values = ["true"] + } + } +} + +resource "aws_iam_policy" "aws_efs_csi_driver" { + name = "AWSEfsCsiController-${var.cluster_name}" + description = "Allow the driver to manage AWS EFS" + + # Verbatim contents of + # https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/refs/heads/master/docs/iam-policy-example.json + # (except for whitespace changes from terraform fmt). + policy = data.aws_iam_policy_document.aws_efs_csi_driver.json +} diff --git a/terraform/deployments/cluster-infrastructure/outputs.tf b/terraform/deployments/cluster-infrastructure/outputs.tf index bfcf50747..58e657374 100644 --- a/terraform/deployments/cluster-infrastructure/outputs.tf +++ b/terraform/deployments/cluster-infrastructure/outputs.tf @@ -18,6 +18,11 @@ output "aws_ebs_csi_driver_iam_role_arn" { value = module.aws_ebs_csi_driver_iam_role.iam_role_arn } +output "aws_efs_csi_driver_iam_role_arn" { + description = "IAM role ARN for AWS EFS CSI controller role" + value = module.aws_efs_csi_driver_iam_role.iam_role_arn +} + output "control_plane_security_group_id" { description = "ID of the security group which contains the (AWS-owned) control plane nodes." value = module.eks.cluster_primary_security_group_id @@ -108,6 +113,11 @@ output "aws_ebs_csi_driver_controller_service_account_name" { value = local.ebs_csi_driver_controller_service_account_name } +output "aws_efs_csi_driver_controller_service_account_name" { + description = "Name of the k8s service account for the AWS EFS CSI Controller" + value = local.efs_csi_driver_controller_service_account_name +} + output "grafana_iam_role_arn" { description = "IAM role ARN corresponding to the k8s service account for Grafana." value = module.grafana_iam_role.iam_role_arn diff --git a/terraform/deployments/cluster-services/aws_efs_csi_driver.tf b/terraform/deployments/cluster-services/aws_efs_csi_driver.tf new file mode 100644 index 000000000..6659edeb1 --- /dev/null +++ b/terraform/deployments/cluster-services/aws_efs_csi_driver.tf @@ -0,0 +1,24 @@ +resource "helm_release" "efs_csi_driver" { + chart = "aws-efs-csi-driver" + name = "aws-efs-csi-driver" + namespace = "kube-system" + repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver" + version = "3.1.1" # TODO: Dependabot or equivalent so this doesn't get neglected. + + values = [yamlencode({ + controller = { + serviceAccount = { + create = true + name = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.aws_efs_csi_driver_controller_service_account_name + annotations = { + "eks.amazonaws.com/role-arn" = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.aws_efs_csi_driver_iam_role_arn + } + } + } + storageClasses = [{ + name = "assets_efs-efs-sc" + apiVersion = "storage.k8s.io/v1" + reclaimPolicy = "Retain" + }] + })] +} diff --git a/terraform/deployments/govuk-publishing-infrastructure/outputs.tf b/terraform/deployments/govuk-publishing-infrastructure/outputs.tf deleted file mode 100644 index 9c6dc319f..000000000 --- a/terraform/deployments/govuk-publishing-infrastructure/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "eks_ingress_www_origin_security_group_name" { - value = aws_security_group.eks_ingress_www_origin.name -}