From 90aa50ce001fd59011834a1ec2648267431fe493 Mon Sep 17 00:00:00 2001 From: nimalank7 Date: Fri, 29 Nov 2024 17:47:23 +0000 Subject: [PATCH] Add AWS EFS CSI Driver Configuration Description: - Add a statically provisioned EFS CSI Driver as part of a series of PRs to move asset-manager to `nfs` volume type - PR only installs the CSI Driver but doesn't provision an EFS volume until a `PersistentVolumeClaim` is created in the Helm Chart - See https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/examples/kubernetes/static_provisioning - As part of https://github.com/alphagov/govuk-helm-charts/issues/1883 --- .../cluster-infrastructure/aws_efs_csi_iam.tf | 251 ++++++++++++++++++ .../cluster-infrastructure/outputs.tf | 10 + .../cluster-services/aws_efs_csi_driver.tf | 29 ++ .../deployments/cluster-services/remote.tf | 5 + .../outputs.tf | 5 + 5 files changed, 300 insertions(+) create mode 100644 terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf create mode 100644 terraform/deployments/cluster-services/aws_efs_csi_driver.tf diff --git a/terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf b/terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf new file mode 100644 index 000000000..b6e5c3e9b --- /dev/null +++ b/terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf @@ -0,0 +1,251 @@ +locals { + efs_csi_driver_controller_service_account_name = "efs-csi-controller-sa" +} + +module "aws_efs_csi_driver_iam_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "~> 5.0" + create_role = true + role_name = "${local.efs_csi_driver_controller_service_account_name}-${var.cluster_name}" + role_description = "Role for the AWS EFS CSI driver controller. Corresponds to ${local.efs_csi_driver_controller_service_account_name} k8s ServiceAccount." + provider_url = module.eks.oidc_provider + role_policy_arns = [aws_iam_policy.aws_efs_csi_driver.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:${local.efs_csi_driver_controller_service_account_name}"] +} + +data "aws_iam_policy_document" "aws_efs_csi_driver" { + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:DescribeAccessPoints", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:DescribeMountTargets", + "ec2:DescribeAvailabilityZones" + ] + + resources = ["*"] + } + + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:CreateAccessPoint" + ] + + resources = [ + "arn:aws:ec2:*:*:volume/*", + "arn:aws:ec2:*:*:snapshot/*" + ] + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + values = ["CreateVolume", "CreateSnapshot"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteTags" + ] + + resources = [ + "arn:aws:ec2:*:*:volume/*", + "arn:aws:ec2:*:*:snapshot/*" + ] + } + + statement { + effect = "Allow" + + actions = [ + "ec2:CreateVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/ebs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:CreateVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/CSIVolumeName" + values = ["*"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:CreateVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/kubernetes.io/cluster/*" + values = ["owned"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "ec2:ResourceTag/ebs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "ec2:ResourceTag/CSIVolumeName" + values = ["*"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteVolume" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "ec2:ResourceTag/kubernetes.io/cluster/*" + values = ["owned"] + } + } + + statement { + effect = "Allow" + + actions = [ + "ec2:DeleteSnapshot" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "ec2:ResourceTag/CSIVolumeSnapshotName" + values = ["*"] + } + } + + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:CreateAccessPoint" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/efs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:TagResource" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/efs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + effect = "Allow" + + actions = [ + "elasticfilesystem:DeleteAccessPoint" + ] + + resources = [ + "*" + ] + + condition { + test = "StringLike" + variable = "aws:RequestTag/efs.csi.aws.com/cluster" + values = ["true"] + } + } +} + +resource "aws_iam_policy" "aws_efs_csi_driver" { + name = "AWSEfsCsiController-${var.cluster_name}" + description = "Allow the driver to manage AWS EFS" + + # Verbatim contents of + # https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/refs/heads/master/docs/iam-policy-example.json + # (except for whitespace changes from terraform fmt). + policy = data.aws_iam_policy_document.aws_efs_csi_driver.json +} diff --git a/terraform/deployments/cluster-infrastructure/outputs.tf b/terraform/deployments/cluster-infrastructure/outputs.tf index bfcf50747..58e657374 100644 --- a/terraform/deployments/cluster-infrastructure/outputs.tf +++ b/terraform/deployments/cluster-infrastructure/outputs.tf @@ -18,6 +18,11 @@ output "aws_ebs_csi_driver_iam_role_arn" { value = module.aws_ebs_csi_driver_iam_role.iam_role_arn } +output "aws_efs_csi_driver_iam_role_arn" { + description = "IAM role ARN for AWS EFS CSI controller role" + value = module.aws_efs_csi_driver_iam_role.iam_role_arn +} + output "control_plane_security_group_id" { description = "ID of the security group which contains the (AWS-owned) control plane nodes." value = module.eks.cluster_primary_security_group_id @@ -108,6 +113,11 @@ output "aws_ebs_csi_driver_controller_service_account_name" { value = local.ebs_csi_driver_controller_service_account_name } +output "aws_efs_csi_driver_controller_service_account_name" { + description = "Name of the k8s service account for the AWS EFS CSI Controller" + value = local.efs_csi_driver_controller_service_account_name +} + output "grafana_iam_role_arn" { description = "IAM role ARN corresponding to the k8s service account for Grafana." value = module.grafana_iam_role.iam_role_arn diff --git a/terraform/deployments/cluster-services/aws_efs_csi_driver.tf b/terraform/deployments/cluster-services/aws_efs_csi_driver.tf new file mode 100644 index 000000000..096c6248f --- /dev/null +++ b/terraform/deployments/cluster-services/aws_efs_csi_driver.tf @@ -0,0 +1,29 @@ +resource "helm_release" "efs_csi_driver" { + chart = "aws-efs-csi-driver" + name = "aws-efs-csi-driver" + namespace = "kube-system" + repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver" + version = "3.1.1" # TODO: Dependabot or equivalent so this doesn't get neglected. + + values = [yamlencode({ + controller = { + serviceAccount = { + create = true + name = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.aws_efs_csi_driver_controller_service_account_name + annotations = { + "eks.amazonaws.com/role-arn" = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.aws_efs_csi_driver_iam_role_arn + } + } + } + storageClasses = [{ + name = "assets_efs-efs-sc" + apiVersion = "storage.k8s.io/v1" + mountOptions = ["tls"] + parameters = { + fileSystemId = data.tfe_outputs.govuk_publishing_infrastructure.nonsensitive_values.assets_efs_id + } + reclaimPolicy = "Retain" + volumeBindingMode = "WaitForFirstConsumer" + }] + })] +} diff --git a/terraform/deployments/cluster-services/remote.tf b/terraform/deployments/cluster-services/remote.tf index fc3cc239e..60888110a 100644 --- a/terraform/deployments/cluster-services/remote.tf +++ b/terraform/deployments/cluster-services/remote.tf @@ -11,3 +11,8 @@ data "tfe_outputs" "vpc" { organization = "govuk" workspace = "vpc-${var.govuk_environment}" } + +data "tfe_outputs" "govuk_publishing_infrastructure" { + organization = "govuk" + workspace = "govuk-publishing-infrastructure-${var.govuk_environment}" +} diff --git a/terraform/deployments/govuk-publishing-infrastructure/outputs.tf b/terraform/deployments/govuk-publishing-infrastructure/outputs.tf index 9c6dc319f..7d5a67532 100644 --- a/terraform/deployments/govuk-publishing-infrastructure/outputs.tf +++ b/terraform/deployments/govuk-publishing-infrastructure/outputs.tf @@ -1,3 +1,8 @@ output "eks_ingress_www_origin_security_group_name" { value = aws_security_group.eks_ingress_www_origin.name } + +output "assets_efs_id" { + description = "EFS Filesystem ID for assets" + value = aws_efs_file_system.assets_efs.id +} \ No newline at end of file