Add AWS EFS CSI Driver Configuration

Description:
- Add a statically provisioned EFS CSI Driver as part of a series of PRs to move asset-manager to `nfs` volume type. This is because dynamic provisioning causes errors - see https://trello.com/c/2XkadNJ2/1011-resolve-asset-manager-pvc-related-errors
- PR only installs the CSI Driver but doesn't provision an EFS volume until a `PersistentVolumeClaim` is created in the Helm Chart
- See https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/examples/kubernetes/static_provisioning
- As part of alphagov/govuk-helm-charts#1883

terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf

@@ -0,0 +1,251 @@
+locals {
+  efs_csi_driver_controller_service_account_name = "efs-csi-controller-sa"
+}
+
+module "aws_efs_csi_driver_iam_role" {
+  source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  version                       = "~> 5.0"
+  create_role                   = true
+  role_name                     = "${local.efs_csi_driver_controller_service_account_name}-${var.cluster_name}"
+  role_description              = "Role for the AWS EFS CSI driver controller. Corresponds to ${local.efs_csi_driver_controller_service_account_name} k8s ServiceAccount."
+  provider_url                  = module.eks.oidc_provider
+  role_policy_arns              = [aws_iam_policy.aws_efs_csi_driver.arn]
+  oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:${local.efs_csi_driver_controller_service_account_name}"]
+}
+
+data "aws_iam_policy_document" "aws_efs_csi_driver" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "elasticfilesystem:DescribeAccessPoints",
+      "elasticfilesystem:DescribeFileSystems",
+      "elasticfilesystem:DescribeMountTargets",
+      "ec2:DescribeAvailabilityZones"
+    ]
+
+    resources = ["*"]
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "elasticfilesystem:CreateAccessPoint"
+    ]
+
+    resources = [
+      "arn:aws:ec2:*:*:volume/*",
+      "arn:aws:ec2:*:*:snapshot/*"
+    ]
+
+    condition {
+      test     = "StringEquals"
+      variable = "ec2:CreateAction"
+      values   = ["CreateVolume", "CreateSnapshot"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:DeleteTags"
+    ]
+
+    resources = [
+      "arn:aws:ec2:*:*:volume/*",
+      "arn:aws:ec2:*:*:snapshot/*"
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:CreateVolume"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/ebs.csi.aws.com/cluster"
+      values   = ["true"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:CreateVolume"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/CSIVolumeName"
+      values   = ["*"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:CreateVolume"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/kubernetes.io/cluster/*"
+      values   = ["owned"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:DeleteVolume"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "ec2:ResourceTag/ebs.csi.aws.com/cluster"
+      values   = ["true"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:DeleteVolume"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "ec2:ResourceTag/CSIVolumeName"
+      values   = ["*"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:DeleteVolume"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "ec2:ResourceTag/kubernetes.io/cluster/*"
+      values   = ["owned"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:DeleteSnapshot"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "ec2:ResourceTag/CSIVolumeSnapshotName"
+      values   = ["*"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "elasticfilesystem:CreateAccessPoint"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/efs.csi.aws.com/cluster"
+      values   = ["true"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "elasticfilesystem:TagResource"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/efs.csi.aws.com/cluster"
+      values   = ["true"]
+    }
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "elasticfilesystem:DeleteAccessPoint"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/efs.csi.aws.com/cluster"
+      values   = ["true"]
+    }
+  }
+}
+
+resource "aws_iam_policy" "aws_efs_csi_driver" {
+  name        = "AWSEfsCsiController-${var.cluster_name}"
+  description = "Allow the driver to manage AWS EFS"
+
+  # Verbatim contents of
+  # https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/refs/heads/master/docs/iam-policy-example.json
+  # (except for whitespace changes from terraform fmt).
+  policy = data.aws_iam_policy_document.aws_efs_csi_driver.json
+}

terraform/deployments/cluster-infrastructure/outputs.tf

@@ -18,6 +18,11 @@ output "aws_ebs_csi_driver_iam_role_arn" {
   value       = module.aws_ebs_csi_driver_iam_role.iam_role_arn
 }
 
+output "aws_efs_csi_driver_iam_role_arn" {
+  description = "IAM role ARN for AWS EFS CSI controller role"
+  value       = module.aws_efs_csi_driver_iam_role.iam_role_arn
+}
+
 output "control_plane_security_group_id" {
   description = "ID of the security group which contains the (AWS-owned) control plane nodes."
   value       = module.eks.cluster_primary_security_group_id
@@ -108,6 +113,11 @@ output "aws_ebs_csi_driver_controller_service_account_name" {
   value       = local.ebs_csi_driver_controller_service_account_name
 }
 
+output "aws_efs_csi_driver_controller_service_account_name" {
+  description = "Name of the k8s service account for the AWS EFS CSI Controller"
+  value       = local.efs_csi_driver_controller_service_account_name
+}
+
 output "grafana_iam_role_arn" {
   description = "IAM role ARN corresponding to the k8s service account for Grafana."
   value       = module.grafana_iam_role.iam_role_arn

terraform/deployments/cluster-services/aws_efs_csi_driver.tf

@@ -0,0 +1,24 @@
+resource "helm_release" "efs_csi_driver" {
+  chart      = "aws-efs-csi-driver"
+  name       = "aws-efs-csi-driver"
+  namespace  = "kube-system"
+  repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver"
+  version    = "3.1.1" # TODO: Dependabot or equivalent so this doesn't get neglected.
+
+  values = [yamlencode({
+    controller = {
+      serviceAccount = {
+        create = true
+        name   = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.aws_efs_csi_driver_controller_service_account_name
+        annotations = {
+          "eks.amazonaws.com/role-arn" = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.aws_efs_csi_driver_iam_role_arn
+        }
+      }
+    }
+    storageClasses = [{
+      name          = "assets_efs-efs-sc"
+      apiVersion    = "storage.k8s.io/v1"
+      reclaimPolicy = "Retain"
+    }]
+  })]
+}

terraform/deployments/cluster-services/remote.tf

@@ -11,3 +11,8 @@ data "tfe_outputs" "vpc" {
   organization = "govuk"
   workspace    = "vpc-${var.govuk_environment}"
 }
+
+data "tfe_outputs" "govuk_publishing_infrastructure" {
+  organization = "govuk"
+  workspace    = "govuk-publishing-infrastructure-${var.govuk_environment}"
+}

terraform/deployments/govuk-publishing-infrastructure/outputs.tf

@@ -1,3 +1,8 @@
 output "eks_ingress_www_origin_security_group_name" {
   value = aws_security_group.eks_ingress_www_origin.name
 }
+
+output "assets_efs_id" {
+  description = "EFS Filesystem ID for assets"
+  value       = aws_efs_file_system.assets_efs.id
+}