From d854f7a80bf641ead7596d3226322fd5cc0b8c79 Mon Sep 17 00:00:00 2001 From: Simon Hughesdon Date: Wed, 15 May 2024 11:05:25 +0100 Subject: [PATCH] Remove resources required for Learning to Rank Learning to Rank is no longer used in search, so we don't need to train or host models any more. This won't remove any of the actual Sagemaker resources, as this was done in [the deploy pipeline](https://github.com/alphagov/search-api/blob/f037f11c6f0299b8bd4d065a1d947b8f827bbc0d/ltr/sagemaker/deploy.py). We'll remove these separately. --- terraform/projects/app-search/README.md | 27 --- terraform/projects/app-search/main.tf | 236 ----------------------- terraform/projects/app-search/outputs.tf | 20 -- 3 files changed, 283 deletions(-) diff --git a/terraform/projects/app-search/README.md b/terraform/projects/app-search/README.md index 84def16a8..7cce8b537 100644 --- a/terraform/projects/app-search/README.md +++ b/terraform/projects/app-search/README.md @@ -26,34 +26,11 @@ Search application servers | Name | Type | |------|------| -| [aws_autoscaling_group.learntorank-generation](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/autoscaling_group) | resource | -| [aws_ecr_repository.repo](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/ecr_repository) | resource | -| [aws_ecr_repository_policy.policy](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/ecr_repository_policy) | resource | -| [aws_iam_instance_profile.learntorank-generation](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_instance_profile) | resource | -| [aws_iam_policy.scale-learntorank-generation-asg-policy](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_policy) | resource | -| [aws_iam_policy.search_relevancy_bucket_access](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_policy) | resource | | [aws_iam_policy.sitemaps_bucket_access](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_policy) | resource | -| [aws_iam_policy.use_sagemaker](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_policy) | resource | -| [aws_iam_role.learntorank](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_role) | resource | -| [aws_iam_role_policy_attachment.learntorank-bucket](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.learntorank-ecr](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.learntorank-sagemaker](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.scale-learntorank-generation](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.search_relevancy_bucket_access_iam_role_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.sitemaps_bucket_access_iam_role_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.use_sagemaker](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/iam_role_policy_attachment) | resource | -| [aws_key_pair.learntorank-generation-key](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/key_pair) | resource | -| [aws_launch_template.learntorank-generation](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/launch_template) | resource | -| [aws_s3_bucket.search_relevancy_bucket](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/s3_bucket) | resource | | [aws_s3_bucket.sitemaps_bucket](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/resources/s3_bucket) | resource | | [null_resource.user_data](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [aws_ami.ubuntu_focal](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/data-sources/ami) | data source | -| [aws_iam_policy_document.ecr-usage](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.learntorank-assume-role](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.scale-learntorank-generation-asg](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.search_relevancy_bucket_policy](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.sitemaps_bucket_policy](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.use_sagemaker](https://registry.terraform.io/providers/hashicorp/aws/2.46.0/docs/data-sources/iam_policy_document) | data source | | [terraform_remote_state.infra_monitoring](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | | [terraform_remote_state.infra_networking](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | | [terraform_remote_state.infra_root_dns_zones](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | @@ -87,8 +64,4 @@ Search application servers | Name | Description | |------|-------------| -| [ecr\_repository\_url](#output\_ecr\_repository\_url) | URL of the ECR repository | -| [ltr\_role\_arn](#output\_ltr\_role\_arn) | LTR role ARN | -| [scale\_learntorank\_asg\_policy\_arn](#output\_scale\_learntorank\_asg\_policy\_arn) | ARN of the policy used by to scale the ASG for learn to rank | -| [search\_relevancy\_s3\_policy\_arn](#output\_search\_relevancy\_s3\_policy\_arn) | ARN of the policy used to access the search-relevancy S3 bucket | | [sitemaps\_s3\_policy\_arn](#output\_sitemaps\_s3\_policy\_arn) | ARN of the policy used to access the sitemaps S3 bucket | diff --git a/terraform/projects/app-search/main.tf b/terraform/projects/app-search/main.tf index fd334f370..fb694f28e 100644 --- a/terraform/projects/app-search/main.tf +++ b/terraform/projects/app-search/main.tf @@ -89,239 +89,3 @@ data "aws_iam_policy_document" "sitemaps_bucket_policy" { ] } } - -resource "aws_iam_role_policy_attachment" "use_sagemaker" { - role = module.search.instance_iam_role_name - policy_arn = aws_iam_policy.use_sagemaker.arn -} - -resource "aws_iam_policy" "use_sagemaker" { - name = "govuk-${var.aws_environment}-search-use-sagemaker-policy" - policy = data.aws_iam_policy_document.use_sagemaker.json - description = "Allows invoking and describing SageMaker endpoints" -} - -data "aws_iam_policy_document" "use_sagemaker" { - statement { - sid = "InvokeSagemaker" - actions = [ - "sagemaker:DescribeEndpoint", - "sagemaker:InvokeEndpoint", - ] - resources = ["arn:aws:sagemaker:*"] - } -} - -resource "aws_s3_bucket" "search_relevancy_bucket" { - bucket = "govuk-${var.aws_environment}-search-relevancy" - region = var.aws_region - - tags = { - Name = "govuk-${var.aws_environment}-search-relevancy" - Description = "S3 bucket for Search Relevancy" - aws_environment = var.aws_environment - } - - logging { - target_bucket = data.terraform_remote_state.infra_monitoring.outputs.aws_logging_bucket_id - target_prefix = "s3/govuk-${var.aws_environment}-search-relevancy/" - } - - lifecycle_rule { - id = "expire_training_data" - prefix = "data/" - enabled = true - expiration { days = 7 } - } - - lifecycle_rule { - id = "expire_models" - prefix = "model/" - enabled = true - expiration { days = 7 } - } -} - -resource "aws_iam_role_policy_attachment" "search_relevancy_bucket_access_iam_role_policy_attachment" { - role = module.search.instance_iam_role_name - policy_arn = aws_iam_policy.search_relevancy_bucket_access.arn -} - -resource "aws_iam_policy" "search_relevancy_bucket_access" { - name = "govuk-${var.aws_environment}-search-relevancy-bucket-access-policy" - policy = data.aws_iam_policy_document.search_relevancy_bucket_policy.json - description = "Allows reading and writing of the search relevancy bucket" -} - -data "aws_iam_policy_document" "search_relevancy_bucket_policy" { - statement { - sid = "ReadListOfBuckets" - actions = ["s3:ListAllMyBuckets"] - resources = ["*"] - } - - statement { - sid = "SearchRelevancyAccess" - actions = [ - "s3:DeleteObject", - "s3:Put*", - "s3:Get*", - "s3:List*", - ] - resources = [ - "arn:aws:s3:::${aws_s3_bucket.search_relevancy_bucket.id}", - "arn:aws:s3:::${aws_s3_bucket.search_relevancy_bucket.id}/*", - ] - } -} - -# Daily learn-to-rank - -resource "aws_iam_role" "learntorank" { - name = "govuk-${var.aws_environment}-search-learntorank-role" - assume_role_policy = data.aws_iam_policy_document.learntorank-assume-role.json -} - -data "aws_iam_policy_document" "learntorank-assume-role" { - statement { - actions = ["sts:AssumeRole"] - principals { - type = "Service" - identifiers = ["sagemaker.amazonaws.com"] - } - principals { - type = "Service" - identifiers = ["ec2.amazonaws.com"] - } - } -} - -resource "aws_iam_role_policy_attachment" "learntorank-bucket" { - role = aws_iam_role.learntorank.name - policy_arn = aws_iam_policy.search_relevancy_bucket_access.arn -} - -# this grants much broader permissions than we need, so we might want -# to narrow this down in the future. -resource "aws_iam_role_policy_attachment" "learntorank-sagemaker" { - role = aws_iam_role.learntorank.name - policy_arn = "arn:aws:iam::aws:policy/AmazonSageMakerFullAccess" -} - -resource "aws_iam_role_policy_attachment" "learntorank-ecr" { - role = aws_iam_role.learntorank.name - policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" -} - -resource "aws_ecr_repository" "repo" { - name = "search" - image_tag_mutability = "MUTABLE" -} - -resource "aws_ecr_repository_policy" "policy" { - repository = aws_ecr_repository.repo.name - policy = data.aws_iam_policy_document.ecr-usage.json -} - -data "aws_iam_policy_document" "ecr-usage" { - statement { - sid = "read" - actions = [ - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:DescribeRepositories", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:ListImages", - ] - principals { - type = "AWS" - identifiers = [aws_iam_role.learntorank.arn] - } - principals { - type = "Service" - identifiers = ["sagemaker.amazonaws.com"] - } - } -} - -resource "aws_iam_instance_profile" "learntorank-generation" { - name = "govuk-${var.aws_environment}-search-ltr-generation" - role = aws_iam_role.learntorank.name -} - -resource "aws_key_pair" "learntorank-generation-key" { - key_name = "govuk-${var.aws_environment}-search-ltr-generation-key" - public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDGNF9QVcjq9LUioccV8Fw161BrM3EHPRtwzp+2IRaYrgpwgBoIxgK2q1LLrLSwcoLfbDyU3dW0cMN0wpbxFzHTFuDqzm5fdAzhijZJCKMyPtMSPOhUev+/JfHVZj7JGXHV2SOMM1Q1XkEBwgenPmR2Hz6fMs3+R/LdeNkMTU1H/fOXl6WU9DY1XAUdYzfufRXiDt2aCGCOknAWqOAdT+22FZcmgc657tt9xbOJYzVoEAqBArCxixpf5N7Tha0QUac8QGQQxw01LENHRN1S4NLtvUEBqI3m99f8NleOlO4eD7XBkcwPXMrFP7/4IqAPq+JgoD2OrDSX3HiE8HNtJTLr0vmP5plBiwH3Bd+32oILQiw4HqXt8JpTfr/fAJXlsHCmYkxlEzhhZ46H1VZsgU9BM69C/bWTvGWCFAYrWbu2vt9Gbo1nbZVTQjLBfKgY3vxk5Tmj4b43AGI1tprPdBh43IdQvvYu9oiTodzxetaQoK8fUMKPVoQruPJNfKcu3Yukm8DvVmwQqoAgik5iYk7up9gX1L//L0dJIpjWSlU5ytpmG+M5k+Abbg+kkIjnCXXkS2Icwnh3BEIvxLIt9MaMf89Lxi4Jin1uNu727Z9cXGRp8Fyz5GdDEKz37P5k7jFEV70KYLwl3r7qxp66RafXgRx/fRRVHdTNf43O6UqDUQ== concourse-worker" -} - -data "aws_ami" "ubuntu_focal" { - most_recent = true - owners = ["099720109477", "696911096973"] # Canonical - filter { - name = "name" - values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] - } -} - -resource "aws_launch_template" "learntorank-generation" { - name = "govuk-${var.aws_environment}-search-ltr-generation" - image_id = data.aws_ami.ubuntu_focal.id - instance_type = "c5.large" - vpc_security_group_ids = [data.terraform_remote_state.infra_security_groups.outputs.sg_search-ltr-generation_id] - key_name = aws_key_pair.learntorank-generation-key.key_name - - iam_instance_profile { name = aws_iam_instance_profile.learntorank-generation.name } - lifecycle { create_before_destroy = true } - instance_initiated_shutdown_behavior = "terminate" - - block_device_mappings { - device_name = "/dev/sda1" - ebs { volume_size = 32 } - } -} - -resource "aws_autoscaling_group" "learntorank-generation" { - name = "govuk-${var.aws_environment}-search-ltr-generation" - min_size = 0 - max_size = 1 - desired_capacity = 0 - - launch_template { - id = aws_launch_template.learntorank-generation.id - version = "$Latest" - } - - vpc_zone_identifier = data.terraform_remote_state.infra_networking.outputs.public_subnet_ids - - tag { - key = "Name" - value = "govuk-${var.aws_environment}-search-ltr-generation" - propagate_at_launch = true - } -} - -data "aws_iam_policy_document" "scale-learntorank-generation-asg" { - statement { - actions = [ - "autoscaling:DescribeAutoScalingGroups", - "ec2:DescribeInstances", - "ec2:DescribeInstanceStatus", - ] - resources = ["*"] - } - statement { - actions = ["autoscaling:SetDesiredCapacity"] - resources = [aws_autoscaling_group.learntorank-generation.arn] - } -} - -resource "aws_iam_policy" "scale-learntorank-generation-asg-policy" { - name = "govuk-${var.aws_environment}-scale-search-ltr-generation-asg" - policy = data.aws_iam_policy_document.scale-learntorank-generation-asg.json -} - -resource "aws_iam_role_policy_attachment" "scale-learntorank-generation" { - role = aws_iam_role.learntorank.name - policy_arn = aws_iam_policy.scale-learntorank-generation-asg-policy.arn -} diff --git a/terraform/projects/app-search/outputs.tf b/terraform/projects/app-search/outputs.tf index d78cb269b..8f3cda714 100644 --- a/terraform/projects/app-search/outputs.tf +++ b/terraform/projects/app-search/outputs.tf @@ -1,23 +1,3 @@ -output "scale_learntorank_asg_policy_arn" { - value = aws_iam_policy.scale-learntorank-generation-asg-policy.arn - description = "ARN of the policy used by to scale the ASG for learn to rank" -} - -output "ltr_role_arn" { - value = aws_iam_role.learntorank.arn - description = "LTR role ARN" -} - -output "ecr_repository_url" { - value = aws_ecr_repository.repo.repository_url - description = "URL of the ECR repository" -} - -output "search_relevancy_s3_policy_arn" { - value = aws_iam_policy.search_relevancy_bucket_access.arn - description = "ARN of the policy used to access the search-relevancy S3 bucket" -} - output "sitemaps_s3_policy_arn" { value = aws_iam_policy.sitemaps_bucket_access.arn description = "ARN of the policy used to access the sitemaps S3 bucket"