From 5ccef7098f772778bfed4c3166a8bf883fe2b810 Mon Sep 17 00:00:00 2001 From: Simon Hughesdon Date: Wed, 15 May 2024 11:05:25 +0100 Subject: [PATCH] Remove resources required for Learning to Rank This is no longer used in search, so we don't need to train or host models any more. --- terraform/projects/app-search/main.tf | 173 ----------------------- terraform/projects/app-search/outputs.tf | 15 -- 2 files changed, 188 deletions(-) diff --git a/terraform/projects/app-search/main.tf b/terraform/projects/app-search/main.tf index fd334f370..cf7bff6dc 100644 --- a/terraform/projects/app-search/main.tf +++ b/terraform/projects/app-search/main.tf @@ -90,28 +90,6 @@ data "aws_iam_policy_document" "sitemaps_bucket_policy" { } } -resource "aws_iam_role_policy_attachment" "use_sagemaker" { - role = module.search.instance_iam_role_name - policy_arn = aws_iam_policy.use_sagemaker.arn -} - -resource "aws_iam_policy" "use_sagemaker" { - name = "govuk-${var.aws_environment}-search-use-sagemaker-policy" - policy = data.aws_iam_policy_document.use_sagemaker.json - description = "Allows invoking and describing SageMaker endpoints" -} - -data "aws_iam_policy_document" "use_sagemaker" { - statement { - sid = "InvokeSagemaker" - actions = [ - "sagemaker:DescribeEndpoint", - "sagemaker:InvokeEndpoint", - ] - resources = ["arn:aws:sagemaker:*"] - } -} - resource "aws_s3_bucket" "search_relevancy_bucket" { bucket = "govuk-${var.aws_environment}-search-relevancy" region = var.aws_region @@ -174,154 +152,3 @@ data "aws_iam_policy_document" "search_relevancy_bucket_policy" { ] } } - -# Daily learn-to-rank - -resource "aws_iam_role" "learntorank" { - name = "govuk-${var.aws_environment}-search-learntorank-role" - assume_role_policy = data.aws_iam_policy_document.learntorank-assume-role.json -} - -data "aws_iam_policy_document" "learntorank-assume-role" { - statement { - actions = ["sts:AssumeRole"] - principals { - type = "Service" - identifiers = ["sagemaker.amazonaws.com"] - } - principals { - type = "Service" - identifiers = ["ec2.amazonaws.com"] - } - } -} - -resource "aws_iam_role_policy_attachment" "learntorank-bucket" { - role = aws_iam_role.learntorank.name - policy_arn = aws_iam_policy.search_relevancy_bucket_access.arn -} - -# this grants much broader permissions than we need, so we might want -# to narrow this down in the future. -resource "aws_iam_role_policy_attachment" "learntorank-sagemaker" { - role = aws_iam_role.learntorank.name - policy_arn = "arn:aws:iam::aws:policy/AmazonSageMakerFullAccess" -} - -resource "aws_iam_role_policy_attachment" "learntorank-ecr" { - role = aws_iam_role.learntorank.name - policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" -} - -resource "aws_ecr_repository" "repo" { - name = "search" - image_tag_mutability = "MUTABLE" -} - -resource "aws_ecr_repository_policy" "policy" { - repository = aws_ecr_repository.repo.name - policy = data.aws_iam_policy_document.ecr-usage.json -} - -data "aws_iam_policy_document" "ecr-usage" { - statement { - sid = "read" - actions = [ - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:DescribeRepositories", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:ListImages", - ] - principals { - type = "AWS" - identifiers = [aws_iam_role.learntorank.arn] - } - principals { - type = "Service" - identifiers = ["sagemaker.amazonaws.com"] - } - } -} - -resource "aws_iam_instance_profile" "learntorank-generation" { - name = "govuk-${var.aws_environment}-search-ltr-generation" - role = aws_iam_role.learntorank.name -} - -resource "aws_key_pair" "learntorank-generation-key" { - key_name = "govuk-${var.aws_environment}-search-ltr-generation-key" - public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDGNF9QVcjq9LUioccV8Fw161BrM3EHPRtwzp+2IRaYrgpwgBoIxgK2q1LLrLSwcoLfbDyU3dW0cMN0wpbxFzHTFuDqzm5fdAzhijZJCKMyPtMSPOhUev+/JfHVZj7JGXHV2SOMM1Q1XkEBwgenPmR2Hz6fMs3+R/LdeNkMTU1H/fOXl6WU9DY1XAUdYzfufRXiDt2aCGCOknAWqOAdT+22FZcmgc657tt9xbOJYzVoEAqBArCxixpf5N7Tha0QUac8QGQQxw01LENHRN1S4NLtvUEBqI3m99f8NleOlO4eD7XBkcwPXMrFP7/4IqAPq+JgoD2OrDSX3HiE8HNtJTLr0vmP5plBiwH3Bd+32oILQiw4HqXt8JpTfr/fAJXlsHCmYkxlEzhhZ46H1VZsgU9BM69C/bWTvGWCFAYrWbu2vt9Gbo1nbZVTQjLBfKgY3vxk5Tmj4b43AGI1tprPdBh43IdQvvYu9oiTodzxetaQoK8fUMKPVoQruPJNfKcu3Yukm8DvVmwQqoAgik5iYk7up9gX1L//L0dJIpjWSlU5ytpmG+M5k+Abbg+kkIjnCXXkS2Icwnh3BEIvxLIt9MaMf89Lxi4Jin1uNu727Z9cXGRp8Fyz5GdDEKz37P5k7jFEV70KYLwl3r7qxp66RafXgRx/fRRVHdTNf43O6UqDUQ== concourse-worker" -} - -data "aws_ami" "ubuntu_focal" { - most_recent = true - owners = ["099720109477", "696911096973"] # Canonical - filter { - name = "name" - values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] - } -} - -resource "aws_launch_template" "learntorank-generation" { - name = "govuk-${var.aws_environment}-search-ltr-generation" - image_id = data.aws_ami.ubuntu_focal.id - instance_type = "c5.large" - vpc_security_group_ids = [data.terraform_remote_state.infra_security_groups.outputs.sg_search-ltr-generation_id] - key_name = aws_key_pair.learntorank-generation-key.key_name - - iam_instance_profile { name = aws_iam_instance_profile.learntorank-generation.name } - lifecycle { create_before_destroy = true } - instance_initiated_shutdown_behavior = "terminate" - - block_device_mappings { - device_name = "/dev/sda1" - ebs { volume_size = 32 } - } -} - -resource "aws_autoscaling_group" "learntorank-generation" { - name = "govuk-${var.aws_environment}-search-ltr-generation" - min_size = 0 - max_size = 1 - desired_capacity = 0 - - launch_template { - id = aws_launch_template.learntorank-generation.id - version = "$Latest" - } - - vpc_zone_identifier = data.terraform_remote_state.infra_networking.outputs.public_subnet_ids - - tag { - key = "Name" - value = "govuk-${var.aws_environment}-search-ltr-generation" - propagate_at_launch = true - } -} - -data "aws_iam_policy_document" "scale-learntorank-generation-asg" { - statement { - actions = [ - "autoscaling:DescribeAutoScalingGroups", - "ec2:DescribeInstances", - "ec2:DescribeInstanceStatus", - ] - resources = ["*"] - } - statement { - actions = ["autoscaling:SetDesiredCapacity"] - resources = [aws_autoscaling_group.learntorank-generation.arn] - } -} - -resource "aws_iam_policy" "scale-learntorank-generation-asg-policy" { - name = "govuk-${var.aws_environment}-scale-search-ltr-generation-asg" - policy = data.aws_iam_policy_document.scale-learntorank-generation-asg.json -} - -resource "aws_iam_role_policy_attachment" "scale-learntorank-generation" { - role = aws_iam_role.learntorank.name - policy_arn = aws_iam_policy.scale-learntorank-generation-asg-policy.arn -} diff --git a/terraform/projects/app-search/outputs.tf b/terraform/projects/app-search/outputs.tf index d78cb269b..265045d07 100644 --- a/terraform/projects/app-search/outputs.tf +++ b/terraform/projects/app-search/outputs.tf @@ -1,18 +1,3 @@ -output "scale_learntorank_asg_policy_arn" { - value = aws_iam_policy.scale-learntorank-generation-asg-policy.arn - description = "ARN of the policy used by to scale the ASG for learn to rank" -} - -output "ltr_role_arn" { - value = aws_iam_role.learntorank.arn - description = "LTR role ARN" -} - -output "ecr_repository_url" { - value = aws_ecr_repository.repo.repository_url - description = "URL of the ECR repository" -} - output "search_relevancy_s3_policy_arn" { value = aws_iam_policy.search_relevancy_bucket_access.arn description = "ARN of the policy used to access the search-relevancy S3 bucket"