From b1cc207460e770ff3c3558ecc6fb137368f73a75 Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Tue, 23 Jan 2024 16:39:14 +0000 Subject: [PATCH 01/11] Use a dynamic block for DGU domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DGU has two domains www.data.gov.uk and data.gov.uk I’m not using `toset` as according to the docs explicit type conversions are rarely necessary in Terraform because it will convert types automatically where required. I’m using value because according to the dynamic block documentation: - key is the map key or list element index for the current element. If the `for_each` expression produces a set value then key is identical to value and should not be used - value is the value of the current element https://developer.hashicorp.com/terraform/language/expressions/dynamic-blocks --- modules/datagovuk/service.tf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/datagovuk/service.tf b/modules/datagovuk/service.tf index aad8904..fb71832 100644 --- a/modules/datagovuk/service.tf +++ b/modules/datagovuk/service.tf @@ -44,8 +44,12 @@ resource "fastly_service_vcl" "service" { name = "${title(local.template_values["environment"])} data.gov.uk" comment = "" - domain { - name = local.template_values["hostname"] + dynamic "domain" { + for_each = lookup(local.template_values, "hostnames", []) + iterator = each + content { + name = each.value + } } vcl { From 3c273dd9e5deb316839b98361dbeb55a06244c74 Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Thu, 25 Jan 2024 10:10:49 +0000 Subject: [PATCH 02/11] Remove obsolete conditions from DGU Since we only have one back end (no mirrors) for DGU those conditions are obsolete. `beresp.saintmode` marks the backend that was used for this request as unhealthy for the period of time specified. --- modules/datagovuk/datagovuk.vcl.tftpl | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/modules/datagovuk/datagovuk.vcl.tftpl b/modules/datagovuk/datagovuk.vcl.tftpl index c28c6bb..3cb3534 100644 --- a/modules/datagovuk/datagovuk.vcl.tftpl +++ b/modules/datagovuk/datagovuk.vcl.tftpl @@ -126,11 +126,6 @@ sub vcl_fetch { set beresp.http.Fastly-Backend-Name = req.http.Fastly-Backend-Name; - if ((beresp.status >= 500 && beresp.status <= 599) && req.restarts < 3 && (req.request == "GET" || req.request == "HEAD") && !beresp.http.No-Fallback) { - set beresp.saintmode = 5s; - return (restart); - } - if (req.restarts == 0) { # Keep stale for origin set beresp.grace = 24h; @@ -200,11 +195,6 @@ sub vcl_error { } } - # Assume we've hit vcl_error() because the backend is unavailable - if (req.restarts < 2) { - return (restart); - } - synthetic {" From 2fb8e33b85d65258859fa036ed3e7f33d5865479 Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Thu, 25 Jan 2024 10:13:31 +0000 Subject: [PATCH 03/11] Remove unnecessary vcl_hash subroutine from DGU Since we are using the default there's no need to declare it explicitly. This subroutine is executed when Fastly needs to calculate the address of an object in the cache. --- modules/datagovuk/datagovuk.vcl.tftpl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/datagovuk/datagovuk.vcl.tftpl b/modules/datagovuk/datagovuk.vcl.tftpl index 3cb3534..21e050b 100644 --- a/modules/datagovuk/datagovuk.vcl.tftpl +++ b/modules/datagovuk/datagovuk.vcl.tftpl @@ -220,8 +220,4 @@ sub vcl_error { sub vcl_pass { #FASTLY pass -} - -sub vcl_hash { -#FASTLY hash } \ No newline at end of file From d8a8b0efa9c266c6c0c229ccd863995537c62ffc Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Thu, 25 Jan 2024 10:17:24 +0000 Subject: [PATCH 04/11] Don't use deprecated `grace` variable --- modules/datagovuk/datagovuk.vcl.tftpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/datagovuk/datagovuk.vcl.tftpl b/modules/datagovuk/datagovuk.vcl.tftpl index 21e050b..3eb946a 100644 --- a/modules/datagovuk/datagovuk.vcl.tftpl +++ b/modules/datagovuk/datagovuk.vcl.tftpl @@ -98,7 +98,7 @@ sub vcl_recv { } # Serve from stale for 24 hours if origin is sick - set req.grace = 24h; + set req.max_stale_if_error = 24h; # Default backend. set req.backend = F_origin; @@ -128,7 +128,7 @@ sub vcl_fetch { if (req.restarts == 0) { # Keep stale for origin - set beresp.grace = 24h; + set beresp.stale_if_error = 24h; } if(req.restarts > 0 ) { @@ -149,7 +149,7 @@ sub vcl_fetch { if (beresp.status == 500 || beresp.status == 503) { set beresp.ttl = 1s; - set beresp.grace = 5s; + set beresp.stale_if_error = 5s; return (deliver); } From 8f87f1b0e35803c5400bc74703b995f2654ca1d1 Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Thu, 25 Jan 2024 10:19:48 +0000 Subject: [PATCH 05/11] Use correct name for DGU backend --- modules/datagovuk/datagovuk.vcl.tftpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/datagovuk/datagovuk.vcl.tftpl b/modules/datagovuk/datagovuk.vcl.tftpl index 3eb946a..33708db 100644 --- a/modules/datagovuk/datagovuk.vcl.tftpl +++ b/modules/datagovuk/datagovuk.vcl.tftpl @@ -1,4 +1,4 @@ -backend F_origin { +backend F_cname_find_eks_${environment}_govuk_digital { .connect_timeout = 5s; .dynamic = true; .port = "${origin_port}"; From 9ab84b3b117d146c9e86b24ab9fb7e234a139fc3 Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Thu, 25 Jan 2024 10:26:21 +0000 Subject: [PATCH 06/11] Add datagovuk-integration module --- datagovuk.tf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/datagovuk.tf b/datagovuk.tf index 3d66e3b..3ce538b 100644 --- a/datagovuk.tf +++ b/datagovuk.tf @@ -2,14 +2,14 @@ variable "datagovuk_integration" { type = string } -# module "datagovuk-integration" { -# source = "./modules/datagovuk" +module "datagovuk-integration" { + source = "./modules/datagovuk" -# configuration = { -# environment = "integration" -# git_hash = var.TFC_CONFIGURATION_VERSION_GIT_COMMIT_SHA -# probe = "/" -# } + configuration = { + environment = "integration" + git_hash = var.TFC_CONFIGURATION_VERSION_GIT_COMMIT_SHA + probe = "/" + } -# secrets = yamldecode(var.datagovuk_integration) -# } + secrets = yamldecode(var.datagovuk_integration) +} From 2714a0d848fbc06a59c38f9684e1a4400a124910 Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Thu, 25 Jan 2024 14:00:51 +0000 Subject: [PATCH 07/11] Add datagovuk_staging and datagovuk_production variables to be able to import the terraform state. `terraform import module.datagovuk-integration.fastly_service_vcl.service $FASTLY_SERVICE_ID` was failing with "Value for undeclared variable" errors. --- datagovuk.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/datagovuk.tf b/datagovuk.tf index 3ce538b..e5d68c7 100644 --- a/datagovuk.tf +++ b/datagovuk.tf @@ -13,3 +13,11 @@ module "datagovuk-integration" { secrets = yamldecode(var.datagovuk_integration) } + +variable "datagovuk_staging" { + type = string +} + +variable "datagovuk_production" { + type = string +} \ No newline at end of file From 4f6a8856d05d89a0e14f72c489fd037e6411a00c Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Thu, 25 Jan 2024 14:02:23 +0000 Subject: [PATCH 08/11] Import datagovuk-integration state to TF Cloud Generated with: `terraform import module.datagovuk-integration.fastly_service_vcl.service $FASTLY_SERVICE_ID` --- .terraform.lock.hcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 421eb0e..b9a7ae6 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -6,6 +6,7 @@ provider "registry.terraform.io/fastly/fastly" { constraints = "5.3.1" hashes = [ "h1:J/H5WvnjYggHWfzlpiMaMUblgjzaE5MPydM0aOCLV/c=", + "h1:OZFGLNoSydPR9FGVZop9704nJRZRjM65FRq6RmZIFFM=", "zh:1a5d0cd333f66aa0df26000e945df7678b045a8740921cc08f3e74412893d4f4", "zh:29d9728b0195d8e16dd31816d75a5ed069b0524d5e50b04a6533f0b9177895f7", "zh:2da9a3ecc0d5e3a66cb0cb5001b0c853ae22490204cc15fa14c44bbb848fcbc6", @@ -27,6 +28,7 @@ provider "registry.terraform.io/hashicorp/http" { version = "3.4.0" constraints = "3.4.0" hashes = [ + "h1:AaRLrzxA1t02OIwO32uLp85npqRLZSwPFgrHxb9qp0c=", "h1:m0d6+9xK/9TJSE9Z6nM4IwHXZgod4/jkdsf7CZSpUvo=", "zh:56712497a87bc4e91bbaf1a5a2be4b3f9cfa2384baeb20fc9fad0aff8f063914", "zh:6661355e1090ebacab16a40ede35b029caffc279d67da73a000b6eecf0b58eba", From c726670d9a3705d53d34d96d8cf07de4fbd75546 Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Fri, 26 Jan 2024 10:53:08 +0000 Subject: [PATCH 09/11] Add data.gov.uk to www.data.gov.uk redirect location header Migrated from: https://github.com/alphagov/govuk-aws/blob/5364552f8e334b81fe87062c938708389c471591/terraform/projects/fastly-datagovuk/main.tf#L127-L134 --- modules/datagovuk/service.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/datagovuk/service.tf b/modules/datagovuk/service.tf index fb71832..da8b37b 100644 --- a/modules/datagovuk/service.tf +++ b/modules/datagovuk/service.tf @@ -97,6 +97,15 @@ resource "fastly_service_vcl" "service" { } } + header { + name = "${local.template_values["environment"]}.data.gov.uk to www.${local.template_values["environment"]}.data.gov.uk redirect location header" + action = "set" + type = "response" + destination = "http.Location" + source = "\"https://www.${local.template_values["environment"]}.data.gov.uk\" + req.url" + response_condition = "${local.template_values["environment"]}.data.gov.uk to www.${local.template_values["environment"]}.data.gov.uk redirect response condition" + } + dynamic "logging_splunk" { for_each = { for splunk in lookup(var.secrets, "splunk", []) : splunk.name => splunk From 005a0f0905b8ec8cf0d6d4a85d00b65704f12ae9 Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Fri, 26 Jan 2024 10:54:03 +0000 Subject: [PATCH 10/11] Add Force TLS request setting for DGU Migrated from: https://github.com/alphagov/govuk-aws/blob/5364552f8e334b81fe87062c938708389c471591/terraform/projects/fastly-datagovuk/main.tf#L85-L88 --- modules/datagovuk/service.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/datagovuk/service.tf b/modules/datagovuk/service.tf index da8b37b..aebb397 100644 --- a/modules/datagovuk/service.tf +++ b/modules/datagovuk/service.tf @@ -106,6 +106,11 @@ resource "fastly_service_vcl" "service" { response_condition = "${local.template_values["environment"]}.data.gov.uk to www.${local.template_values["environment"]}.data.gov.uk redirect response condition" } + request_setting { + name = "Force TLS" + force_ssl = true + } + dynamic "logging_splunk" { for_each = { for splunk in lookup(var.secrets, "splunk", []) : splunk.name => splunk From a9c58934c0bd6a55fee744476fa8f525087f53de Mon Sep 17 00:00:00 2001 From: Aga Dufrat Date: Fri, 26 Jan 2024 11:01:24 +0000 Subject: [PATCH 11/11] Add $ENV to www.$ENV redirect synthetic response for DGU Migrated from https://github.com/alphagov/govuk-aws/blob/5364552f8e334b81fe87062c938708389c471591/terraform/projects/fastly-datagovuk/main.tf#L115-L119 --- modules/datagovuk/service.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/datagovuk/service.tf b/modules/datagovuk/service.tf index aebb397..9a430d8 100644 --- a/modules/datagovuk/service.tf +++ b/modules/datagovuk/service.tf @@ -106,6 +106,12 @@ resource "fastly_service_vcl" "service" { response_condition = "${local.template_values["environment"]}.data.gov.uk to www.${local.template_values["environment"]}.data.gov.uk redirect response condition" } + response_object { + name = "${local.template_values["environment"]}.data.gov.uk to www.${local.template_values["environment"]}.data.gov.uk redirect synthetic response" + status = 301 + request_condition = "${local.template_values["environment"]}.data.gov.uk to www.${local.template_values["environment"]}.data.gov.uk redirect request condition" + } + request_setting { name = "Force TLS" force_ssl = true