Load static assets from S3 directly

Description:
- Static assets (such as CSS, images and Javascript) are stored in an S3 bucket. Currently to retrieve a static asset Fastly will make a request to an nginx pod running in router. nginx will then retrieve the static asset from S3 before passing it to Fastly.
- This commit makes Fastly talk directly to S3 without going through nginx and Router by adding a new backend for Fastly to talk to
- Currently if the asset requested doesn’t exist in the S3 bucket then S3 will return a 403 which nginx maps to a static 403 error page stored under the /error_pages prefix in S3. Status codes between 500-599 are handled in the if block above which triggers a restart causing Fastly to poll the S3 mirror instead. Any other non-200 status code is handled by the new if block which also sets up a restart so that any subsequent requests (e.g. req.restarts > 0) will go to the mirror
- See the diagram in #73 for the previous method and the proposed new implementation

modules/www/service.tf

@@ -14,6 +14,10 @@ locals {
       ssl_ciphers          = "ECDHE-RSA-AES256-GCM-SHA384"
       basic_authentication = null
 
+      s3_static_assets_port      = 443
+      s3_static_assets_hostname  = null
+      s3_static_assets_probe     = null
+
       # these values are needed even if mirrors aren't enabled in an environment
       s3_mirror_hostname         = null
       s3_mirror_prefix           = null

modules/www/www.vcl.tftpl

@@ -36,6 +36,38 @@ backend F_origin {
 %{ endif ~}
 }
 
+backend F_staticAssetsS3 {
+    .connect_timeout = 1s;
+    .dynamic = true;
+    .port = "${s3_static_assets_port}";
+    .host = "${s3_static_assets_hostname}";
+    .first_byte_timeout = 15s;
+    .max_connections = 200;
+    .between_bytes_timeout = 10s;
+
+    .ssl = true;
+    .ssl_check_cert = always;
+    .min_tls_version = "${minimum_tls_version}";
+    .ssl_ciphers = "${ssl_ciphers}";
+    .ssl_cert_hostname = "${s3_static_assets_hostname}";
+    .ssl_sni_hostname = "${s3_static_assets_hostname}";
+
+    .probe = {
+        .dummy = ${probe_dns_only};
+        .request =
+            "HEAD ${s3_static_assets_probe} HTTP/1.1"
+            "Host: ${s3_static_assets_hostname}"
+            "User-Agent: Fastly healthcheck (Git commit: ${git_hash})"
+            "Connection: close";
+        .threshold = 1;
+        .window = 2;
+        .timeout = 5s;
+        .initial = 1;
+        .expected_response = 200;
+        .interval = ${probe_interval};
+    }
+}
+
 %{ if contains(["staging", "production"], environment) ~}
 # Mirror backend for S3
 backend F_mirrorS3 {
@@ -342,6 +374,12 @@ sub vcl_recv {
     unset req.http.Cookie;
   }
 
+  if (req.url.path ~ "^\/assets(\/.*)?\z") {
+    set req.backend = F_staticAssetsS3;
+    set req.http.host = "${s3_static_assets_hostname}";
+    set req.http.Fastly-Backend-Name = "staticAssetsS3";
+  }
+
   return(lookup);
 }
 
@@ -377,6 +415,12 @@ sub vcl_fetch {
     set beresp.stale_if_error = 24h;
   }
 
+  # If the status code isn't 200 and not within 500-599 then retry and fall back to the mirrors
+  if (req.restarts == 0 && req.status != 200 && beresp.http.Fastly-Backend-Name ~ "^staticAssetsS3$") {
+    set beresp.saintmode = 5s;
+    return (restart);
+  }
+
   if(req.restarts > 0 ) {
     set beresp.http.Fastly-Restarts = req.restarts;
   }
@@ -433,7 +477,24 @@ sub vcl_fetch {
       set beresp.ttl = 900s;
       set beresp.http.Cache-Control = "max-age=900";
     }
+
+    # Static Assets S3 bucket do not set cache headers by default. Override TTL and add cache-control to 31536000s.
+    # Strip out common S3 headers
+    if (beresp.http.Fastly-Backend-Name ~ "^staticAssetsS3$") {
+      set beresp.ttl = 31536000s;
+      set beresp.http.Cache-Control = "max-age=31536000, public, immutable";
+      set beresp.http.Access-Control-Allow-Origin = "*";
+      set beresp.http.Access-Control-Allow-Methods = "GET, OPTIONS";
+      set beresp.http.Access-Control-Allow-Headers = "origin";
+
+      unset beresp.http.x-amz-id2;
+      unset beresp.http.x-amz-meta-server-side-encryption;
+      unset beresp.http.x-amz-request-id;
+      unset beresp.http.x-amz-version-id;
+      unset beresp.http.x-amz-server-side-encryption;
+    }
   }
+
   # Strip cookies from outbound requests. Corresponding rule in vcl_recv{}
   if (req.url !~ "^/(apply-for-a-licence|email|sign-in/callback)") {
     unset beresp.http.Set-Cookie;