Merge pull request #1133 from wellcomecollection/Ingest-inspector-sub…

…domain

Make Ingest Inspector accessible from a wellcomecollection.org subdomain

.github/workflows/deploy-ingest-inspector.yml

@@ -0,0 +1,29 @@
+name: Deploy Ingest Inspector
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  id-token: write
+
+jobs:
+  deploy_ingest_inspector:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-west-1
+          role-to-assume: ${{ secrets.DEPLOY_INGEST_INSPECTOR_GHA_ROLE_ARN }}
+      - name: Deploy Ingest Inspector backend
+        run: |
+          pip3 install -r ./builds/requirements.txt
+          python3 ./builds/publish_lambda_zip.py monitoring/ingest_inspector/backend_lambda \
+              --bucket=wellcomecollection-storage-infra \
+              --key=lambdas/monitoring/ingest_inspector_backend.zip
+          ./builds/deploy_lambda_zip.sh monitoring/ingest_inspector_backend ingest_inspector_backend
+      - name: Deploy Ingest Inspector frontend
+        run: |
+          cd monitoring/ingest_inspector/frontend
+          sh deploy.sh

builds/deploy_lambda_zip.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Usage: ./deploy_lambda_zip.sh <project_name> <lambda_name>
+# Example: ./deploy_lambda_zip.sh monitoring/ingest_inspector_backend ingest_inspector_backend
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+PROJECT_NAME=$1
+LAMBDA_NAME=$2
+
+S3_BUCKET="wellcomecollection-storage-infra"
+S3_KEY="lambdas/$PROJECT_NAME.zip"
+
+echo "Identifying function: $LAMBDA_NAME"
+FUNCTION_ARN=$(aws lambda get-function-configuration \
+    --function-name $LAMBDA_NAME \
+    --query "FunctionArn" \
+    --output text)
+
+echo "Updating function: $FUNCTION_ARN from s3://$S3_BUCKET/$S3_KEY"
+REVISION_ID=$(aws lambda update-function-code \
+    --function-name $LAMBDA_NAME \
+    --s3-bucket $S3_BUCKET \
+    --s3-key $S3_KEY \
+    --query "RevisionId" \
+    --output text)
+
+echo "Revision id: $REVISION_ID"
+
+echo "Awaiting function update"
+aws lambda wait function-updated \
+    --function-name $LAMBDA_NAME
+
+echo "Done"

builds/requirements.in

@@ -0,0 +1,2 @@
+boto3
+docopt

builds/requirements.txt

@@ -0,0 +1,26 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+boto3==1.34.140
+    # via -r requirements.in
+botocore==1.34.140
+    # via
+    #   boto3
+    #   s3transfer
+docopt==0.6.2
+    # via -r requirements.in
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+python-dateutil==2.9.0.post0
+    # via botocore
+s3transfer==0.10.2
+    # via boto3
+six==1.16.0
+    # via python-dateutil
+urllib3==2.2.2
+    # via botocore

monitoring/daily_reporter/src/slack.py

@@ -29,7 +29,7 @@ def _get_slack_message(label, ingests):
         if ingests.get(status, []):
             result += "\n" + status.title() + ":"
             for i in ingests[status][:15]:
-                result += f"\n- <https://wellcome-ingest-inspector.glitch.me/ingests/{i['id']}|`{i['id']}`> – {i['space']}/{i['externalIdentifier']}"
+                result += f"\n- <https://ingest-inspector.wellcomecollection.org/ingests/{i['id']}|`{i['id']}`> – {i['space']}/{i['externalIdentifier']}"
                 if i["version"]:
                     result += "/" + i["version"]
 

monitoring/daily_reporter/src/templates/report.html

@@ -169,7 +169,7 @@
                   {% for ingest in ingests_by_status[label].get(classification) | sort(attribute="lastModifiedDate", reverse=True) %}
                   <tr class="status-{{ ingest.status }}">
                     <td class="ingest_id">
-                      <a href="https://wellcome-ingest-inspector.glitch.me/ingests/{{ ingest.id }}">{{ ingest.id }}</a></td>
+                      <a href="https://ingest-inspector.wellcomecollection.org/ingests/{{ ingest.id }}">{{ ingest.id }}</a></td>
                     <td>{{ ingest.space }}</td>
                     <td>{{ ingest.externalIdentifier }}</td>
                     <td>{{ ingest.version or "(no version)" }}</td>

monitoring/end_to_end_bag_test/src/end_to_end_bag_test.py

@@ -53,7 +53,7 @@ def main(*args):
     print(ingest_location)
 
     ingest_id = ingest_location.split("/")[-1]
-    return f"https://wellcome-ingest-inspector.glitch.me/ingests/{ingest_id}"
+    return f"https://ingest-inspector.wellcomecollection.org/ingests/{ingest_id}"
 
 
 if __name__ == "__main__":

monitoring/ingest_inspector/README.md

@@ -132,7 +132,7 @@ The app has a number of features designed to make it as easy as possible for som
 
 ## Usage
 
-You can use the app by visiting <https://d1dti1kbjcy26.cloudfront.net>.
+You can use the app by visiting <https://ingest-inspector.wellcomecollection.org/>.
 
 ## Development
 ### Backend
@@ -164,6 +164,6 @@ The frontend is hosted as a static website on S3 and served via CloudFront. To b
 the `ingest_inspector/frontend` directory and run:
 
 ```shell
-sh deploy.sh
+AWS_PROFILE=storage-developer sh deploy.sh
 ```
 

monitoring/ingest_inspector/backend_lambda/src/ingest_inspector_backend.py

@@ -1,6 +1,7 @@
 import argparse
 import json
 import functools
+import os
 
 import boto3
 from wellcome_storage_service import (
@@ -18,15 +19,18 @@
 STAGING_URL = "https://api-stage.wellcomecollection.org/storage/v1"
 PRODUCTION_URL = "https://api.wellcomecollection.org/storage/v1"
 
+COGNITO_CLIENT_ID_SECRET_NAME = os.environ["COGNITO_CLIENT_ID_SECRET_NAME"]
+COGNITO_CLIENT_SECRET_SECRET_NAME = os.environ["COGNITO_CLIENT_SECRET_SECRET_NAME"]
+
 
 def _client_from_environment(api_url):
     secretsmanager = boto3.Session().client("secretsmanager")
 
     def _get_secretsmanager_value(secret_id: str):
         return secretsmanager.get_secret_value(SecretId=secret_id)["SecretString"]
 
-    client_id = _get_secretsmanager_value("ingest-inspector/cognito-client-id")
-    client_secret = _get_secretsmanager_value("ingest-inspector/cognito-client-secret")
+    client_id = _get_secretsmanager_value(COGNITO_CLIENT_ID_SECRET_NAME)
+    client_secret = _get_secretsmanager_value(COGNITO_CLIENT_SECRET_SECRET_NAME)
 
     return RequestsOAuthStorageServiceClient(
         api_url=api_url,

monitoring/ingest_inspector/frontend/deploy.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+echo "Installing packages..."
+npm install
+
 echo "Building app..."
 npm run build
 
@@ -9,7 +12,7 @@ if [ $? -ne 0 ]; then
 fi
 
 echo "Uploading to S3..."
-AWS_PROFILE=storage-developer aws s3 cp out s3://wellcomecollection-ingest-inspector-frontend --recursive --only-show-errors
+aws s3 cp out s3://wellcomecollection-ingest-inspector-frontend --recursive --only-show-errors
 
 if [ $? -eq 0 ]; then
   echo "Success!"

monitoring/ingest_inspector/frontend/src/app/components/Form.tsx

@@ -19,7 +19,7 @@ export const Form = ({ defaultIngestId }: FormProps) => {
     const ingestId = data.get("ingest-id") as string;
 
     if (ingestId?.length > 0) {
-      router.push(`/?ingestId=${ingestId}`);
+      router.push(`/?ingestId=${encodeURIComponent(ingestId)}`);
     }
   };
 

monitoring/ingest_inspector/frontend/src/app/components/IngestError.tsx

@@ -34,7 +34,7 @@ const IngestError = ({ ingestId, error }: IngestErrorProps) => {
 
   return (
     <div className="mt-12">
-      <h3 className="text-2xl">{errorMessage}</h3>
+      <h3 className="text-2xl break-all">{errorMessage}</h3>
       <p className="mt-4 text-lg">
         Developers can{" "}
         <a href={KIBANA_ERROR_URL} target="_blank" rel="noreferrer">

monitoring/ingest_inspector/frontend/src/app/hooks/useGetIngest.ts

@@ -5,7 +5,7 @@ import NProgress from "nprogress";
 import { IngestData } from "@/app/types";
 
 const BASE_API_URL =
-  "https://xkgpnijmy5.execute-api.eu-west-1.amazonaws.com/v1/ingest";
+  "https://gzz79crkhl.execute-api.eu-west-1.amazonaws.com/v1/ingest";
 
 export const APIErrors = {
   INVALID_INGEST_ID: "Invalid ingest ID.",

scripts/ss_get_ingest.py

@@ -7,7 +7,7 @@
 The script will attempt to find the ingest ID in both the prod and staging APIs.
 
 For most use cases, you can use the web inspector:
-https://wellcome-ingest-inspector.glitch.me/
+https://ingest-inspector.wellcomecollection.org
 
 This script is useful if you need to see the raw JSON response direct from
 the ingests API.  We don't expose the raw JSON online -- it's publicly visible,

scripts/ss_retry_unpacking.py

@@ -65,4 +65,6 @@ def create_context(ingest):
     resp = sns.publish(TopicArn=topic_arn, Message=json.dumps(payload))
     print(resp)
 
-    webbrowser.open(f"https://wellcome-ingest-inspector.glitch.me/ingests/{ingest_id}")
+    webbrowser.open(
+        f"https://ingest-inspector.wellcomecollection.org/ingests/{ingest_id}"
+    )

terraform/app_clients/clients.tf

@@ -38,16 +38,6 @@ module "dds_client" {
   generate_secret = null
 }
 
-module "alex_glitch_dashboard" {
-  source = "../modules/app_client"
-
-  name         = "Alex Glitch dashboard"
-  user_pool_id = aws_cognito_user_pool.pool.id
-
-  allow_bags_access    = false
-  allow_ingests_access = true
-}
-
 module "ingest_inspector_dashboard" {
   source = "../modules/app_client"
 

terraform/monitoring/ingest_inspector.tf

@@ -0,0 +1,6 @@
+module "ingest_inspector" {
+  source = "./ingest_inspector"
+
+  domain_name                = "ingest-inspector.wellcomecollection.org"
+  serve_frontend_bucket_name = "wellcomecollection-ingest-inspector-frontend"
+}

terraform/monitoring/ingest_inspector/certificate.tf

@@ -0,0 +1,12 @@
+module "ingest_inspector_certificate" {
+  source = "github.com/wellcomecollection/terraform-aws-acm-certificate?ref=v1.0.0"
+
+  domain_name = var.domain_name
+  zone_id     = data.aws_route53_zone.weco_zone.id
+
+  providers = {
+    # The ACM certificate must be created in the us-east-1 region to work with CloudFront
+    aws     = aws.us-east-1
+    aws.dns = aws.dns
+  }
+}

terraform/monitoring/ingest_inspector/data.tf

@@ -0,0 +1,4 @@
+data "aws_route53_zone" "weco_zone" {
+  provider = aws.dns
+  name     = "wellcomecollection.org."
+}

terraform/monitoring/ingest_inspector/dns.tf

@@ -0,0 +1,14 @@
+# Add an alias A record to the wellcomecollection.org hosted zone, which maps the Ingest Inspector domain name
+# to the CloudFront distribution
+resource "aws_route53_record" "cdn" {
+  provider = aws.dns
+  zone_id  = data.aws_route53_zone.weco_zone.id
+  name     = var.domain_name
+  type     = "A"
+
+  alias {
+    name                   = aws_cloudfront_distribution.ingest_inspector_cloudfront_distribution.domain_name
+    zone_id                = aws_cloudfront_distribution.ingest_inspector_cloudfront_distribution.hosted_zone_id
+    evaluate_target_health = false
+  }
+}

terraform/monitoring/ingest_inspector_lambda.tf → terraform/monitoring/ingest_inspector/lambda_backend.tf

@@ -20,7 +20,10 @@ module "ingest_inspector_lambda" {
 
   environment = {
     variables = {
-      "HOME" = "dummy_value"
+      # The wellcome_storage_service Python package will not successfully import unless there is a HOME environment variable
+      "HOME"                              = "dummy_value"
+      "COGNITO_CLIENT_ID_SECRET_NAME"     = aws_secretsmanager_secret.ingest_inspector_cognito_client_id.name
+      "COGNITO_CLIENT_SECRET_SECRET_NAME" = aws_secretsmanager_secret.ingest_inspector_cognito_client_secret.name
     }
   }
 

terraform/monitoring/ingest_inspector/provider.tf

@@ -0,0 +1,17 @@
+provider "aws" {
+  region = "eu-west-1"
+  alias  = "dns"
+
+  assume_role {
+    role_arn = "arn:aws:iam::267269328833:role/wellcomecollection-assume_role_hosted_zone_update"
+  }
+}
+
+provider "aws" {
+  assume_role {
+    role_arn = "arn:aws:iam::975596993436:role/storage-developer"
+  }
+
+  region = "us-east-1"
+  alias  = "us-east-1"
+}

terraform/monitoring/ingest_inspector_frontend.tf → terraform/monitoring/ingest_inspector/s3_cloudfront_frontend.tf

@@ -1,5 +1,5 @@
 resource "aws_s3_bucket" "ingest_inspector_frontend" {
-  bucket = "wellcomecollection-ingest-inspector-frontend"
+  bucket = var.serve_frontend_bucket_name
 }
 
 locals {
@@ -17,6 +17,8 @@ resource "aws_cloudfront_distribution" "ingest_inspector_cloudfront_distribution
     origin_access_control_id = aws_cloudfront_origin_access_control.ingest_inspector_oac.id
   }
 
+  aliases = [var.domain_name]
+
   default_cache_behavior {
     target_origin_id = local.s3_origin_id
     allowed_methods  = ["GET", "HEAD"]
@@ -43,7 +45,9 @@ resource "aws_cloudfront_distribution" "ingest_inspector_cloudfront_distribution
   }
 
   viewer_certificate {
-    cloudfront_default_certificate = true
+    acm_certificate_arn      = module.ingest_inspector_certificate.arn
+    ssl_support_method       = "sni-only"
+    minimum_protocol_version = "TLSv1.2_2021"
   }
 
   price_class = "PriceClass_100"

terraform/monitoring/ingest_inspector/secrets.tf

@@ -0,0 +1,8 @@
+resource "aws_secretsmanager_secret" "ingest_inspector_cognito_client_id" {
+  name = "ingest-inspector-backend/cognito-client-id"
+}
+
+resource "aws_secretsmanager_secret" "ingest_inspector_cognito_client_secret" {
+  name = "ingest-inspector-backend/cognito-client-secret"
+}
+

terraform/monitoring/ingest_inspector/variables.tf

@@ -0,0 +1,7 @@
+variable "domain_name" {
+  type = string
+}
+
+variable "serve_frontend_bucket_name" {
+  type = string
+}

terraform/monitoring/secrets.tf

@@ -10,13 +10,3 @@ module "reporting_secrets" {
   }
 }
 
-resource "aws_secretsmanager_secret" "ingest_inspector_cognito_client_id" {
-  name = "ingest-inspector/cognito-client-id"
-}
-
-resource "aws_secretsmanager_secret" "ingest_inspector_cognito_client_secret" {
-  name = "ingest-inspector/cognito-client-secret"
-}
-
-
-