Roles for our pathogen-repo-build GitHub Actions workflow, take 3

README.md

@@ -29,6 +29,9 @@ Make those changes so:
 > You'll need ambiently-configured AWS credentials with broad admin-level
 > access to read (and optionally modify) resources in our account.
 >
+> You'll also need a `GITHUB_TOKEN` in the environment with the `actions:write`
+> fine-grained token permission on our repos.
+>
 > Please step cautiously and be careful when using them!
 
 

env/production/aws-iam-policy-NextstrainPathogen@.tf

@@ -0,0 +1,75 @@
+# Per-pathogen policy, granting access to a single pathogen's data
+resource "aws_iam_policy" "NextstrainPathogen" {
+  for_each = local.pathogen_repos
+
+  name = "NextstrainPathogen@${each.key}"
+  description = "Provides permissions to upload datasets, workflow files, etc. for a Nextstrain pathogen"
+
+  policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+      # Technically we don't need to include the public buckets
+      # nextstrain-data and nextstrain-staging in this statement since they
+      # already allow a superset of this with their bucket policies, but it's
+      # good to be explicit about what permissions we require.
+      #   -trs, 16 Feb 2024
+      {
+        "Sid": "List",
+        "Effect": "Allow",
+        "Action": [
+          "s3:ListBucket",
+          "s3:ListBucketVersions",
+          "s3:GetBucketLocation",
+          "s3:GetBucketVersioning",
+        ],
+        "Resource": [
+          "arn:aws:s3:::nextstrain-data",
+          "arn:aws:s3:::nextstrain-data-private",
+          "arn:aws:s3:::nextstrain-staging",
+        ],
+        "Condition": {
+          "StringLike": {
+            "s3:prefix": [
+              "${each.key}.json",
+              "${each.key}_*.json",
+              "files/workflows/${each.key}/*",
+              "files/datasets/${each.key}/*",
+            ]
+          }
+        }
+      },
+      {
+        "Sid": "ReadWrite",
+        "Effect": "Allow",
+        "Action": [
+          "s3:GetObject",
+          "s3:GetObjectTagging",
+          "s3:GetObjectVersion",
+          "s3:GetObjectVersionTagging",
+          "s3:PutObject",
+          "s3:PutObjectTagging",
+          "s3:DeleteObject",
+          # but NOT s3:DeleteObjectVersion so objects can't be completely wiped
+        ],
+        "Resource": [
+          # Auspice dataset JSONs
+          "arn:aws:s3:::nextstrain-data/${each.key}.json",
+          "arn:aws:s3:::nextstrain-data/${each.key}_*.json",
+          "arn:aws:s3:::nextstrain-staging/${each.key}.json",
+          "arn:aws:s3:::nextstrain-staging/${each.key}_*.json",
+          "arn:aws:s3:::nextstrain-staging/trial_*_${each.key}.json",
+          "arn:aws:s3:::nextstrain-staging/trial_*_${each.key}_*.json",
+
+          # Associated data files
+          # <https://docs.nextstrain.org/en/latest/reference/data-files.html>
+          "arn:aws:s3:::nextstrain-data/files/workflows/${each.key}/*",
+          "arn:aws:s3:::nextstrain-data/files/datasets/${each.key}/*",
+          "arn:aws:s3:::nextstrain-data-private/files/workflows/${each.key}/*",
+          "arn:aws:s3:::nextstrain-data-private/files/datasets/${each.key}/*",
+          "arn:aws:s3:::nextstrain-staging/files/workflows/${each.key}/*",
+          "arn:aws:s3:::nextstrain-staging/files/datasets/${each.key}/*",
+        ],
+      },
+    ]
+  })
+}

env/production/aws-iam-policy-NextstrainPathogenNcovPrivate.tf

@@ -0,0 +1,43 @@
+# Single-pathogen policy, special-case for the historical reason that
+# nextstrain-ncov-private predates the more general nextstrain-data-private.
+resource "aws_iam_policy" "NextstrainPathogenNcovPrivate" {
+  name = "NextstrainPathogen@ncov+private"
+  description = "Provides permissions to upload datasets, workflow files, etc. to the ncov-private bucket for the Nextstrain ncov pathogen"
+
+  policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Sid": "NcovPrivateList",
+        "Effect": "Allow",
+        "Action": [
+          "s3:ListBucket",
+          "s3:ListBucketVersions",
+          "s3:GetBucketLocation",
+          "s3:GetBucketVersioning",
+        ],
+        "Resource": [
+          "arn:aws:s3:::nextstrain-ncov-private",
+        ],
+      },
+      {
+        "Sid": "NcovPrivateReadWrite",
+        "Effect": "Allow",
+        "Action": [
+          "s3:GetObject",
+          "s3:GetObjectTagging",
+          "s3:GetObjectVersion",
+          "s3:GetObjectVersionTagging",
+          "s3:PutObject",
+          "s3:PutObjectTagging",
+          "s3:DeleteObject",
+          # but NOT s3:DeleteObjectVersion so objects can't be completely wiped
+        ],
+        "Resource": [
+          # This bucket is akin to nextstrain-data-private/files/{workflows,datasets}/ncov/.
+          "arn:aws:s3:::nextstrain-ncov-private/*",
+        ],
+      },
+    ]
+  })
+}

env/production/aws-iam-role-GitHubActionsRoleNextstrainBatchJobs.tf

@@ -3,9 +3,10 @@ import {
   id = "GitHubActionsRoleNextstrainBatchJobs"
 }
 
+# Multi-repo role, granting access to Batch
 resource "aws_iam_role" "GitHubActionsRoleNextstrainBatchJobs" {
   name = "GitHubActionsRoleNextstrainBatchJobs"
-  description = "Provides permissions to run jobs on AWS Batch via the Nextstrain CLI to select GitHub Actions OIDC workflows."
+  description = "Provides permissions to launch and monitor jobs on AWS Batch via the Nextstrain CLI to select GitHub Actions OIDC workflows."
 
   max_session_duration = 43200 # seconds (12 hours)
 
@@ -21,7 +22,10 @@ resource "aws_iam_role" "GitHubActionsRoleNextstrainBatchJobs" {
         "Condition": {
           "StringLike": {
             "token.actions.githubusercontent.com:aud": "sts.amazonaws.com",
-            "token.actions.githubusercontent.com:sub": "repo:nextstrain/.github:*"
+            "token.actions.githubusercontent.com:sub": [
+              for repo in keys(local.repo_pathogens):
+                "repo:nextstrain/${repo}:*:job_workflow_ref:nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@*"
+            ]
           }
         },
       }

env/production/aws-iam-role-GitHubActionsRoleNextstrainRepo@.tf

@@ -0,0 +1,43 @@
+# Per-repo role, granting access to pathogens
+resource "aws_iam_role" "GitHubActionsRoleNextstrainRepo" {
+  for_each = local.repo_pathogens
+
+  name = "GitHubActionsRoleNextstrainRepo@${each.key}"
+  description = "Provides permissions to upload datasets, workflow files, etc. for a Nextstrain pathogen to select repos and select GitHub Actions OIDC workflows."
+
+  max_session_duration = 43200 # seconds (12 hours)
+
+  assume_role_policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Effect": "Allow",
+        "Principal": {
+          "Federated": aws_iam_openid_connect_provider.github-actions.arn
+        },
+        "Action": "sts:AssumeRoleWithWebIdentity",
+        "Condition": {
+          "StringLike": {
+            "token.actions.githubusercontent.com:aud": "sts.amazonaws.com",
+            "token.actions.githubusercontent.com:sub": "repo:nextstrain/${each.key}:*:job_workflow_ref:nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@*"
+          }
+        },
+      }
+    ]
+  })
+
+  managed_policy_arns = flatten([
+    # Pathogen-specific permissions to standard public/private buckets
+    [for p in each.value: aws_iam_policy.NextstrainPathogen[p].arn],
+
+    # Special-case permissions to nextstrain-ncov-private bucket
+    contains(each.value, "ncov")
+      ? [aws_iam_policy.NextstrainPathogenNcovPrivate.arn]
+      : [],
+
+    # Builds inside the AWS Batch runtime need access to the jobs bucket.
+    aws_iam_policy.NextstrainJobsAccessToBucket.arn,
+  ])
+
+  inline_policy {}
+}

env/production/github-oidc.tf

@@ -0,0 +1,15 @@
+resource "github_actions_repository_oidc_subject_claim_customization_template" "nextstrain" {
+  for_each = toset(keys(local.repo_pathogens))
+  repository = each.key
+
+  # <https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect>
+  use_default = false
+  include_claim_keys = [
+    # The GitHub default…
+    "repo",
+    "context",
+
+    # …plus the <org>/<repo>/<path>@<ref> of the workflow obtaining the token, if any.
+    "job_workflow_ref",
+  ]
+}

env/production/locals.tf

@@ -0,0 +1,32 @@
+locals {
+  # By design our repo names are usually equal to the pathogen names, but
+  # they're two separate things/namespaces and linkages don't always line up
+  # 1:1.  Some resources (roles, policies, etc) are more naturally oriented
+  # per-pathogen (logical), some per-repo (physical).  Use two maps to support
+  # this more easily.  This will likely evolve in the future to better support
+  # our needs.
+  #   -trs, 20 May 2024
+
+  pathogen_repos = tomap({
+    # pathogen name   = [repo name, …]
+    "dengue"          = ["dengue"],
+    "forecasts-ncov"  = ["forecasts-ncov"],
+    "measles"         = ["measles"],
+    "mpox"            = ["mpox"],
+    "ncov"            = ["ncov", "ncov-ingest"],
+    "rsv"             = ["rsv"],
+    "seasonal-flu"    = ["seasonal-flu"],
+    "zika"            = ["zika"],
+  })
+
+  repo_pathogens = merge(
+    # repo name = [pathogen name, …]
+    transpose(local.pathogen_repos),
+
+    tomap({
+      # For testing.  Ensures a role exists but without any pathogen-specific
+      # permissions.
+      ".github" = [],
+    }),
+  )
+}

env/production/terraform.tf

@@ -5,6 +5,10 @@ terraform {
       source  = "registry.terraform.io/hashicorp/aws"
       version = "~> 4.32"
     }
+    github = {
+      source  = "integrations/github"
+      version = "~> 6.0"
+    }
   }
 
   backend "s3" {
@@ -25,3 +29,8 @@ terraform {
 provider "aws" {
   region = "us-east-1"
 }
+
+provider "github" {
+  # Authn is via GITHUB_TOKEN
+  owner = "nextstrain"
+}