Merge pull request 2i2c-org#3431 from yuvipanda/req-pay

Clarify what the requestor_pays flag does

docs/howto/features/cloud-access.md

@@ -36,28 +36,50 @@ This AWS IAM Role is managed via terraform.
 ## Enabling specific cloud access permissions
 
 1. In the `.tfvars` file for the project in which this hub is based off
-   create (or modify) the `hub_cloud_permissions` variable. The config is
-   like:
+   create (or modify) the `hub_cloud_permissions` variable.
 
+   ```{warning}
+   `allow_access_to_external_requester_pays_buckets` is not yet supported on AWS!
    ```
+
+   The config is like:
+
+   `````{tab-set}
+   ````{tab-item} GCP
+   :sync: gcp-key
+   ```yaml
+   hub_cloud_permissions = {
+       "<hub-name-slug>": {
+           allow_access_to_external_requester_pays_buckets : true,
+           bucket_admin_access : ["bucket-1", "bucket-2"]
+           hub_namespace : "<hub-name>"
+       }
+   }
+   ```
+   ````
+
+   ````{tab-item} AWS
+   :sync: aws-key
+   ```bash
    hub_cloud_permissions = {
        "<hub-name-slug>": {
-           requestor_pays : true,
            bucket_admin_access : ["bucket-1", "bucket-2"]
            hub_namespace : "<hub-name>"
        }
    }
    ```
+   ````
+   `````
 
    where:
 
    1. `<hub-name-slug>` is the name of the hub, but restricted in length. This
       and the cluster name together can't be more than 29 characters. `terraform`
       will complain if you go over this limit, so in general just use the name
       of the hub and shorten it only if `terraform` complains.
-   2. (GCP only) `requestor_pays` enables permissions for user pods and dask worker
-      pods to identify as the project while making requests to Google Cloud Storage
-      buckets marked as 'requestor pays'. More details [here](topic:features:cloud:gcp:requestor-pays).
+   2. (GCP only) `allow_access_to_external_requester_pays_buckets` enables permissions for user pods and dask worker
+      pods to identify as the project while making requests to other Google Cloud Storage
+      buckets, outside of this project, that have 'Requester Pays' enabled. More details [here](topic:features:cloud:gcp:requester-pays).
    3. `bucket_admin_access` lists bucket names (as specified in `user_buckets`
       terraform variable) all users on this hub should have full read/write
       access to. Used along with the [user_buckets](howto:features:storage-buckets)

docs/topic/features.md

@@ -23,8 +23,8 @@ improving the security posture of our hubs.
 
 ### GCP
 
-(topic:features:cloud:gcp:requestor-pays)=
-#### 'Requestor Pays' access to Google Cloud Storage buckets
+(topic:features:cloud:gcp:requester-pays)=
+#### 'Requester Pays' access
 
 By default, the organization *hosting* data on Google Cloud pays for both
 storage and bandwidth costs of the data. However, Google Cloud also offers
@@ -33,9 +33,29 @@ option, where the bandwidth costs are paid for by the organization *requesting*
 the data. This is very commonly used by organizations that provide big datasets
 on Google Cloud storage, to sustainably share costs of maintaining the data.
 
+**Requester Pays** is a feature that a bucket can have.
+
+#### Allow access to external `Requester Payes` buckets
+
+If buckets outside the project have the `Requester Payes` flag, then we need to:
+- set `hub_cloud_permissions.allow_access_to_external_requester_pays_buckets`
+  in the terraform config of the cluster (see the guide at [](howto:features:cloud-access:access-perms))
+- this will allow them to be charged on their project for access of such
+  outside buckets
+
+```{warning}
 When this feature is enabled, users on a hub accessing cloud buckets from
-other organizations marked as 'requestor pays' will increase our cloud bill.
+other organizations marked as `Requester Pays` will increase our cloud bill.
 Hence, this is an opt-in feature.
+```
+
+#### Enable `Requester Pays` flag on community buckets
+
+The buckets that we set for communities, inside their projects can also have this flag enabled on them, which means that other people outside will be charged for their usage.
+
+```{warning}
+This is not supported yet by our terraform. Follow https://github.com/2i2c-org/infrastructure/issues/3746 to check when support will be added.
+```
 
 (topic:features:cloud:scratch-buckets)=
 ## 'Scratch' buckets on object storage

terraform/aws/projects/2i2c-aws-us.tfvars

@@ -31,35 +31,29 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "dask-staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-dask-staging"],
     extra_iam_policy : ""
   },
   "showcase" : {
-    requestor_pays : true,
     bucket_admin_access : [
       "scratch-researchdelight",
       "persistent-showcase"
     ],
     extra_iam_policy : ""
   },
   "ncar-cisl" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-ncar-cisl"],
     extra_iam_policy : ""
   },
   "go-bgc" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-go-bgc"],
     extra_iam_policy : ""
   },
   "itcoocean" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-itcoocean"],
     extra_iam_policy : ""
   },

terraform/aws/projects/catalystproject-africa.tfvars

@@ -16,12 +16,10 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : ""
   },

terraform/aws/projects/earthscope.tfvars

@@ -16,12 +16,10 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : ""
   },

terraform/aws/projects/gridsst.tfvars

@@ -16,12 +16,10 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : ""
   },

terraform/aws/projects/jupyter-meets-the-earth.tfvars

@@ -16,7 +16,6 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     # FIXME: Previously, users were granted full S3 permissions.
     # Keep it the same for now
@@ -34,7 +33,6 @@ hub_cloud_permissions = {
 EOT
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     # FIXME: Previously, users were granted full S3 permissions.
     # Keep it the same for now

terraform/aws/projects/nasa-cryo.tfvars

@@ -22,7 +22,6 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging", "persistent-staging"],
     # Provides readonly requestor-pays access to usgs-landsat bucket
     # FIXME: We should find a way to allow access to *all* requester pays
@@ -57,7 +56,6 @@ hub_cloud_permissions = {
   EOT
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch", "persistent"],
     # Provides readonly requestor-pays access to usgs-landsat bucket
     # FIXME: We should find a way to allow access to *all* requester pays

terraform/aws/projects/nasa-esdis.tfvars

@@ -16,12 +16,10 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : ""
   },

terraform/aws/projects/nasa-ghg.tfvars

@@ -16,7 +16,6 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : <<-EOT
     {
@@ -70,7 +69,6 @@ hub_cloud_permissions = {
   EOT
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : <<-EOT
     {

terraform/aws/projects/nasa-veda.tfvars

@@ -16,7 +16,6 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : <<-EOT
     {
@@ -75,7 +74,6 @@ hub_cloud_permissions = {
   EOT
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : <<-EOT
     {

terraform/aws/projects/openscapes.tfvars

@@ -19,12 +19,10 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : ""
   },

terraform/aws/projects/smithsonian.tfvars

@@ -13,12 +13,10 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : ""
   },

terraform/aws/projects/template.tfvars

@@ -16,12 +16,10 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : ""
   },

terraform/aws/projects/ubc-eoas.tfvars

@@ -16,12 +16,10 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : ""
   },

terraform/aws/projects/victor.tfvars

@@ -16,12 +16,10 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch-staging"],
     extra_iam_policy : ""
   },
   "prod" : {
-    requestor_pays : true,
     bucket_admin_access : ["scratch"],
     extra_iam_policy : ""
   },

terraform/aws/variables.tf

@@ -44,20 +44,20 @@ variable "user_buckets" {
 }
 
 variable "hub_cloud_permissions" {
-  type        = map(object({ requestor_pays : bool, bucket_admin_access : set(string), extra_iam_policy : string }))
+  type = map(object({
+    bucket_admin_access : set(string),
+    extra_iam_policy : string
+  }))
   default     = {}
   description = <<-EOT
   Map of cloud permissions given to a particular hub
 
   Key is name of the hub namespace in the cluster, and values are particular
   permissions users running on those hubs should have. Currently supported are:
 
-  1. requestor_pays: Identify as coming from the google cloud project when accessing
-     storage buckets marked as  https://cloud.google.com/storage/docs/requester-pays.
-     This *potentially* incurs cost for us, the originating project, so opt-in.
-  2. bucket_admin_access: List of S3 storage buckets that users on this hub should have read
+  1. bucket_admin_access: List of S3 storage buckets that users on this hub should have read
      and write permissions for.
-  3. extra_iam_policy: An AWS IAM Policy document that grants additional rights to the users
+  2. extra_iam_policy: An AWS IAM Policy document that grants additional rights to the users
      on this hub when talking to AWS services.
   EOT
 }

terraform/gcp/projects/awi-ciroh.tfvars

@@ -63,12 +63,12 @@ dask_nodes = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : false,
+    allow_access_to_external_requester_pays_buckets : false,
     bucket_admin_access : ["scratch-staging", "persistent-staging"],
     hub_namespace : "staging"
   },
   "prod" : {
-    requestor_pays : false,
+    allow_access_to_external_requester_pays_buckets : false,
     bucket_admin_access : ["scratch", "persistent"],
     hub_namespace : "prod"
   }

terraform/gcp/projects/daskhub-template.tfvars

@@ -48,7 +48,7 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "{{ hub_name }}" : {
-    requestor_pays : true,
+    allow_access_to_external_requester_pays_buckets : true,
     bucket_admin_access : ["scratch-{{ hub_name }}"],
     hub_namespace : "{{ hub_name }}"
   },

terraform/gcp/projects/leap.tfvars

@@ -60,13 +60,13 @@ user_buckets = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : true,
+    allow_access_to_external_requester_pays_buckets : true,
     bucket_admin_access : ["scratch-staging", "persistent-staging"],
     bucket_readonly_access : ["persistent-ro-staging"],
     hub_namespace : "staging"
   },
   "prod" : {
-    requestor_pays : true,
+    allow_access_to_external_requester_pays_buckets : true,
     bucket_admin_access : ["scratch", "persistent"],
     bucket_readonly_access : ["persistent-ro"],
     hub_namespace : "prod"

terraform/gcp/projects/linked-earth.tfvars

@@ -61,12 +61,12 @@ dask_nodes = {
 
 hub_cloud_permissions = {
   "staging" : {
-    requestor_pays : false,
+    allow_access_to_external_requester_pays_buckets : false,
     bucket_admin_access : ["scratch-staging"],
     hub_namespace : "staging"
   },
   "prod" : {
-    requestor_pays : false,
+    allow_access_to_external_requester_pays_buckets : false,
     bucket_admin_access : ["scratch"],
     hub_namespace : "prod"
   }