From 5e8825d3a0bf739e76188eb153d9027172ab60fc Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 16 Nov 2023 20:21:36 +0530 Subject: [PATCH 01/18] Clarify what the requestor_pays flag does --- terraform/aws/variables.tf | 14 ++++++++++---- terraform/gcp/variables.tf | 10 ++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 281374d1b5..1874ff88c3 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -44,7 +44,11 @@ variable "user_buckets" { } variable "hub_cloud_permissions" { - type = map(object({ requestor_pays : bool, bucket_admin_access : set(string), extra_iam_policy : string })) + type = map(object({ + allow_access_to_requestor_pays_buckets : optional(bool, false), + bucket_admin_access : set(string), + extra_iam_policy : string + })) default = {} description = <<-EOT Map of cloud permissions given to a particular hub @@ -52,9 +56,11 @@ variable "hub_cloud_permissions" { Key is name of the hub namespace in the cluster, and values are particular permissions users running on those hubs should have. Currently supported are: - 1. requestor_pays: Identify as coming from the google cloud project when accessing - storage buckets marked as https://cloud.google.com/storage/docs/requester-pays. - This *potentially* incurs cost for us, the originating project, so opt-in. + 1. allow_access_to_requestor_pays_buckets: Allow code running in user servers from this + hub to identify as coming from this particular GCP project when accessing GCS buckets + marked as 'requestor_pays'. In this case, the egress costs will + be borne by the project *containing the hub*, rather than the project *containing the bucket*. + Egress costs can get quite expensive, so this is 'opt-in'. 2. bucket_admin_access: List of S3 storage buckets that users on this hub should have read and write permissions for. 3. extra_iam_policy: An AWS IAM Policy document that grants additional rights to the users diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 91a7526f3e..37068e7b8d 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -401,7 +401,7 @@ variable "max_cpu" { variable "hub_cloud_permissions" { type = map( object({ - requestor_pays : bool, + allow_access_to_requestor_pays_buckets : optional(bool, false), bucket_admin_access : set(string), bucket_readonly_access : optional(set(string), []), hub_namespace : string @@ -414,9 +414,11 @@ variable "hub_cloud_permissions" { Key is name of the hub namespace in the cluster, and values are particular permissions users running on those hubs should have. Currently supported are: - 1. requestor_pays: Identify as coming from the google cloud project when accessing - storage buckets marked as https://cloud.google.com/storage/docs/requester-pays. - This *potentially* incurs cost for us, the originating project, so opt-in. + 1. allow_access_to_requestor_pays_buckets: Allow code running in user servers from this + hub to identify as coming from this particular GCP project when accessing GCS buckets + marked as 'requestor_pays'. In this case, the egress costs will + be borne by the project *containing the hub*, rather than the project *containing the bucket*. + Egress costs can get quite expensive, so this is 'opt-in'. 2. bucket_admin_access: List of GCS storage buckets that users on this hub should have read and write permissions for. EOT From 38e0d89f4b872fd522af628f836565327a9b29f5 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 14 Feb 2024 16:06:22 +0200 Subject: [PATCH 02/18] Update requester_pays docs with clarifications --- docs/howto/features/cloud-access.md | 8 ++++---- docs/topic/features.md | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/howto/features/cloud-access.md b/docs/howto/features/cloud-access.md index 5dd9ef9d74..7b50a3c273 100644 --- a/docs/howto/features/cloud-access.md +++ b/docs/howto/features/cloud-access.md @@ -42,7 +42,7 @@ This AWS IAM Role is managed via terraform. ``` hub_cloud_permissions = { "": { - requestor_pays : true, + allow_access_to_requestor_pays_buckets : true, bucket_admin_access : ["bucket-1", "bucket-2"] hub_namespace : "" } @@ -55,9 +55,9 @@ This AWS IAM Role is managed via terraform. and the cluster name together can't be more than 29 characters. `terraform` will complain if you go over this limit, so in general just use the name of the hub and shorten it only if `terraform` complains. - 2. (GCP only) `requestor_pays` enables permissions for user pods and dask worker - pods to identify as the project while making requests to Google Cloud Storage - buckets marked as 'requestor pays'. More details [here](topic:features:cloud:gcp:requestor-pays). + 2. (GCP only) `allow_access_to_requestor_pays_buckets` enables permissions for user pods and dask worker + pods to identify as the project while making requests to other Google Cloud Storage + buckets, outside of this project, that have 'Requester Pays' enabled. More details [here](topic:features:cloud:gcp:requestor-pays). 3. `bucket_admin_access` lists bucket names (as specified in `user_buckets` terraform variable) all users on this hub should have full read/write access to. Used along with the [user_buckets](howto:features:storage-buckets) diff --git a/docs/topic/features.md b/docs/topic/features.md index 517f7fee16..e3235ea69b 100644 --- a/docs/topic/features.md +++ b/docs/topic/features.md @@ -34,9 +34,23 @@ the data. This is very commonly used by organizations that provide big datasets on Google Cloud storage, to sustainably share costs of maintaining the data. When this feature is enabled, users on a hub accessing cloud buckets from -other organizations marked as 'requestor pays' will increase our cloud bill. +other organizations marked as 'Requester Pays' will increase our cloud bill. Hence, this is an opt-in feature. +```{important} +This feature enables the hub users to access `Requester Pays` buckets, +**outside** of their project. + +However, note that this feature **does not** control which buckets **inside** +the project will have `Requester Pays` enabled for themselves. + +This can be checked from the console following these steps in the +[GCP docs](`Requester Pays`). Enabling/disabling the `Requester Pays` +setting on a bucket can be achieved following +[this other section](https://cloud.google.com/storage/docs/using-requester-pays#set) + of the GCP docs. +``` + (topic:features:cloud:scratch-buckets)= ## 'Scratch' buckets on object storage From fec6d4d88c28e8a7f071399629eb47b0cd1073d8 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Thu, 15 Feb 2024 12:28:22 +0200 Subject: [PATCH 03/18] Update the enable/disable instructure to not be overwritable by an apply --- docs/topic/features.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/topic/features.md b/docs/topic/features.md index e3235ea69b..20141c2f91 100644 --- a/docs/topic/features.md +++ b/docs/topic/features.md @@ -44,11 +44,14 @@ This feature enables the hub users to access `Requester Pays` buckets, However, note that this feature **does not** control which buckets **inside** the project will have `Requester Pays` enabled for themselves. -This can be checked from the console following these steps in the -[GCP docs](`Requester Pays`). Enabling/disabling the `Requester Pays` -setting on a bucket can be achieved following -[this other section](https://cloud.google.com/storage/docs/using-requester-pays#set) - of the GCP docs. +1. This can be **checked** from the console following these steps in the +[GCP docs](https://cloud.google.com/storage/docs/using-requester-pays#check) or +by checking the `user_buckets` configuration in the project's terraform config +file if any `requester_pays` flag is specified because it is disabled by default. + +2. **Enabling or disabling** the `Requester Pays` flag on a bucket can be achieved by +setting the [`requestor_pays`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket#requester_pays) flag to true in our terraform config +`user_buckets` variable as described in [](howto:features:storage-buckets). ``` (topic:features:cloud:scratch-buckets)= From 8cf8947babab2326bd43f0a8c7e483d5e4d376f2 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Thu, 15 Feb 2024 12:28:50 +0200 Subject: [PATCH 04/18] Also provide a technical example of the setup --- docs/howto/features/buckets.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/howto/features/buckets.md b/docs/howto/features/buckets.md index c7150fb67d..b3a8a85fbb 100644 --- a/docs/howto/features/buckets.md +++ b/docs/howto/features/buckets.md @@ -106,6 +106,35 @@ user_buckets = { } ``` +## Enabling Requester Pays flag on buckets + +Some hubs want to expose a particular bucket to the broad internet +but not be billed for the charges associated with making and executing the +requests on this bucket. + +By enabling the [Requester Pays flag](https://cloud.google.com/storage/docs/using-requester-pays#using), +the requesters are required to include a billing project in their requests, +which will mean that the billing will happen on the requester's project. + +Enabling Requester Pays is useful, for example, if the communities have a lot +of data that they want to make available to others, but don't want to be +charged for their access to that data. + +This can be enabled by setting the `requester_pays` parameter in `user_buckets` +for the appropriate bucket, and running `terraform apply`. + +Example: + +```terraform +user_buckets = { + "persistent": { + "delete_after": null, + "public_access": true, + "requester_pays": true + } +} +``` + ## Enable access logs for objects in a bucket ### GCP From 90b9ec1739a762f6cbd2510c6ebe1350f76d0aa5 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Tue, 27 Feb 2024 13:47:06 +0200 Subject: [PATCH 05/18] Rename requester_pays again and rm it from aws --- terraform/aws/variables.tf | 10 ++-------- terraform/gcp/variables.tf | 4 ++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 1874ff88c3..332a78ab5a 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -45,7 +45,6 @@ variable "user_buckets" { variable "hub_cloud_permissions" { type = map(object({ - allow_access_to_requestor_pays_buckets : optional(bool, false), bucket_admin_access : set(string), extra_iam_policy : string })) @@ -56,14 +55,9 @@ variable "hub_cloud_permissions" { Key is name of the hub namespace in the cluster, and values are particular permissions users running on those hubs should have. Currently supported are: - 1. allow_access_to_requestor_pays_buckets: Allow code running in user servers from this - hub to identify as coming from this particular GCP project when accessing GCS buckets - marked as 'requestor_pays'. In this case, the egress costs will - be borne by the project *containing the hub*, rather than the project *containing the bucket*. - Egress costs can get quite expensive, so this is 'opt-in'. - 2. bucket_admin_access: List of S3 storage buckets that users on this hub should have read + 1. bucket_admin_access: List of S3 storage buckets that users on this hub should have read and write permissions for. - 3. extra_iam_policy: An AWS IAM Policy document that grants additional rights to the users + 2. extra_iam_policy: An AWS IAM Policy document that grants additional rights to the users on this hub when talking to AWS services. EOT } diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 37068e7b8d..befa640d37 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -401,7 +401,7 @@ variable "max_cpu" { variable "hub_cloud_permissions" { type = map( object({ - allow_access_to_requestor_pays_buckets : optional(bool, false), + allow_access_to_external_requestor_pays_buckets : optional(bool, false), bucket_admin_access : set(string), bucket_readonly_access : optional(set(string), []), hub_namespace : string @@ -414,7 +414,7 @@ variable "hub_cloud_permissions" { Key is name of the hub namespace in the cluster, and values are particular permissions users running on those hubs should have. Currently supported are: - 1. allow_access_to_requestor_pays_buckets: Allow code running in user servers from this + 1. allow_access_to_external_requestor_pays_buckets: Allow code running in user servers from this hub to identify as coming from this particular GCP project when accessing GCS buckets marked as 'requestor_pays'. In this case, the egress costs will be borne by the project *containing the hub*, rather than the project *containing the bucket*. From b7fd9a5250cd868944def210acdf43b1b403d683 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Tue, 27 Feb 2024 14:11:09 +0200 Subject: [PATCH 06/18] Rename requestor to requester to match google's documentation and update docs language for clarity --- docs/topic/features.md | 35 +++++++++++++++++++---------------- terraform/gcp/variables.tf | 4 ++-- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/docs/topic/features.md b/docs/topic/features.md index 20141c2f91..8d04616eab 100644 --- a/docs/topic/features.md +++ b/docs/topic/features.md @@ -23,8 +23,8 @@ improving the security posture of our hubs. ### GCP -(topic:features:cloud:gcp:requestor-pays)= -#### 'Requestor Pays' access to Google Cloud Storage buckets +(topic:features:cloud:gcp:requester-pays)= +#### 'Requester Pays' access By default, the organization *hosting* data on Google Cloud pays for both storage and bandwidth costs of the data. However, Google Cloud also offers @@ -33,25 +33,28 @@ option, where the bandwidth costs are paid for by the organization *requesting* the data. This is very commonly used by organizations that provide big datasets on Google Cloud storage, to sustainably share costs of maintaining the data. +**Requester Pays** is a feature that a bucket can have. + +#### Allow access to external `Requester Payes` buckets + +If buckets outside the project have the `Requester Payes` flag, then we need to: +- set `hub_cloud_permissions.allow_access_to_external_requester_pays_buckets` + in the terraform config of the cluster +- this will allow them to be charged on their project for access of such + outside buckets + +```{warning} When this feature is enabled, users on a hub accessing cloud buckets from -other organizations marked as 'Requester Pays' will increase our cloud bill. +other organizations marked as `Requester Pays` will increase our cloud bill. Hence, this is an opt-in feature. +``` -```{important} -This feature enables the hub users to access `Requester Pays` buckets, -**outside** of their project. - -However, note that this feature **does not** control which buckets **inside** -the project will have `Requester Pays` enabled for themselves. +#### Enable `Requester Pays` flag on community buckets -1. This can be **checked** from the console following these steps in the -[GCP docs](https://cloud.google.com/storage/docs/using-requester-pays#check) or -by checking the `user_buckets` configuration in the project's terraform config -file if any `requester_pays` flag is specified because it is disabled by default. +The buckets that we set for communities, inside their projects can also have this flag enabled on them, which means that other people outside will be charged for their usage. -2. **Enabling or disabling** the `Requester Pays` flag on a bucket can be achieved by -setting the [`requestor_pays`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket#requester_pays) flag to true in our terraform config -`user_buckets` variable as described in [](howto:features:storage-buckets). +```{warning} +This is not supported yet by our terraform. Follow (todo: insert issue link) for when support will be added. ``` (topic:features:cloud:scratch-buckets)= diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index befa640d37..f842bec480 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -401,7 +401,7 @@ variable "max_cpu" { variable "hub_cloud_permissions" { type = map( object({ - allow_access_to_external_requestor_pays_buckets : optional(bool, false), + allow_access_to_external_requester_pays_buckets : optional(bool, false), bucket_admin_access : set(string), bucket_readonly_access : optional(set(string), []), hub_namespace : string @@ -414,7 +414,7 @@ variable "hub_cloud_permissions" { Key is name of the hub namespace in the cluster, and values are particular permissions users running on those hubs should have. Currently supported are: - 1. allow_access_to_external_requestor_pays_buckets: Allow code running in user servers from this + 1. allow_access_to_external_requester_pays_buckets: Allow code running in user servers from this hub to identify as coming from this particular GCP project when accessing GCS buckets marked as 'requestor_pays'. In this case, the egress costs will be borne by the project *containing the hub*, rather than the project *containing the bucket*. From e9023d7befb0ce8544a0d694ec7829b360328538 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 10:46:23 +0200 Subject: [PATCH 07/18] Rm unexistent feature docs --- docs/howto/features/buckets.md | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/docs/howto/features/buckets.md b/docs/howto/features/buckets.md index b3a8a85fbb..c7150fb67d 100644 --- a/docs/howto/features/buckets.md +++ b/docs/howto/features/buckets.md @@ -106,35 +106,6 @@ user_buckets = { } ``` -## Enabling Requester Pays flag on buckets - -Some hubs want to expose a particular bucket to the broad internet -but not be billed for the charges associated with making and executing the -requests on this bucket. - -By enabling the [Requester Pays flag](https://cloud.google.com/storage/docs/using-requester-pays#using), -the requesters are required to include a billing project in their requests, -which will mean that the billing will happen on the requester's project. - -Enabling Requester Pays is useful, for example, if the communities have a lot -of data that they want to make available to others, but don't want to be -charged for their access to that data. - -This can be enabled by setting the `requester_pays` parameter in `user_buckets` -for the appropriate bucket, and running `terraform apply`. - -Example: - -```terraform -user_buckets = { - "persistent": { - "delete_after": null, - "public_access": true, - "requester_pays": true - } -} -``` - ## Enable access logs for objects in a bucket ### GCP From df351740deadbf35e27a85e9317dcc32f89719a6 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 10:46:35 +0200 Subject: [PATCH 08/18] Reference howto docs --- docs/topic/features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topic/features.md b/docs/topic/features.md index 8d04616eab..e121d8096b 100644 --- a/docs/topic/features.md +++ b/docs/topic/features.md @@ -39,7 +39,7 @@ on Google Cloud storage, to sustainably share costs of maintaining the data. If buckets outside the project have the `Requester Payes` flag, then we need to: - set `hub_cloud_permissions.allow_access_to_external_requester_pays_buckets` - in the terraform config of the cluster + in the terraform config of the cluster (see the guide at [](howto:features:cloud-access:access-perms)) - this will allow them to be charged on their project for access of such outside buckets From 92d1bba4fe2ca8d6009dcbfcd0e983b159cd1d0b Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 10:50:25 +0200 Subject: [PATCH 09/18] Rename var in one more place --- docs/howto/features/cloud-access.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/howto/features/cloud-access.md b/docs/howto/features/cloud-access.md index 7b50a3c273..169ac0a000 100644 --- a/docs/howto/features/cloud-access.md +++ b/docs/howto/features/cloud-access.md @@ -42,7 +42,7 @@ This AWS IAM Role is managed via terraform. ``` hub_cloud_permissions = { "": { - allow_access_to_requestor_pays_buckets : true, + "allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["bucket-1", "bucket-2"] hub_namespace : "" } @@ -55,7 +55,7 @@ This AWS IAM Role is managed via terraform. and the cluster name together can't be more than 29 characters. `terraform` will complain if you go over this limit, so in general just use the name of the hub and shorten it only if `terraform` complains. - 2. (GCP only) `allow_access_to_requestor_pays_buckets` enables permissions for user pods and dask worker + 2. (GCP only) `allow_access_to_external_requester_pays_buckets` enables permissions for user pods and dask worker pods to identify as the project while making requests to other Google Cloud Storage buckets, outside of this project, that have 'Requester Pays' enabled. More details [here](topic:features:cloud:gcp:requestor-pays). 3. `bucket_admin_access` lists bucket names (as specified in `user_buckets` From 59fc657603dae76ad9d11563b53849f1f3c96471 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 11:00:13 +0200 Subject: [PATCH 10/18] Rm quote --- docs/howto/features/cloud-access.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/features/cloud-access.md b/docs/howto/features/cloud-access.md index 169ac0a000..c9efc4f048 100644 --- a/docs/howto/features/cloud-access.md +++ b/docs/howto/features/cloud-access.md @@ -42,7 +42,7 @@ This AWS IAM Role is managed via terraform. ``` hub_cloud_permissions = { "": { - "allow_access_to_external_requester_pays_buckets : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["bucket-1", "bucket-2"] hub_namespace : "" } From 60b58e89683c29345b0b2d9d36ee74827d6417a2 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 11:02:43 +0200 Subject: [PATCH 11/18] Rename docs ref --- docs/howto/features/cloud-access.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/features/cloud-access.md b/docs/howto/features/cloud-access.md index c9efc4f048..2a80e2d1bb 100644 --- a/docs/howto/features/cloud-access.md +++ b/docs/howto/features/cloud-access.md @@ -57,7 +57,7 @@ This AWS IAM Role is managed via terraform. of the hub and shorten it only if `terraform` complains. 2. (GCP only) `allow_access_to_external_requester_pays_buckets` enables permissions for user pods and dask worker pods to identify as the project while making requests to other Google Cloud Storage - buckets, outside of this project, that have 'Requester Pays' enabled. More details [here](topic:features:cloud:gcp:requestor-pays). + buckets, outside of this project, that have 'Requester Pays' enabled. More details [here](topic:features:cloud:gcp:requester-pays). 3. `bucket_admin_access` lists bucket names (as specified in `user_buckets` terraform variable) all users on this hub should have full read/write access to. Used along with the [user_buckets](howto:features:storage-buckets) From 2a04cbc23a01742e016b012432fbd50bf519c40e Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 11:05:07 +0200 Subject: [PATCH 12/18] Rename requester in one more terraform config --- terraform/gcp/workload-identity.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/gcp/workload-identity.tf b/terraform/gcp/workload-identity.tf index 99e907c74e..72aca9a19d 100644 --- a/terraform/gcp/workload-identity.tf +++ b/terraform/gcp/workload-identity.tf @@ -47,7 +47,7 @@ resource "google_project_iam_custom_role" "requestor_pays" { } resource "google_project_iam_member" "requestor_pays_binding" { - for_each = toset([for hub_name, permissions in var.hub_cloud_permissions : hub_name if permissions.requestor_pays]) + for_each = toset([for hub_name, permissions in var.hub_cloud_permissions : hub_name if permissions.allow_access_to_external_requester_pays_buckets]) project = var.project_id role = google_project_iam_custom_role.requestor_pays.name member = "serviceAccount:${google_service_account.workload_sa[each.value].email}" From f8c23e03d481ba2484acd3dc4a5f91047650cf35 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 13:43:37 +0200 Subject: [PATCH 13/18] Rm undifinded requestor_pays var from aws terraform config --- terraform/aws/projects/2i2c-aws-us.tfvars | 6 ------ terraform/aws/projects/catalystproject-africa.tfvars | 2 -- terraform/aws/projects/earthscope.tfvars | 2 -- terraform/aws/projects/gridsst.tfvars | 2 -- terraform/aws/projects/jupyter-meets-the-earth.tfvars | 2 -- terraform/aws/projects/nasa-cryo.tfvars | 2 -- terraform/aws/projects/nasa-esdis.tfvars | 2 -- terraform/aws/projects/nasa-ghg.tfvars | 2 -- terraform/aws/projects/nasa-veda.tfvars | 2 -- terraform/aws/projects/openscapes.tfvars | 2 -- terraform/aws/projects/smithsonian.tfvars | 2 -- terraform/aws/projects/template.tfvars | 2 -- terraform/aws/projects/ubc-eoas.tfvars | 2 -- terraform/aws/projects/victor.tfvars | 2 -- 14 files changed, 32 deletions(-) diff --git a/terraform/aws/projects/2i2c-aws-us.tfvars b/terraform/aws/projects/2i2c-aws-us.tfvars index cf56c5e671..a9a3a0cf3d 100644 --- a/terraform/aws/projects/2i2c-aws-us.tfvars +++ b/terraform/aws/projects/2i2c-aws-us.tfvars @@ -31,17 +31,14 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "dask-staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-dask-staging"], extra_iam_policy : "" }, "showcase" : { - requestor_pays : true, bucket_admin_access : [ "scratch-researchdelight", "persistent-showcase" @@ -49,17 +46,14 @@ hub_cloud_permissions = { extra_iam_policy : "" }, "ncar-cisl" : { - requestor_pays : true, bucket_admin_access : ["scratch-ncar-cisl"], extra_iam_policy : "" }, "go-bgc" : { - requestor_pays : true, bucket_admin_access : ["scratch-go-bgc"], extra_iam_policy : "" }, "itcoocean" : { - requestor_pays : true, bucket_admin_access : ["scratch-itcoocean"], extra_iam_policy : "" }, diff --git a/terraform/aws/projects/catalystproject-africa.tfvars b/terraform/aws/projects/catalystproject-africa.tfvars index 70efd99f76..728f18a381 100644 --- a/terraform/aws/projects/catalystproject-africa.tfvars +++ b/terraform/aws/projects/catalystproject-africa.tfvars @@ -16,12 +16,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : "" }, diff --git a/terraform/aws/projects/earthscope.tfvars b/terraform/aws/projects/earthscope.tfvars index 57aeb6fbf9..688977269b 100644 --- a/terraform/aws/projects/earthscope.tfvars +++ b/terraform/aws/projects/earthscope.tfvars @@ -16,12 +16,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : "" }, diff --git a/terraform/aws/projects/gridsst.tfvars b/terraform/aws/projects/gridsst.tfvars index e13b2f1a05..74680c5fcd 100644 --- a/terraform/aws/projects/gridsst.tfvars +++ b/terraform/aws/projects/gridsst.tfvars @@ -16,12 +16,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : "" }, diff --git a/terraform/aws/projects/jupyter-meets-the-earth.tfvars b/terraform/aws/projects/jupyter-meets-the-earth.tfvars index 90615a14a9..73a5a38797 100644 --- a/terraform/aws/projects/jupyter-meets-the-earth.tfvars +++ b/terraform/aws/projects/jupyter-meets-the-earth.tfvars @@ -16,7 +16,6 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], # FIXME: Previously, users were granted full S3 permissions. # Keep it the same for now @@ -34,7 +33,6 @@ hub_cloud_permissions = { EOT }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], # FIXME: Previously, users were granted full S3 permissions. # Keep it the same for now diff --git a/terraform/aws/projects/nasa-cryo.tfvars b/terraform/aws/projects/nasa-cryo.tfvars index 1f45519983..72197c009d 100644 --- a/terraform/aws/projects/nasa-cryo.tfvars +++ b/terraform/aws/projects/nasa-cryo.tfvars @@ -22,7 +22,6 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging", "persistent-staging"], # Provides readonly requestor-pays access to usgs-landsat bucket # FIXME: We should find a way to allow access to *all* requester pays @@ -57,7 +56,6 @@ hub_cloud_permissions = { EOT }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch", "persistent"], # Provides readonly requestor-pays access to usgs-landsat bucket # FIXME: We should find a way to allow access to *all* requester pays diff --git a/terraform/aws/projects/nasa-esdis.tfvars b/terraform/aws/projects/nasa-esdis.tfvars index 186632f934..d97271f449 100644 --- a/terraform/aws/projects/nasa-esdis.tfvars +++ b/terraform/aws/projects/nasa-esdis.tfvars @@ -16,12 +16,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : "" }, diff --git a/terraform/aws/projects/nasa-ghg.tfvars b/terraform/aws/projects/nasa-ghg.tfvars index bc09d26c26..831205b98e 100644 --- a/terraform/aws/projects/nasa-ghg.tfvars +++ b/terraform/aws/projects/nasa-ghg.tfvars @@ -16,7 +16,6 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : <<-EOT { @@ -70,7 +69,6 @@ hub_cloud_permissions = { EOT }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : <<-EOT { diff --git a/terraform/aws/projects/nasa-veda.tfvars b/terraform/aws/projects/nasa-veda.tfvars index a06e4796ae..e1542d44af 100644 --- a/terraform/aws/projects/nasa-veda.tfvars +++ b/terraform/aws/projects/nasa-veda.tfvars @@ -16,7 +16,6 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : <<-EOT { @@ -71,7 +70,6 @@ hub_cloud_permissions = { EOT }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : <<-EOT { diff --git a/terraform/aws/projects/openscapes.tfvars b/terraform/aws/projects/openscapes.tfvars index 80a1e287b2..77d86e6ee1 100644 --- a/terraform/aws/projects/openscapes.tfvars +++ b/terraform/aws/projects/openscapes.tfvars @@ -19,12 +19,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : "" }, diff --git a/terraform/aws/projects/smithsonian.tfvars b/terraform/aws/projects/smithsonian.tfvars index 65acdb6510..1ec655e8e7 100644 --- a/terraform/aws/projects/smithsonian.tfvars +++ b/terraform/aws/projects/smithsonian.tfvars @@ -13,12 +13,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : "" }, diff --git a/terraform/aws/projects/template.tfvars b/terraform/aws/projects/template.tfvars index bb0ff4344f..20f703b97e 100644 --- a/terraform/aws/projects/template.tfvars +++ b/terraform/aws/projects/template.tfvars @@ -16,12 +16,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : "" }, diff --git a/terraform/aws/projects/ubc-eoas.tfvars b/terraform/aws/projects/ubc-eoas.tfvars index c3cba2162d..f38abdf057 100644 --- a/terraform/aws/projects/ubc-eoas.tfvars +++ b/terraform/aws/projects/ubc-eoas.tfvars @@ -16,12 +16,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : "" }, diff --git a/terraform/aws/projects/victor.tfvars b/terraform/aws/projects/victor.tfvars index ec4b6dcffd..f6237fe892 100644 --- a/terraform/aws/projects/victor.tfvars +++ b/terraform/aws/projects/victor.tfvars @@ -16,12 +16,10 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, bucket_admin_access : ["scratch-staging"], extra_iam_policy : "" }, "prod" : { - requestor_pays : true, bucket_admin_access : ["scratch"], extra_iam_policy : "" }, From d703d8bdcc71a079542e9da87f4566055908b65b Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 13:44:02 +0200 Subject: [PATCH 14/18] Rename terraform var in projects --- terraform/gcp/projects/awi-ciroh.tfvars | 4 ++-- terraform/gcp/projects/daskhub-template.tfvars | 2 +- terraform/gcp/projects/leap.tfvars | 4 ++-- terraform/gcp/projects/linked-earth.tfvars | 4 ++-- terraform/gcp/projects/meom-ige.tfvars | 4 ++-- terraform/gcp/projects/pangeo-hubs.tfvars | 6 +++--- terraform/gcp/projects/pilot-hubs.tfvars | 8 ++++---- terraform/gcp/projects/qcl.tfvars | 4 ++-- terraform/gcp/variables.tf | 2 +- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/terraform/gcp/projects/awi-ciroh.tfvars b/terraform/gcp/projects/awi-ciroh.tfvars index 5a6a6ebe94..1a4cd55562 100644 --- a/terraform/gcp/projects/awi-ciroh.tfvars +++ b/terraform/gcp/projects/awi-ciroh.tfvars @@ -63,12 +63,12 @@ dask_nodes = { hub_cloud_permissions = { "staging" : { - requestor_pays : false, + allow_access_to_external_requester_pays_buckets : false, bucket_admin_access : ["scratch-staging", "persistent-staging"], hub_namespace : "staging" }, "prod" : { - requestor_pays : false, + allow_access_to_external_requester_pays_buckets : false, bucket_admin_access : ["scratch", "persistent"], hub_namespace : "prod" } diff --git a/terraform/gcp/projects/daskhub-template.tfvars b/terraform/gcp/projects/daskhub-template.tfvars index 1bcf3e668a..76364e6ddd 100644 --- a/terraform/gcp/projects/daskhub-template.tfvars +++ b/terraform/gcp/projects/daskhub-template.tfvars @@ -33,7 +33,7 @@ user_buckets = { hub_cloud_permissions = { "{{ hub_name }}" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["scratch-{{ hub_name }}"], hub_namespace : "{{ hub_name }}" }, diff --git a/terraform/gcp/projects/leap.tfvars b/terraform/gcp/projects/leap.tfvars index f2862cdb2e..4fca26bb32 100644 --- a/terraform/gcp/projects/leap.tfvars +++ b/terraform/gcp/projects/leap.tfvars @@ -60,13 +60,13 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["scratch-staging", "persistent-staging"], bucket_readonly_access : ["persistent-ro-staging"], hub_namespace : "staging" }, "prod" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["scratch", "persistent"], bucket_readonly_access : ["persistent-ro"], hub_namespace : "prod" diff --git a/terraform/gcp/projects/linked-earth.tfvars b/terraform/gcp/projects/linked-earth.tfvars index 4234fb37a8..86678d6f2f 100644 --- a/terraform/gcp/projects/linked-earth.tfvars +++ b/terraform/gcp/projects/linked-earth.tfvars @@ -61,12 +61,12 @@ dask_nodes = { hub_cloud_permissions = { "staging" : { - requestor_pays : false, + allow_access_to_external_requester_pays_buckets : false, bucket_admin_access : ["scratch-staging"], hub_namespace : "staging" }, "prod" : { - requestor_pays : false, + allow_access_to_external_requester_pays_buckets : false, bucket_admin_access : ["scratch"], hub_namespace : "prod" } diff --git a/terraform/gcp/projects/meom-ige.tfvars b/terraform/gcp/projects/meom-ige.tfvars index 3c25ebda9a..f76778880f 100644 --- a/terraform/gcp/projects/meom-ige.tfvars +++ b/terraform/gcp/projects/meom-ige.tfvars @@ -81,12 +81,12 @@ user_buckets = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["scratch", "data"], hub_namespace : "staging" }, "prod" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["scratch", "data"], hub_namespace : "prod" } diff --git a/terraform/gcp/projects/pangeo-hubs.tfvars b/terraform/gcp/projects/pangeo-hubs.tfvars index 9277761bbd..ddcd8bd49b 100644 --- a/terraform/gcp/projects/pangeo-hubs.tfvars +++ b/terraform/gcp/projects/pangeo-hubs.tfvars @@ -109,17 +109,17 @@ dask_nodes = { hub_cloud_permissions = { "staging" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["scratch-staging"], hub_namespace : "staging" }, "prod" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["scratch"], hub_namespace : "prod" }, "coessing" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["coessing-scratch"], hub_namespace : "coessing" }, diff --git a/terraform/gcp/projects/pilot-hubs.tfvars b/terraform/gcp/projects/pilot-hubs.tfvars index 620d8119a0..02d3769aac 100644 --- a/terraform/gcp/projects/pilot-hubs.tfvars +++ b/terraform/gcp/projects/pilot-hubs.tfvars @@ -58,23 +58,23 @@ user_buckets = { hub_cloud_permissions = { "dask-staging" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : [], hub_namespace : "dask-staging" }, "ohw" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : [], hub_namespace : "ohw" }, # Can't use full name here as it violates line length restriction of service account id "catalyst-coop" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : [], hub_namespace : "catalyst-cooperative" }, "jackeddy" : { - requestor_pays : true, + allow_access_to_external_requester_pays_buckets : true, bucket_admin_access : ["jackeddy-scratch"], hub_namespace : "jackeddy" }, diff --git a/terraform/gcp/projects/qcl.tfvars b/terraform/gcp/projects/qcl.tfvars index 1433331606..a39144325e 100644 --- a/terraform/gcp/projects/qcl.tfvars +++ b/terraform/gcp/projects/qcl.tfvars @@ -66,12 +66,12 @@ notebook_nodes = { hub_cloud_permissions = { "staging" : { - requestor_pays : false, + allow_access_to_external_requester_pays_buckets : false, bucket_admin_access : ["scratch-staging"], hub_namespace : "staging" }, "prod" : { - requestor_pays : false, + allow_access_to_external_requester_pays_buckets : false, bucket_admin_access : ["scratch"], hub_namespace : "prod" } diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index f842bec480..0a514ed612 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -416,7 +416,7 @@ variable "hub_cloud_permissions" { 1. allow_access_to_external_requester_pays_buckets: Allow code running in user servers from this hub to identify as coming from this particular GCP project when accessing GCS buckets - marked as 'requestor_pays'. In this case, the egress costs will + marked as 'Requester Pays'. In this case, the egress costs will be borne by the project *containing the hub*, rather than the project *containing the bucket*. Egress costs can get quite expensive, so this is 'opt-in'. 2. bucket_admin_access: List of GCS storage buckets that users on this hub should have read From 248381336a5a35a8af6eafa5e017a8d5fc5b5e97 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 14:08:06 +0200 Subject: [PATCH 15/18] Link to issue --- docs/topic/features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topic/features.md b/docs/topic/features.md index e121d8096b..5206241489 100644 --- a/docs/topic/features.md +++ b/docs/topic/features.md @@ -54,7 +54,7 @@ Hence, this is an opt-in feature. The buckets that we set for communities, inside their projects can also have this flag enabled on them, which means that other people outside will be charged for their usage. ```{warning} -This is not supported yet by our terraform. Follow (todo: insert issue link) for when support will be added. +This is not supported yet by our terraform. Follow https://github.com/2i2c-org/infrastructure/issues/3746 to check when support will be added. ``` (topic:features:cloud:scratch-buckets)= From 7bdb40795cb437696adf52a8dc10e57b8e233197 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 14:21:26 +0200 Subject: [PATCH 16/18] Add warning about allow_access_to_external_requester_pays_buckets not being supported on aws --- docs/howto/features/cloud-access.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/howto/features/cloud-access.md b/docs/howto/features/cloud-access.md index 2a80e2d1bb..a284e6edf2 100644 --- a/docs/howto/features/cloud-access.md +++ b/docs/howto/features/cloud-access.md @@ -39,7 +39,10 @@ This AWS IAM Role is managed via terraform. create (or modify) the `hub_cloud_permissions` variable. The config is like: - ``` + `````{tab-set} + ````{tab-item} GCP + :sync: gcp-key + ```yaml hub_cloud_permissions = { "": { allow_access_to_external_requester_pays_buckets : true, @@ -48,7 +51,24 @@ This AWS IAM Role is managed via terraform. } } ``` + ```` + + ````{tab-item} AWS + :sync: aws-key + ```bash + hub_cloud_permissions = { + "": { + bucket_admin_access : ["bucket-1", "bucket-2"] + hub_namespace : "" + } + } + ``` + ```` + ````` + ```{warning} + `allow_access_to_external_requester_pays_buckets` is not yet supported on AWS! + ``` where: 1. `` is the name of the hub, but restricted in length. This From 80143da1311ddaae9a5ffa6f7cbc42b76153b54c Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 28 Feb 2024 16:08:08 +0200 Subject: [PATCH 17/18] Move the warning higher-up --- docs/howto/features/cloud-access.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/howto/features/cloud-access.md b/docs/howto/features/cloud-access.md index a284e6edf2..7d3c07114d 100644 --- a/docs/howto/features/cloud-access.md +++ b/docs/howto/features/cloud-access.md @@ -36,8 +36,13 @@ This AWS IAM Role is managed via terraform. ## Enabling specific cloud access permissions 1. In the `.tfvars` file for the project in which this hub is based off - create (or modify) the `hub_cloud_permissions` variable. The config is - like: + create (or modify) the `hub_cloud_permissions` variable. + + ```{warning} + `allow_access_to_external_requester_pays_buckets` is not yet supported on AWS! + ``` + + The config is like: `````{tab-set} ````{tab-item} GCP @@ -66,9 +71,6 @@ This AWS IAM Role is managed via terraform. ```` ````` - ```{warning} - `allow_access_to_external_requester_pays_buckets` is not yet supported on AWS! - ``` where: 1. `` is the name of the hub, but restricted in length. This From c544d97fe1196f553aedc4fec8ab35ce72103249 Mon Sep 17 00:00:00 2001 From: Georgiana Date: Wed, 28 Feb 2024 16:09:26 +0200 Subject: [PATCH 18/18] Rephrase for clarity Co-authored-by: Sarah Gibson <44771837+sgibson91@users.noreply.github.com> --- terraform/gcp/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 435d85849d..530665756a 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -416,7 +416,7 @@ variable "hub_cloud_permissions" { permissions users running on those hubs should have. Currently supported are: 1. allow_access_to_external_requester_pays_buckets: Allow code running in user servers from this - hub to identify as coming from this particular GCP project when accessing GCS buckets + hub to identify as coming from this particular GCP project when accessing GCS buckets in other projects marked as 'Requester Pays'. In this case, the egress costs will be borne by the project *containing the hub*, rather than the project *containing the bucket*. Egress costs can get quite expensive, so this is 'opt-in'.