From dde74ef5d7405bed4b211f4c77ecea1b74ea097d Mon Sep 17 00:00:00 2001 From: Sergey Kisel Date: Thu, 12 Oct 2023 13:57:11 +0200 Subject: [PATCH 1/9] Initial implementation for the fixed GCP Secret Manager --- .../PicoCliGcpSecretManagerParameters.java | 64 +++++++++ .../subcommands/Eth2SubCommand.java | 28 ++++ .../pegasys/web3signer/core/Eth2Runner.java | 19 +++ .../core/config/HealthCheckNames.java | 1 + .../allowed-licenses.json | 9 ++ keystorage/build.gradle | 2 + .../keystorage/gcp/GcpSecretManager.java | 130 ++++++++++++++++++ signing/build.gradle | 2 + .../signing/bulkloading/BlsGcpBulkLoader.java | 50 +++++++ .../config/GcpSecretManagerParameters.java | 25 ++++ .../signing/config/metadata/SignerOrigin.java | 1 + 11 files changed, 331 insertions(+) create mode 100644 commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliGcpSecretManagerParameters.java create mode 100644 keystorage/src/main/java/tech/pegasys/web3signer/keystorage/gcp/GcpSecretManager.java create mode 100644 signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/BlsGcpBulkLoader.java create mode 100644 signing/src/main/java/tech/pegasys/web3signer/signing/config/GcpSecretManagerParameters.java diff --git a/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliGcpSecretManagerParameters.java b/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliGcpSecretManagerParameters.java new file mode 100644 index 000000000..d671ba134 --- /dev/null +++ b/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliGcpSecretManagerParameters.java @@ -0,0 +1,64 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.commandline; + +import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters; + +import java.util.Optional; + +import picocli.CommandLine.Option; + +public class PicoCliGcpSecretManagerParameters implements GcpSecretManagerParameters { + public static final String GCP_SECRETS_ENABLED_OPTION = "--gcp-secrets-enabled"; + public static final String GCP_SECRETS_FILTER_OPTION = "--gcp-secrets-filter"; + public static final String GCP_PROJECT_ID_OPTION = "--gcp-project-id"; + + @Option( + names = GCP_SECRETS_ENABLED_OPTION, + description = + "Set to true to enable bulk loading from the GCP Secret Manager service." + + " (Default: ${DEFAULT-VALUE})", + paramLabel = "") + private boolean gcpSecretsEnabledOption = false; + + @Option( + names = {GCP_PROJECT_ID_OPTION}, + description = + "A globally unique identifier for the GCP project where the secrets are stored.", + paramLabel = "") + private String projectId; + + @Option( + names = GCP_SECRETS_FILTER_OPTION, + description = + "Filter string for loading secrets into the application, adhering to the rules in " + + "[List-operation filtering](https://cloud.google.com/secret-manager/docs/filtering). " + + "Only secrets matching the filter will be loaded. If filter is empty, all secrets from the " + + "specified project are loaded into the application.") + private Optional filter = Optional.empty(); + + @Override + public boolean isEnabled() { + return gcpSecretsEnabledOption; + } + + @Override + public String getProjectId() { + return projectId; + } + + @Override + public Optional getFilter() { + return filter; + } +} diff --git a/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth2SubCommand.java b/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth2SubCommand.java index f54e57b08..930134a33 100644 --- a/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth2SubCommand.java +++ b/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth2SubCommand.java @@ -27,6 +27,7 @@ import tech.pegasys.teku.spec.networks.Eth2Network; import tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters; import tech.pegasys.web3signer.commandline.PicoCliEth2AzureKeyVaultParameters; +import tech.pegasys.web3signer.commandline.PicoCliGcpSecretManagerParameters; import tech.pegasys.web3signer.commandline.PicoCliSlashingProtectionParameters; import tech.pegasys.web3signer.commandline.config.PicoKeystoresParameters; import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; @@ -145,6 +146,7 @@ private static class NetworkCliCompletionCandidates extends ArrayList { @Mixin private PicoCliEth2AzureKeyVaultParameters azureKeyVaultParameters; @Mixin private PicoKeystoresParameters keystoreParameters; @Mixin private PicoCliAwsSecretsManagerParameters awsSecretsManagerParameters; + @Mixin private PicoCliGcpSecretManagerParameters gcpSecretManagerParameters; private tech.pegasys.teku.spec.Spec eth2Spec; public Eth2SubCommand() { @@ -161,6 +163,7 @@ public Runner createRunner() { azureKeyVaultParameters, keystoreParameters, awsSecretsManagerParameters, + gcpSecretManagerParameters, eth2Spec, isKeyManagerApiEnabled); } @@ -236,6 +239,23 @@ protected void validateArgs() { validateAzureParameters(); validateKeystoreParameters(keystoreParameters); validateAwsSecretsManageParameters(); + validateGcpSecretManagerParameters(); + } + + private void validateGcpSecretManagerParameters() { + if (gcpSecretManagerParameters.isEnabled()) { + final List specifiedAuthModeMissingFields = + missingGcpSecretManagerParametersForSpecified(); + if (!specifiedAuthModeMissingFields.isEmpty()) { + final String errorMsg = + String.format( + "%s=%s, but the following parameters were missing [%s].", + PicoCliGcpSecretManagerParameters.GCP_SECRETS_ENABLED_OPTION, + PicoCliGcpSecretManagerParameters.GCP_PROJECT_ID_OPTION, + String.join(", ", specifiedAuthModeMissingFields)); + throw new ParameterException(commandSpec.commandLine(), errorMsg); + } + } } private void validateAzureParameters() { @@ -302,6 +322,14 @@ private void validateAwsSecretsManageParameters() { } } + private List missingGcpSecretManagerParametersForSpecified() { + final List missingFields = Lists.newArrayList(); + if (gcpSecretManagerParameters.getProjectId() == null) { + missingFields.add(PicoCliGcpSecretManagerParameters.GCP_PROJECT_ID_OPTION); + } + return missingFields; + } + private List missingAwsSecretsManagerParametersForSpecified() { final List missingFields = Lists.newArrayList(); if (awsSecretsManagerParameters.getAuthenticationMode() == AwsAuthenticationMode.SPECIFIED) { diff --git a/core/src/main/java/tech/pegasys/web3signer/core/Eth2Runner.java b/core/src/main/java/tech/pegasys/web3signer/core/Eth2Runner.java index b0c6271e6..8b5b18bc4 100644 --- a/core/src/main/java/tech/pegasys/web3signer/core/Eth2Runner.java +++ b/core/src/main/java/tech/pegasys/web3signer/core/Eth2Runner.java @@ -15,6 +15,7 @@ import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_AWS_BULK_LOADING; import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_AZURE_BULK_LOADING; import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_CONFIG_FILE_LOADING; +import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_GCP_BULK_LOADING; import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_KEYSTORE_BULK_LOADING; import static tech.pegasys.web3signer.core.config.HealthCheckNames.SLASHING_PROTECTION_DB; import static tech.pegasys.web3signer.signing.KeyType.BLS; @@ -45,11 +46,13 @@ import tech.pegasys.web3signer.signing.KeystoreFileManager; import tech.pegasys.web3signer.signing.ValidatorManager; import tech.pegasys.web3signer.signing.bulkloading.BlsAwsBulkLoader; +import tech.pegasys.web3signer.signing.bulkloading.BlsGcpBulkLoader; import tech.pegasys.web3signer.signing.bulkloading.BlsKeystoreBulkLoader; import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultFactory; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.DefaultArtifactSignerProvider; +import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; import tech.pegasys.web3signer.signing.config.SignerLoader; import tech.pegasys.web3signer.signing.config.metadata.AbstractArtifactSignerFactory; @@ -97,6 +100,7 @@ public class Eth2Runner extends Runner { private final Optional slashingProtectionContext; private final AzureKeyVaultParameters azureKeyVaultParameters; private final AwsVaultParameters awsVaultParameters; + private final GcpSecretManagerParameters gcpSecretManagerParameters; private final SlashingProtectionParameters slashingProtectionParameters; private final boolean pruningEnabled; private final KeystoresParameters keystoresParameters; @@ -109,6 +113,7 @@ public Eth2Runner( final AzureKeyVaultParameters azureKeyVaultParameters, final KeystoresParameters keystoresParameters, final AwsVaultParameters awsVaultParameters, + final GcpSecretManagerParameters gcpSecretManagerParameters, final Spec eth2Spec, final boolean isKeyManagerApiEnabled) { super(baseConfig); @@ -120,6 +125,7 @@ public Eth2Runner( this.eth2Spec = eth2Spec; this.isKeyManagerApiEnabled = isKeyManagerApiEnabled; this.awsVaultParameters = awsVaultParameters; + this.gcpSecretManagerParameters = gcpSecretManagerParameters; } private Optional createSlashingProtection( @@ -360,6 +366,19 @@ private MappedResults bulkLoadSigners( results = MappedResults.merge(results, awsResult); } + if (gcpSecretManagerParameters.isEnabled()) { + LOG.info("Bulk loading keys from GCP Secret Manager ... "); + final BlsGcpBulkLoader blsGcpBulkLoader = new BlsGcpBulkLoader(); + final MappedResults gcpResult = + blsGcpBulkLoader.load(gcpSecretManagerParameters); + LOG.info( + "Keys loaded from GCP Secret Manager: [{}], with error count: [{}]", + gcpResult.getValues().size(), + gcpResult.getErrorCount()); + registerSignerLoadingHealthCheck(KEYS_CHECK_GCP_BULK_LOADING, gcpResult); + results = MappedResults.merge(results, gcpResult); + } + return results; } diff --git a/core/src/main/java/tech/pegasys/web3signer/core/config/HealthCheckNames.java b/core/src/main/java/tech/pegasys/web3signer/core/config/HealthCheckNames.java index 5f9159b8e..64e3861f3 100644 --- a/core/src/main/java/tech/pegasys/web3signer/core/config/HealthCheckNames.java +++ b/core/src/main/java/tech/pegasys/web3signer/core/config/HealthCheckNames.java @@ -17,6 +17,7 @@ public interface HealthCheckNames { String SLASHING_PROTECTION_DB = "slashing-protection-db-health-check"; String KEYS_CHECK_UNEXPECTED = "keys-check/unexpected"; String KEYS_CHECK_AWS_BULK_LOADING = "keys-check/aws-bulk-loading"; + String KEYS_CHECK_GCP_BULK_LOADING = "keys-check/gcp-bulk-loading"; String KEYS_CHECK_AZURE_BULK_LOADING = "keys-check/azure-bulk-loading"; String KEYS_CHECK_KEYSTORE_BULK_LOADING = "keys-check/keystores-bulk-loading"; String KEYS_CHECK_CONFIG_FILE_LOADING = "keys-check/config-files-loading"; diff --git a/gradle/license-report-config/allowed-licenses.json b/gradle/license-report-config/allowed-licenses.json index 98e175007..b92cf4239 100644 --- a/gradle/license-report-config/allowed-licenses.json +++ b/gradle/license-report-config/allowed-licenses.json @@ -80,6 +80,15 @@ }, { "moduleName": "com.squareup.okio:okio" + }, + { + "moduleName": "com.google.re2j:re2j", + "moduleLicense": "Go License" + }, + { + "moduleName": "com.google.cloud:libraries-bom", + "moduleLicense": null, + "moduleVersion": "26.12.0" } ], "overrideLicenses": [ diff --git a/keystorage/build.gradle b/keystorage/build.gradle index e37ace020..146b42519 100644 --- a/keystorage/build.gradle +++ b/keystorage/build.gradle @@ -50,6 +50,8 @@ dependencies { implementation 'software.amazon.awssdk:auth' implementation 'software.amazon.awssdk:secretsmanager' implementation 'software.amazon.awssdk:kms' + implementation platform('com.google.cloud:libraries-bom:26.12.0') + implementation 'com.google.cloud:google-cloud-secretmanager' runtimeOnly 'software.amazon.awssdk:sts' runtimeOnly 'org.apache.logging.log4j:log4j-core' diff --git a/keystorage/src/main/java/tech/pegasys/web3signer/keystorage/gcp/GcpSecretManager.java b/keystorage/src/main/java/tech/pegasys/web3signer/keystorage/gcp/GcpSecretManager.java new file mode 100644 index 000000000..48d2c0a5f --- /dev/null +++ b/keystorage/src/main/java/tech/pegasys/web3signer/keystorage/gcp/GcpSecretManager.java @@ -0,0 +1,130 @@ +/* + * Copyright 2023 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.keystorage.gcp; + +import tech.pegasys.web3signer.keystorage.common.MappedResults; +import tech.pegasys.web3signer.keystorage.common.SecretValueMapperUtil; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; + +import com.google.cloud.secretmanager.v1.AccessSecretVersionRequest; +import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse; +import com.google.cloud.secretmanager.v1.ListSecretsRequest; +import com.google.cloud.secretmanager.v1.ProjectName; +import com.google.cloud.secretmanager.v1.Secret; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import com.google.cloud.secretmanager.v1.SecretPayload; +import com.google.protobuf.ByteString; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; + +public class GcpSecretManager implements Closeable { + + private static final Logger LOG = LogManager.getLogger(); + private final SecretManagerServiceClient secretManagerServiceClient; + + public GcpSecretManager() throws IOException { + secretManagerServiceClient = SecretManagerServiceClient.create(); + } + + @Override + public void close() throws IOException { + secretManagerServiceClient.close(); + } + + /** + * Bulk load secrets. + * + * @param projectId GCP Project Id + * @param filter GCP Resource filter + * @param mapper The mapper function that can convert secret value to appropriate type + * @return SecretValueResult with collection of secret values and error count if any. + */ + public MappedResults mapSecrets( + final String projectId, + final Optional filter, + final BiFunction mapper) { + + final Set result = ConcurrentHashMap.newKeySet(); + final AtomicInteger errorCount = new AtomicInteger(0); + try { + listSecrets(projectId, filter) + .forEach( + secretEntry -> { + try { + final Optional secretValue = fetchStringSecret(secretEntry.getName()); + if (secretValue.isEmpty()) { + LOG.warn( + "Failed to fetch secret name '{}', and was discarded", + secretEntry.getName()); + errorCount.incrementAndGet(); + } else { + MappedResults multiResult = + SecretValueMapperUtil.mapSecretValue( + mapper, secretEntry.getName(), secretValue.get()); + result.addAll(multiResult.getValues()); + errorCount.addAndGet(multiResult.getErrorCount()); + } + } catch (final Exception e) { + LOG.warn( + "Failed to map secret '{}' to requested object type due to: {}.", + secretEntry.getName(), + e.getMessage()); + errorCount.incrementAndGet(); + } + }); + } catch (final Exception e) { + LOG.warn("Unexpected error during GCP list-secrets operation", e); + errorCount.incrementAndGet(); + } + return MappedResults.newInstance(result, errorCount.intValue()); + } + + private Iterable listSecrets(String projectId, Optional filter) { + ListSecretsRequest request = listSecretsRequest(projectId, filter); + return secretManagerServiceClient.listSecrets(request).iterateAll(); + } + + @NotNull + private static ListSecretsRequest listSecretsRequest(String projectId, Optional filter) { + ListSecretsRequest.Builder builder = ListSecretsRequest.newBuilder(); + builder.setParent(ProjectName.of(projectId).toString()); + filter.ifPresent(builder::setFilter); + return builder.build(); + } + + private Optional fetchStringSecret(String secretName) { + AccessSecretVersionResponse accessSecretVersionResponse = fetchSecret(secretName); + if (accessSecretVersionResponse.hasPayload()) { + SecretPayload payload = accessSecretVersionResponse.getPayload(); + ByteString payloadData = payload.getData(); + return Optional.of(payloadData.toString(StandardCharsets.UTF_8)); + } else { + return Optional.empty(); + } + } + + private AccessSecretVersionResponse fetchSecret(String secretName) { + AccessSecretVersionRequest accessSecretVersionRequest = + AccessSecretVersionRequest.newBuilder().setName(secretName + "/versions/latest").build(); + return secretManagerServiceClient.accessSecretVersion(accessSecretVersionRequest); + } +} diff --git a/signing/build.gradle b/signing/build.gradle index f7c3f0460..6c9e6a323 100644 --- a/signing/build.gradle +++ b/signing/build.gradle @@ -41,6 +41,8 @@ dependencies { implementation 'com.github.ben-manes.caffeine:caffeine:3.1.5' implementation 'software.amazon.awssdk:auth' implementation 'software.amazon.awssdk:kms' + implementation platform('com.google.cloud:libraries-bom:26.12.0') + implementation 'com.google.cloud:google-cloud-secretmanager' runtimeOnly 'com.squareup.okhttp3:okhttp' runtimeOnly 'org.apache.logging.log4j:log4j-slf4j2-impl' diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/BlsGcpBulkLoader.java b/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/BlsGcpBulkLoader.java new file mode 100644 index 000000000..0f4434364 --- /dev/null +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/BlsGcpBulkLoader.java @@ -0,0 +1,50 @@ +/* + * Copyright 2023 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.signing.bulkloading; + +import tech.pegasys.teku.bls.BLSKeyPair; +import tech.pegasys.teku.bls.BLSSecretKey; +import tech.pegasys.web3signer.keystorage.common.MappedResults; +import tech.pegasys.web3signer.keystorage.gcp.GcpSecretManager; +import tech.pegasys.web3signer.signing.ArtifactSigner; +import tech.pegasys.web3signer.signing.BlsArtifactSigner; +import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters; +import tech.pegasys.web3signer.signing.config.metadata.SignerOrigin; + +import java.io.IOException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +public class BlsGcpBulkLoader { + private static final Logger LOG = LogManager.getLogger(); + + public MappedResults load(final GcpSecretManagerParameters parameters) { + try (final GcpSecretManager gcpSecretManager = new GcpSecretManager()) { + return gcpSecretManager.mapSecrets( + parameters.getProjectId(), + parameters.getFilter(), + (key, value) -> { + final Bytes privateKeyBytes = Bytes.fromHexString(value); + final BLSKeyPair keyPair = + new BLSKeyPair(BLSSecretKey.fromBytes(Bytes32.wrap(privateKeyBytes))); + return new BlsArtifactSigner(keyPair, SignerOrigin.GCP); + }); + } catch (IOException e) { + LOG.error("Error reading GCP secrets", e); + return MappedResults.errorResult(); + } + } +} diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/config/GcpSecretManagerParameters.java b/signing/src/main/java/tech/pegasys/web3signer/signing/config/GcpSecretManagerParameters.java new file mode 100644 index 000000000..b9f3064dd --- /dev/null +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/config/GcpSecretManagerParameters.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.signing.config; + +import java.util.Optional; + +public interface GcpSecretManagerParameters { + boolean isEnabled(); + + String getProjectId(); + + default Optional getFilter() { + return Optional.empty(); + } +} diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/config/metadata/SignerOrigin.java b/signing/src/main/java/tech/pegasys/web3signer/signing/config/metadata/SignerOrigin.java index ca2b7d2fd..fc675c84b 100644 --- a/signing/src/main/java/tech/pegasys/web3signer/signing/config/metadata/SignerOrigin.java +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/config/metadata/SignerOrigin.java @@ -16,6 +16,7 @@ public enum SignerOrigin { AZURE, HASHICORP, AWS, + GCP, INTERLOCK, YUBI_HSM, FILE_KEYSTORE, From b84cc6f53906880d8e1daf707a3af31bde78270c Mon Sep 17 00:00:00 2001 From: Sergey Kisel Date: Thu, 12 Oct 2023 14:19:52 +0200 Subject: [PATCH 2/9] bump library versions --- keystorage/build.gradle | 2 +- signing/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/keystorage/build.gradle b/keystorage/build.gradle index 146b42519..f300e854a 100644 --- a/keystorage/build.gradle +++ b/keystorage/build.gradle @@ -50,7 +50,7 @@ dependencies { implementation 'software.amazon.awssdk:auth' implementation 'software.amazon.awssdk:secretsmanager' implementation 'software.amazon.awssdk:kms' - implementation platform('com.google.cloud:libraries-bom:26.12.0') + implementation platform('com.google.cloud:libraries-bom:26.24.0') implementation 'com.google.cloud:google-cloud-secretmanager' runtimeOnly 'software.amazon.awssdk:sts' diff --git a/signing/build.gradle b/signing/build.gradle index 6c9e6a323..a7cb65ba0 100644 --- a/signing/build.gradle +++ b/signing/build.gradle @@ -41,7 +41,7 @@ dependencies { implementation 'com.github.ben-manes.caffeine:caffeine:3.1.5' implementation 'software.amazon.awssdk:auth' implementation 'software.amazon.awssdk:kms' - implementation platform('com.google.cloud:libraries-bom:26.12.0') + implementation platform('com.google.cloud:libraries-bom:26.24.0') implementation 'com.google.cloud:google-cloud-secretmanager' runtimeOnly 'com.squareup.okhttp3:okhttp' From c7674e401a47118c5eaddba3ae86126ee9eb8e61 Mon Sep 17 00:00:00 2001 From: Sergey Kisel Date: Thu, 12 Oct 2023 16:18:52 +0200 Subject: [PATCH 3/9] license cleanup --- .../allowed-licenses.json | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/gradle/license-report-config/allowed-licenses.json b/gradle/license-report-config/allowed-licenses.json index b92cf4239..d8ed1a4d6 100644 --- a/gradle/license-report-config/allowed-licenses.json +++ b/gradle/license-report-config/allowed-licenses.json @@ -84,11 +84,6 @@ { "moduleName": "com.google.re2j:re2j", "moduleLicense": "Go License" - }, - { - "moduleName": "com.google.cloud:libraries-bom", - "moduleLicense": null, - "moduleVersion": "26.12.0" } ], "overrideLicenses": [ @@ -159,6 +154,30 @@ { "moduleName": "org.java-websocket:Java-WebSocket", "moduleLicense": "MIT License" + }, + { + "moduleName": "com.google.api:api-common", + "moduleLicense": "The BSD License" + }, + { + "moduleName": "com.google.api:gax", + "moduleLicense": "The BSD License" + }, + { + "moduleName": "com.google.api:gax-grpc", + "moduleLicense": "The BSD License" + }, + { + "moduleName": "com.google.api:gax-httpjson", + "moduleLicense": "The BSD License" + }, + { + "moduleName": "com.google.cloud:libraries-bom", + "moduleLicense": "Apache License, Version 2.0" + }, + { + "moduleName": "com.google.errorprone:error_prone_annotations", + "moduleLicense": "Apache License, Version 2.0" } ] } From 315521767c930da74cec279f14f002652747a9eb Mon Sep 17 00:00:00 2001 From: Sergey Kisel Date: Thu, 12 Oct 2023 16:39:28 +0200 Subject: [PATCH 4/9] adding test --- .../commandline/subcommands/Eth2SubCommand.java | 2 +- .../commandline/CommandlineParserTest.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth2SubCommand.java b/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth2SubCommand.java index 930134a33..d6d0be3d9 100644 --- a/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth2SubCommand.java +++ b/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth2SubCommand.java @@ -251,7 +251,7 @@ private void validateGcpSecretManagerParameters() { String.format( "%s=%s, but the following parameters were missing [%s].", PicoCliGcpSecretManagerParameters.GCP_SECRETS_ENABLED_OPTION, - PicoCliGcpSecretManagerParameters.GCP_PROJECT_ID_OPTION, + true, String.join(", ", specifiedAuthModeMissingFields)); throw new ParameterException(commandSpec.commandLine(), errorMsg); } diff --git a/commandline/src/test/java/tech/pegasys/web3signer/commandline/CommandlineParserTest.java b/commandline/src/test/java/tech/pegasys/web3signer/commandline/CommandlineParserTest.java index 80bc077aa..a6c4cf025 100644 --- a/commandline/src/test/java/tech/pegasys/web3signer/commandline/CommandlineParserTest.java +++ b/commandline/src/test/java/tech/pegasys/web3signer/commandline/CommandlineParserTest.java @@ -420,6 +420,23 @@ void keystoreOptionsWithBothPasswordDirAndPasswordFileFailsToParse() { "Error parsing parameters: Only one of --keystores-passwords-path or --keystores-password-file options can be specified"); } + @Test + void gcpSpecifiedProjectIdFailsToParseWithoutRequiredParameters() { + String cmdline = validBaseCommandOptions(); + cmdline += + String.format( + "eth2 --slashing-protection-enabled=false %s=true", + PicoCliGcpSecretManagerParameters.GCP_SECRETS_ENABLED_OPTION); + + parser.registerSubCommands(new MockEth2SubCommand()); + final int result = parser.parseCommandLine(cmdline.split(" ")); + + assertThat(result).isNotZero(); + assertThat(commandError.toString()) + .contains( + "Error parsing parameters: --gcp-secrets-enabled=true, but the following parameters were missing [--gcp-project-id]."); + } + @Test void awsSpecifiedAuthModeFailsToParseWithoutRequiredParameters() { String cmdline = validBaseCommandOptions(); From 325624a7461fbadbdc1cca7ca2766ca02178f84a Mon Sep 17 00:00:00 2001 From: Sergey Kisel Date: Thu, 12 Oct 2023 17:05:02 +0200 Subject: [PATCH 5/9] dependency management cleaup --- gradle/versions.gradle | 1 + keystorage/build.gradle | 1 - signing/build.gradle | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 043f111e6..81c4280fb 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -144,6 +144,7 @@ dependencyManagement { dependency 'org.flywaydb:flyway-core:6.1.1' + dependency 'com.google.cloud:google-cloud-secretmanager:2.27.0' dependency 'io.zonky.test.postgres:embedded-postgres-binaries-bom:11.19.0' dependency 'io.zonky.test:embedded-postgres:2.0.3' diff --git a/keystorage/build.gradle b/keystorage/build.gradle index f300e854a..46e97c6b7 100644 --- a/keystorage/build.gradle +++ b/keystorage/build.gradle @@ -50,7 +50,6 @@ dependencies { implementation 'software.amazon.awssdk:auth' implementation 'software.amazon.awssdk:secretsmanager' implementation 'software.amazon.awssdk:kms' - implementation platform('com.google.cloud:libraries-bom:26.24.0') implementation 'com.google.cloud:google-cloud-secretmanager' runtimeOnly 'software.amazon.awssdk:sts' diff --git a/signing/build.gradle b/signing/build.gradle index a7cb65ba0..99b134b56 100644 --- a/signing/build.gradle +++ b/signing/build.gradle @@ -41,7 +41,6 @@ dependencies { implementation 'com.github.ben-manes.caffeine:caffeine:3.1.5' implementation 'software.amazon.awssdk:auth' implementation 'software.amazon.awssdk:kms' - implementation platform('com.google.cloud:libraries-bom:26.24.0') implementation 'com.google.cloud:google-cloud-secretmanager' runtimeOnly 'com.squareup.okhttp3:okhttp' From e1e5ce54940163d33d0c592935ffe2fa433bf3b8 Mon Sep 17 00:00:00 2001 From: Sergey Kisel Date: Fri, 13 Oct 2023 16:13:38 +0200 Subject: [PATCH 6/9] address small concerns --- .../web3signer/keystorage/gcp/GcpSecretManager.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/keystorage/src/main/java/tech/pegasys/web3signer/keystorage/gcp/GcpSecretManager.java b/keystorage/src/main/java/tech/pegasys/web3signer/keystorage/gcp/GcpSecretManager.java index 48d2c0a5f..431396cba 100644 --- a/keystorage/src/main/java/tech/pegasys/web3signer/keystorage/gcp/GcpSecretManager.java +++ b/keystorage/src/main/java/tech/pegasys/web3signer/keystorage/gcp/GcpSecretManager.java @@ -34,7 +34,6 @@ import com.google.protobuf.ByteString; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; public class GcpSecretManager implements Closeable { @@ -99,20 +98,19 @@ public MappedResults mapSecrets( } private Iterable listSecrets(String projectId, Optional filter) { - ListSecretsRequest request = listSecretsRequest(projectId, filter); + final ListSecretsRequest request = listSecretsRequest(projectId, filter); return secretManagerServiceClient.listSecrets(request).iterateAll(); } - @NotNull private static ListSecretsRequest listSecretsRequest(String projectId, Optional filter) { - ListSecretsRequest.Builder builder = ListSecretsRequest.newBuilder(); + final ListSecretsRequest.Builder builder = ListSecretsRequest.newBuilder(); builder.setParent(ProjectName.of(projectId).toString()); filter.ifPresent(builder::setFilter); return builder.build(); } private Optional fetchStringSecret(String secretName) { - AccessSecretVersionResponse accessSecretVersionResponse = fetchSecret(secretName); + final AccessSecretVersionResponse accessSecretVersionResponse = fetchSecret(secretName); if (accessSecretVersionResponse.hasPayload()) { SecretPayload payload = accessSecretVersionResponse.getPayload(); ByteString payloadData = payload.getData(); @@ -123,7 +121,7 @@ private Optional fetchStringSecret(String secretName) { } private AccessSecretVersionResponse fetchSecret(String secretName) { - AccessSecretVersionRequest accessSecretVersionRequest = + final AccessSecretVersionRequest accessSecretVersionRequest = AccessSecretVersionRequest.newBuilder().setName(secretName + "/versions/latest").build(); return secretManagerServiceClient.accessSecretVersion(accessSecretVersionRequest); } From 9dc79386f9124cadbea0a02897562e4d313d9535 Mon Sep 17 00:00:00 2001 From: Sergey Kisel Date: Fri, 13 Oct 2023 18:17:51 +0200 Subject: [PATCH 7/9] GCP Secret Manager acceptance test --- .../dsl/signer/SignerConfiguration.java | 8 + .../signer/SignerConfigurationBuilder.java | 9 + .../runner/CmdLineParamsConfigFileImpl.java | 30 ++++ .../runner/CmdLineParamsDefaultImpl.java | 27 +++ .../GcpSecretManagerAcceptanceTest.java | 158 ++++++++++++++++++ signing/build.gradle | 1 + .../web3signer/GcpSecretManagerUtil.java | 72 ++++++++ .../GcpSecretManagerParametersBuilder.java | 74 ++++++++ 8 files changed, 379 insertions(+) create mode 100644 acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/GcpSecretManagerAcceptanceTest.java create mode 100644 signing/src/testFixtures/java/tech/pegasys/web3signer/GcpSecretManagerUtil.java create mode 100644 signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/GcpSecretManagerParametersBuilder.java diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java index 37bde8250..533bc6f96 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java @@ -18,6 +18,7 @@ import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition; import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; +import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; import java.nio.file.Path; @@ -42,6 +43,7 @@ public class SignerConfiguration { private final boolean metricsEnabled; private final Optional azureKeyVaultParameters; private final Optional awsSecretsManagerParameters; + private final Optional gcpSecretManagerParameters; private final Optional keystoresParameters; private final Optional serverTlsOptions; private final Optional overriddenCaTrustStore; @@ -90,6 +92,7 @@ public SignerConfiguration( final boolean metricsEnabled, final Optional azureKeyVaultParameters, final Optional awsSecretsManagerParameters, + final Optional gcpSecretManagerParameters, final Optional keystoresParameters, final Optional serverTlsOptions, final Optional overriddenCaTrustStore, @@ -134,6 +137,7 @@ public SignerConfiguration( this.metricsEnabled = metricsEnabled; this.azureKeyVaultParameters = azureKeyVaultParameters; this.awsSecretsManagerParameters = awsSecretsManagerParameters; + this.gcpSecretManagerParameters = gcpSecretManagerParameters; this.keystoresParameters = keystoresParameters; this.serverTlsOptions = serverTlsOptions; this.overriddenCaTrustStore = overriddenCaTrustStore; @@ -225,6 +229,10 @@ public Optional getAwsParameters() { return awsSecretsManagerParameters; } + public Optional getGcpParameters() { + return gcpSecretManagerParameters; + } + public Optional getKeystoresParameters() { return keystoresParameters; } diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java index 23a0286d5..0a085fb59 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java @@ -22,6 +22,7 @@ import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition; import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; +import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; import java.nio.file.Path; @@ -51,6 +52,7 @@ public class SignerConfigurationBuilder { private String mode; private AzureKeyVaultParameters azureKeyVaultParameters; private AwsVaultParameters awsVaultParameters; + private GcpSecretManagerParameters gcpSecretManagerParameters; private Map web3SignerEnvironment; private Duration startupTimeout = Boolean.getBoolean("debugSubProcess") ? Duration.ofHours(1) : Duration.ofSeconds(30); @@ -148,6 +150,12 @@ public SignerConfigurationBuilder withAwsParameters(final AwsVaultParameters aws return this; } + public SignerConfigurationBuilder withGcpParameters( + final GcpSecretManagerParameters gcpSecretManagerParameters) { + this.gcpSecretManagerParameters = gcpSecretManagerParameters; + return this; + } + public SignerConfigurationBuilder withKeystoresParameters( final KeystoresParameters keystoresParameters) { this.keystoresParameters = keystoresParameters; @@ -332,6 +340,7 @@ public SignerConfiguration build() { metricsEnabled, Optional.ofNullable(azureKeyVaultParameters), Optional.ofNullable(awsVaultParameters), + Optional.ofNullable(gcpSecretManagerParameters), Optional.ofNullable(keystoresParameters), Optional.ofNullable(serverTlsOptions), Optional.ofNullable(overriddenCaTrustStore), diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java index ba9dfc315..4a5078ace 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java @@ -32,6 +32,7 @@ import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PASSWORD_FILE; import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PATH; +import tech.pegasys.web3signer.commandline.PicoCliGcpSecretManagerParameters; import tech.pegasys.web3signer.core.config.ClientAuthConstraints; import tech.pegasys.web3signer.core.config.TlsOptions; import tech.pegasys.web3signer.core.config.client.ClientTlsOptions; @@ -40,6 +41,7 @@ import tech.pegasys.web3signer.dsl.utils.DatabaseUtil; import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; +import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; import java.io.IOException; @@ -152,6 +154,9 @@ public List createCmdLineParams() { .getAwsParameters() .ifPresent( awsParams -> yamlConfig.append(awsSecretsManagerBulkLoadingOptions(awsParams))); + signerConfig + .getGcpParameters() + .ifPresent(gcpParameters -> yamlConfig.append(gcpBulkLoadingOptions(gcpParameters))); final CommandArgs subCommandArgs = createSubCommandArgs(); params.addAll(subCommandArgs.params); @@ -574,6 +579,31 @@ private String awsSecretsManagerBulkLoadingOptions(final AwsVaultParameters awsV return yamlConfig.toString(); } + private String gcpBulkLoadingOptions( + final GcpSecretManagerParameters gcpSecretManagerParameters) { + final StringBuilder yamlConfig = new StringBuilder(); + yamlConfig.append( + String.format( + YAML_BOOLEAN_FMT, + "eth2." + PicoCliGcpSecretManagerParameters.GCP_SECRETS_ENABLED_OPTION.substring(2), + gcpSecretManagerParameters.isEnabled())); + if (gcpSecretManagerParameters.getProjectId() != null) { + yamlConfig.append( + String.format( + YAML_STRING_FMT, + "eth2." + PicoCliGcpSecretManagerParameters.GCP_PROJECT_ID_OPTION.substring(2), + gcpSecretManagerParameters.getProjectId())); + } + if (gcpSecretManagerParameters.getFilter().isPresent()) { + yamlConfig.append( + String.format( + YAML_STRING_FMT, + "eth2." + PicoCliGcpSecretManagerParameters.GCP_SECRETS_FILTER_OPTION.substring(2), + gcpSecretManagerParameters.getFilter().get())); + } + return yamlConfig.toString(); + } + private String awsKmsBulkLoadingOptions(final AwsVaultParameters awsVaultParameters) { final StringBuilder yamlConfig = new StringBuilder(); diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java index ab9be6f13..e33d7d9ed 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java @@ -32,6 +32,7 @@ import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PASSWORD_FILE; import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PATH; +import tech.pegasys.web3signer.commandline.PicoCliGcpSecretManagerParameters; import tech.pegasys.web3signer.core.config.ClientAuthConstraints; import tech.pegasys.web3signer.core.config.TlsOptions; import tech.pegasys.web3signer.core.config.client.ClientTlsOptions; @@ -40,6 +41,7 @@ import tech.pegasys.web3signer.dsl.utils.DatabaseUtil; import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; +import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; import java.nio.file.Path; @@ -129,6 +131,9 @@ public List createCmdLineParams() { signerConfig .getAwsParameters() .ifPresent(awsParams -> params.addAll(awsSecretsManagerBulkLoadingOptions(awsParams))); + signerConfig + .getGcpParameters() + .ifPresent(gcpParams -> params.addAll(gcpSecretManagerBulkLoadingOptions(gcpParams))); } else if (signerConfig.getMode().equals("eth1")) { params.add("--downstream-http-port"); params.add(Integer.toString(signerConfig.getDownstreamHttpPort())); @@ -310,6 +315,28 @@ private Collection createEth2Args() { return params; } + private Collection gcpSecretManagerBulkLoadingOptions( + final GcpSecretManagerParameters gcpSecretManagerParameters) { + final List params = new ArrayList<>(); + params.add( + PicoCliGcpSecretManagerParameters.GCP_SECRETS_ENABLED_OPTION + + "=" + + gcpSecretManagerParameters.isEnabled()); + if (gcpSecretManagerParameters.getProjectId() != null) { + params.add( + PicoCliGcpSecretManagerParameters.GCP_PROJECT_ID_OPTION + + "=" + + gcpSecretManagerParameters.getProjectId()); + } + if (gcpSecretManagerParameters.getFilter().isPresent()) { + params.add( + PicoCliGcpSecretManagerParameters.GCP_SECRETS_FILTER_OPTION + + "=" + + gcpSecretManagerParameters.getFilter().get()); + } + return params; + } + private Collection awsSecretsManagerBulkLoadingOptions( final AwsVaultParameters awsVaultParameters) { final List params = new ArrayList<>(); diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/GcpSecretManagerAcceptanceTest.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/GcpSecretManagerAcceptanceTest.java new file mode 100644 index 000000000..1d6e92cab --- /dev/null +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/GcpSecretManagerAcceptanceTest.java @@ -0,0 +1,158 @@ +/* + * Copyright 2023 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.tests.bulkloading; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; +import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_GCP_BULK_LOADING; +import static tech.pegasys.web3signer.dsl.utils.HealthCheckResultUtil.getHealtcheckKeysLoaded; +import static tech.pegasys.web3signer.dsl.utils.HealthCheckResultUtil.getHealthcheckErrorCount; +import static tech.pegasys.web3signer.dsl.utils.HealthCheckResultUtil.getHealthcheckStatusValue; + +import tech.pegasys.teku.bls.BLSKeyPair; +import tech.pegasys.web3signer.GcpSecretManagerUtil; +import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder; +import tech.pegasys.web3signer.signing.KeyType; +import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters; +import tech.pegasys.web3signer.signing.config.GcpSecretManagerParametersBuilder; +import tech.pegasys.web3signer.tests.AcceptanceTestBase; + +import java.io.IOException; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.List; + +import io.restassured.http.ContentType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +@EnabledIfEnvironmentVariable( + named = "GCP_PROJECT_ID", + matches = ".*", + disabledReason = "GCP_PROJECT_ID env variable is required") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) // same instance is shared across test methods +public class GcpSecretManagerAcceptanceTest extends AcceptanceTestBase { + private static final Logger LOG = LogManager.getLogger(); + private static final String GCP_PROJECT_ID = System.getenv("GCP_PROJECT_ID"); + + private GcpSecretManagerUtil gcpSecretManagerUtil; + private final List blsKeyPairs = new ArrayList<>(); + private final List secretNames = new ArrayList<>(); + + @BeforeAll + void setupGcpResources() throws IOException { + gcpSecretManagerUtil = new GcpSecretManagerUtil(GCP_PROJECT_ID); + final SecureRandom secureRandom = new SecureRandom(); + + for (int i = 0; i < 4; i++) { + final BLSKeyPair blsKeyPair = BLSKeyPair.random(secureRandom); + String secretName = + gcpSecretManagerUtil.createSecret( + "Secret%d-%s".formatted(i, blsKeyPair.getPublicKey().toString()), + blsKeyPair.getSecretKey().toBytes().toHexString()); + blsKeyPairs.add(blsKeyPair); + secretNames.add(secretName); + } + } + + @ParameterizedTest(name = "{index} - Using config file: {0}") + @ValueSource(booleans = {true, false}) + void secretsAreLoadedFromGCPSecretManagerAndReportedByPublicApi(final boolean useConfigFile) { + final GcpSecretManagerParameters gcpSecretManagerParameters = + GcpSecretManagerParametersBuilder.aGcpParameters() + .withEnabled(true) + .withProjectId(GCP_PROJECT_ID) + .withFilter("name:Secret0 OR name:Secret1") + .build(); + + final SignerConfigurationBuilder configBuilder = + new SignerConfigurationBuilder() + .withUseConfigFile(useConfigFile) + .withMode("eth2") + .withGcpParameters(gcpSecretManagerParameters); + + startSigner(configBuilder.build()); + + final String healthCheckJsonBody = signer.healthcheck().body().asString(); + int keysLoaded = getHealtcheckKeysLoaded(healthCheckJsonBody, KEYS_CHECK_GCP_BULK_LOADING); + + assertThat(keysLoaded).isEqualTo(2); + + signer + .callApiPublicKeys(KeyType.BLS) + .then() + .statusCode(200) + .contentType(ContentType.JSON) + .body( + "", + containsInAnyOrder( + blsKeyPairs.get(0).getPublicKey().toString(), + blsKeyPairs.get(1).getPublicKey().toString()), + "", + hasSize(2)); + } + + @Test + void healthCheckErrorCountWhenInvalidCredentialsAreUsed() { + final boolean useConfigFile = false; + final GcpSecretManagerParameters invalidGcpParams = + GcpSecretManagerParametersBuilder.aGcpParameters() + .withEnabled(true) + .withProjectId("NON_EXISTING_PROJECT") + .build(); + + final SignerConfigurationBuilder configBuilder = + new SignerConfigurationBuilder() + .withUseConfigFile(useConfigFile) + .withMode("eth2") + .withGcpParameters(invalidGcpParams); + + startSigner(configBuilder.build()); + + final String healthCheckJsonBody = signer.healthcheck().body().asString(); + + int keysLoaded = getHealtcheckKeysLoaded(healthCheckJsonBody, KEYS_CHECK_GCP_BULK_LOADING); + int errorCount = getHealthcheckErrorCount(healthCheckJsonBody, KEYS_CHECK_GCP_BULK_LOADING); + + assertThat(keysLoaded).isEqualTo(0); + assertThat(errorCount).isEqualTo(1); + assertThat(getHealthcheckStatusValue(healthCheckJsonBody)).isEqualTo("DOWN"); + } + + @AfterAll + void cleanUpAwsResources() { + if (gcpSecretManagerUtil != null) { + secretNames.forEach( + secretName -> { + try { + gcpSecretManagerUtil.deleteSecret(secretName); + } catch (final RuntimeException e) { + LOG.warn( + "Unexpected error while deleting key {}{}: {}", + gcpSecretManagerUtil.getSecretsManagerPrefix(), + secretName, + e.getMessage()); + } + }); + gcpSecretManagerUtil.close(); + } + } +} diff --git a/signing/build.gradle b/signing/build.gradle index 99b134b56..acb204e36 100644 --- a/signing/build.gradle +++ b/signing/build.gradle @@ -65,5 +65,6 @@ dependencies { testFixturesImplementation 'software.amazon.awssdk:auth' testFixturesImplementation 'software.amazon.awssdk:secretsmanager' testFixturesImplementation 'software.amazon.awssdk:kms' + testFixturesImplementation 'com.google.cloud:google-cloud-secretmanager' testFixturesImplementation project(":common") } diff --git a/signing/src/testFixtures/java/tech/pegasys/web3signer/GcpSecretManagerUtil.java b/signing/src/testFixtures/java/tech/pegasys/web3signer/GcpSecretManagerUtil.java new file mode 100644 index 000000000..d0c01d0e6 --- /dev/null +++ b/signing/src/testFixtures/java/tech/pegasys/web3signer/GcpSecretManagerUtil.java @@ -0,0 +1,72 @@ +/* + * Copyright 2023 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer; + +import java.io.IOException; +import java.util.UUID; + +import com.google.cloud.secretmanager.v1.AddSecretVersionRequest; +import com.google.cloud.secretmanager.v1.ProjectName; +import com.google.cloud.secretmanager.v1.Replication; +import com.google.cloud.secretmanager.v1.Secret; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import com.google.cloud.secretmanager.v1.SecretName; +import com.google.cloud.secretmanager.v1.SecretPayload; +import com.google.protobuf.ByteString; + +public class GcpSecretManagerUtil { + + private final SecretManagerServiceClient secretManagerServiceClient; + private static final String SECRET_MANAGER_PREFIX = "signers-gcp-integration-"; + private final String secretNamePrefix; + private final String projectId; + + public GcpSecretManagerUtil(final String projectId) throws IOException { + this.secretNamePrefix = SECRET_MANAGER_PREFIX + UUID.randomUUID(); + this.projectId = projectId; + this.secretManagerServiceClient = SecretManagerServiceClient.create(); + } + + public String getSecretsManagerPrefix() { + return secretNamePrefix; + } + + public String createSecret(final String providedSecretName, final String secretValue) { + final String secretName = secretNamePrefix + providedSecretName; + final Secret secret = + Secret.newBuilder() + .setReplication( + Replication.newBuilder() + .setAutomatic(Replication.Automatic.newBuilder().build()) + .build()) + .build(); + secretManagerServiceClient.createSecret(ProjectName.of(projectId), secretName, secret); + + final AddSecretVersionRequest request = + AddSecretVersionRequest.newBuilder() + .setParent(SecretName.of(projectId, secretName).toString()) + .setPayload( + SecretPayload.newBuilder().setData(ByteString.copyFromUtf8(secretValue)).build()) + .build(); + secretManagerServiceClient.addSecretVersion(request); + return secretName; + } + + public void deleteSecret(final String secretName) { + secretManagerServiceClient.deleteSecret(SecretName.of(projectId, secretName)); + } + + public void close() { + secretManagerServiceClient.close(); + } +} diff --git a/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/GcpSecretManagerParametersBuilder.java b/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/GcpSecretManagerParametersBuilder.java new file mode 100644 index 000000000..1302488c0 --- /dev/null +++ b/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/GcpSecretManagerParametersBuilder.java @@ -0,0 +1,74 @@ +/* + * Copyright 2023 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.signing.config; + +import java.util.Optional; + +public final class GcpSecretManagerParametersBuilder { + private boolean enabled; + private String projectId; + private Optional filter = Optional.empty(); + + private GcpSecretManagerParametersBuilder() {} + + public static GcpSecretManagerParametersBuilder aGcpParameters() { + return new GcpSecretManagerParametersBuilder(); + } + + public GcpSecretManagerParametersBuilder withEnabled(final boolean enabled) { + this.enabled = enabled; + return this; + } + + public GcpSecretManagerParametersBuilder withProjectId(final String projectId) { + this.projectId = projectId; + return this; + } + + public GcpSecretManagerParametersBuilder withFilter(final String filter) { + this.filter = Optional.of(filter); + return this; + } + + public GcpSecretManagerParameters build() { + return new TestGcpSecretManagerParameters(enabled, projectId, filter); + } + + private static class TestGcpSecretManagerParameters implements GcpSecretManagerParameters { + private final boolean enabled; + private final String projectId; + private final Optional filter; + + private TestGcpSecretManagerParameters( + boolean enabled, String projectId, Optional filter) { + this.enabled = enabled; + this.projectId = projectId; + this.filter = filter; + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public String getProjectId() { + return projectId; + } + + @Override + public Optional getFilter() { + return filter; + } + } +} From 99c03ac03f785fdefd69045e15b2da3d86f03a5f Mon Sep 17 00:00:00 2001 From: Sergey Kisel Date: Sun, 29 Oct 2023 13:18:49 +0100 Subject: [PATCH 8/9] bump library version --- gradle/versions.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 81c4280fb..38c59f286 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -144,7 +144,7 @@ dependencyManagement { dependency 'org.flywaydb:flyway-core:6.1.1' - dependency 'com.google.cloud:google-cloud-secretmanager:2.27.0' + dependency 'com.google.cloud:google-cloud-secretmanager:2.29.0' dependency 'io.zonky.test.postgres:embedded-postgres-binaries-bom:11.19.0' dependency 'io.zonky.test:embedded-postgres:2.0.3' From 10bf087b67f8d043d1c07b73b5db553bbeec8a55 Mon Sep 17 00:00:00 2001 From: Sergey Kisel Date: Mon, 30 Oct 2023 14:30:36 +0100 Subject: [PATCH 9/9] exceptions for more grpc libs --- gradle/versions.gradle | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 38c59f286..fdc5f3b3d 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -222,6 +222,18 @@ dependencyManagement { entry 'grpc-core' entry 'grpc-netty' entry 'grpc-stub' + entry 'grpc-alts' + entry 'grpc-api' + entry 'grpc-auth' + entry 'grpc-context' + entry 'grpc-googleapis' + entry 'grpc-grpclb' + entry 'grpc-inprocess' + entry 'grpc-netty-shaded' + entry 'grpc-protobuf' + entry 'grpc-protobuf-lite' + entry 'grpc-services' + entry 'grpc-xds' } // used in tests to assert log message