Google Secret Manager support

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java

@@ -18,6 +18,7 @@
 import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition;
 import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
 import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
+import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters;
 import tech.pegasys.web3signer.signing.config.KeystoresParameters;
 
 import java.nio.file.Path;
@@ -42,6 +43,7 @@ public class SignerConfiguration {
   private final boolean metricsEnabled;
   private final Optional<AzureKeyVaultParameters> azureKeyVaultParameters;
   private final Optional<AwsVaultParameters> awsSecretsManagerParameters;
+  private final Optional<GcpSecretManagerParameters> gcpSecretManagerParameters;
   private final Optional<KeystoresParameters> keystoresParameters;
   private final Optional<TlsOptions> serverTlsOptions;
   private final Optional<TlsCertificateDefinition> overriddenCaTrustStore;
@@ -89,6 +91,7 @@ public SignerConfiguration(
       final boolean metricsEnabled,
       final Optional<AzureKeyVaultParameters> azureKeyVaultParameters,
       final Optional<AwsVaultParameters> awsSecretsManagerParameters,
+      final Optional<GcpSecretManagerParameters> gcpSecretManagerParameters,
       final Optional<KeystoresParameters> keystoresParameters,
       final Optional<TlsOptions> serverTlsOptions,
       final Optional<TlsCertificateDefinition> overriddenCaTrustStore,
@@ -132,6 +135,7 @@ public SignerConfiguration(
     this.metricsEnabled = metricsEnabled;
     this.azureKeyVaultParameters = azureKeyVaultParameters;
     this.awsSecretsManagerParameters = awsSecretsManagerParameters;
+    this.gcpSecretManagerParameters = gcpSecretManagerParameters;
     this.keystoresParameters = keystoresParameters;
     this.serverTlsOptions = serverTlsOptions;
     this.overriddenCaTrustStore = overriddenCaTrustStore;
@@ -222,6 +226,10 @@ public Optional<AwsVaultParameters> getAwsParameters() {
     return awsSecretsManagerParameters;
   }
 
+  public Optional<GcpSecretManagerParameters> getGcpParameters() {
+    return gcpSecretManagerParameters;
+  }
+
   public Optional<KeystoresParameters> getKeystoresParameters() {
     return keystoresParameters;
   }

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java

@@ -22,6 +22,7 @@
 import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition;
 import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
 import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
+import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters;
 import tech.pegasys.web3signer.signing.config.KeystoresParameters;
 
 import java.nio.file.Path;
@@ -51,6 +52,7 @@ public class SignerConfigurationBuilder {
   private String mode;
   private AzureKeyVaultParameters azureKeyVaultParameters;
   private AwsVaultParameters awsVaultParameters;
+  private GcpSecretManagerParameters gcpSecretManagerParameters;
   private Map<String, String> web3SignerEnvironment;
   private Duration startupTimeout =
       Boolean.getBoolean("debugSubProcess") ? Duration.ofHours(1) : Duration.ofSeconds(30);
@@ -147,6 +149,12 @@ public SignerConfigurationBuilder withAwsParameters(final AwsVaultParameters aws
     return this;
   }
 
+  public SignerConfigurationBuilder withGcpParameters(
+      final GcpSecretManagerParameters gcpSecretManagerParameters) {
+    this.gcpSecretManagerParameters = gcpSecretManagerParameters;
+    return this;
+  }
+
   public SignerConfigurationBuilder withKeystoresParameters(
       final KeystoresParameters keystoresParameters) {
     this.keystoresParameters = keystoresParameters;
@@ -326,6 +334,7 @@ public SignerConfiguration build() {
         metricsEnabled,
         Optional.ofNullable(azureKeyVaultParameters),
         Optional.ofNullable(awsVaultParameters),
+        Optional.ofNullable(gcpSecretManagerParameters),
         Optional.ofNullable(keystoresParameters),
         Optional.ofNullable(serverTlsOptions),
         Optional.ofNullable(overriddenCaTrustStore),

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java

@@ -32,6 +32,7 @@
 import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PASSWORD_FILE;
 import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PATH;
 
+import tech.pegasys.web3signer.commandline.PicoCliGcpSecretManagerParameters;
 import tech.pegasys.web3signer.core.config.ClientAuthConstraints;
 import tech.pegasys.web3signer.core.config.TlsOptions;
 import tech.pegasys.web3signer.core.config.client.ClientTlsOptions;
@@ -40,6 +41,7 @@
 import tech.pegasys.web3signer.dsl.utils.DatabaseUtil;
 import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
 import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
+import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters;
 import tech.pegasys.web3signer.signing.config.KeystoresParameters;
 
 import java.io.IOException;
@@ -152,6 +154,9 @@ public List<String> createCmdLineParams() {
           .getAwsParameters()
           .ifPresent(
               awsParams -> yamlConfig.append(awsSecretsManagerBulkLoadingOptions(awsParams)));
+      signerConfig
+          .getGcpParameters()
+          .ifPresent(gcpParameters -> yamlConfig.append(gcpBulkLoadingOptions(gcpParameters)));
 
       final CommandArgs subCommandArgs = createSubCommandArgs();
       params.addAll(subCommandArgs.params);
@@ -568,6 +573,31 @@ private String awsSecretsManagerBulkLoadingOptions(final AwsVaultParameters awsV
     return yamlConfig.toString();
   }
 
+  private String gcpBulkLoadingOptions(
+      final GcpSecretManagerParameters gcpSecretManagerParameters) {
+    final StringBuilder yamlConfig = new StringBuilder();
+    yamlConfig.append(
+        String.format(
+            YAML_BOOLEAN_FMT,
+            "eth2." + PicoCliGcpSecretManagerParameters.GCP_SECRETS_ENABLED_OPTION.substring(2),
+            gcpSecretManagerParameters.isEnabled()));
+    if (gcpSecretManagerParameters.getProjectId() != null) {
+      yamlConfig.append(
+          String.format(
+              YAML_STRING_FMT,
+              "eth2." + PicoCliGcpSecretManagerParameters.GCP_PROJECT_ID_OPTION.substring(2),
+              gcpSecretManagerParameters.getProjectId()));
+    }
+    if (gcpSecretManagerParameters.getFilter().isPresent()) {
+      yamlConfig.append(
+          String.format(
+              YAML_STRING_FMT,
+              "eth2." + PicoCliGcpSecretManagerParameters.GCP_SECRETS_FILTER_OPTION.substring(2),
+              gcpSecretManagerParameters.getFilter().get()));
+    }
+    return yamlConfig.toString();
+  }
+
   private String awsKmsBulkLoadingOptions(final AwsVaultParameters awsVaultParameters) {
     final StringBuilder yamlConfig = new StringBuilder();
 

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java

@@ -32,6 +32,7 @@
 import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PASSWORD_FILE;
 import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PATH;
 
+import tech.pegasys.web3signer.commandline.PicoCliGcpSecretManagerParameters;
 import tech.pegasys.web3signer.core.config.ClientAuthConstraints;
 import tech.pegasys.web3signer.core.config.TlsOptions;
 import tech.pegasys.web3signer.core.config.client.ClientTlsOptions;
@@ -40,6 +41,7 @@
 import tech.pegasys.web3signer.dsl.utils.DatabaseUtil;
 import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
 import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
+import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters;
 import tech.pegasys.web3signer.signing.config.KeystoresParameters;
 
 import java.nio.file.Path;
@@ -129,6 +131,9 @@ public List<String> createCmdLineParams() {
       signerConfig
           .getAwsParameters()
           .ifPresent(awsParams -> params.addAll(awsSecretsManagerBulkLoadingOptions(awsParams)));
+      signerConfig
+          .getGcpParameters()
+          .ifPresent(gcpParams -> params.addAll(gcpSecretManagerBulkLoadingOptions(gcpParams)));
     } else if (signerConfig.getMode().equals("eth1")) {
       params.add("--downstream-http-port");
       params.add(Integer.toString(signerConfig.getDownstreamHttpPort()));
@@ -305,6 +310,28 @@ private Collection<String> createEth2Args() {
     return params;
   }
 
+  private Collection<String> gcpSecretManagerBulkLoadingOptions(
+      final GcpSecretManagerParameters gcpSecretManagerParameters) {
+    final List<String> params = new ArrayList<>();
+    params.add(
+        PicoCliGcpSecretManagerParameters.GCP_SECRETS_ENABLED_OPTION
+            + "="
+            + gcpSecretManagerParameters.isEnabled());
+    if (gcpSecretManagerParameters.getProjectId() != null) {
+      params.add(
+          PicoCliGcpSecretManagerParameters.GCP_PROJECT_ID_OPTION
+              + "="
+              + gcpSecretManagerParameters.getProjectId());
+    }
+    if (gcpSecretManagerParameters.getFilter().isPresent()) {
+      params.add(
+          PicoCliGcpSecretManagerParameters.GCP_SECRETS_FILTER_OPTION
+              + "="
+              + gcpSecretManagerParameters.getFilter().get());
+    }
+    return params;
+  }
+
   private Collection<String> awsSecretsManagerBulkLoadingOptions(
       final AwsVaultParameters awsVaultParameters) {
     final List<String> params = new ArrayList<>();

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/GcpSecretManagerAcceptanceTest.java

@@ -0,0 +1,158 @@
+/*
+ * Copyright 2023 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.web3signer.tests.bulkloading;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.hasSize;
+import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_GCP_BULK_LOADING;
+import static tech.pegasys.web3signer.dsl.utils.HealthCheckResultUtil.getHealtcheckKeysLoaded;
+import static tech.pegasys.web3signer.dsl.utils.HealthCheckResultUtil.getHealthcheckErrorCount;
+import static tech.pegasys.web3signer.dsl.utils.HealthCheckResultUtil.getHealthcheckStatusValue;
+
+import tech.pegasys.teku.bls.BLSKeyPair;
+import tech.pegasys.web3signer.GcpSecretManagerUtil;
+import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder;
+import tech.pegasys.web3signer.signing.KeyType;
+import tech.pegasys.web3signer.signing.config.GcpSecretManagerParameters;
+import tech.pegasys.web3signer.signing.config.GcpSecretManagerParametersBuilder;
+import tech.pegasys.web3signer.tests.AcceptanceTestBase;
+
+import java.io.IOException;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.List;
+
+import io.restassured.http.ContentType;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+@EnabledIfEnvironmentVariable(
+    named = "GCP_PROJECT_ID",
+    matches = ".*",
+    disabledReason = "GCP_PROJECT_ID env variable is required")
+@TestInstance(TestInstance.Lifecycle.PER_CLASS) // same instance is shared across test methods
+public class GcpSecretManagerAcceptanceTest extends AcceptanceTestBase {
+  private static final Logger LOG = LogManager.getLogger();
+  private static final String GCP_PROJECT_ID = System.getenv("GCP_PROJECT_ID");
+
+  private GcpSecretManagerUtil gcpSecretManagerUtil;
+  private final List<BLSKeyPair> blsKeyPairs = new ArrayList<>();
+  private final List<String> secretNames = new ArrayList<>();
+
+  @BeforeAll
+  void setupGcpResources() throws IOException {
+    gcpSecretManagerUtil = new GcpSecretManagerUtil(GCP_PROJECT_ID);
+    final SecureRandom secureRandom = new SecureRandom();
+
+    for (int i = 0; i < 4; i++) {
+      final BLSKeyPair blsKeyPair = BLSKeyPair.random(secureRandom);
+      String secretName =
+          gcpSecretManagerUtil.createSecret(
+              "Secret%d-%s".formatted(i, blsKeyPair.getPublicKey().toString()),
+              blsKeyPair.getSecretKey().toBytes().toHexString());
+      blsKeyPairs.add(blsKeyPair);
+      secretNames.add(secretName);
+    }
+  }
+
+  @ParameterizedTest(name = "{index} - Using config file: {0}")
+  @ValueSource(booleans = {true, false})
+  void secretsAreLoadedFromGCPSecretManagerAndReportedByPublicApi(final boolean useConfigFile) {
+    final GcpSecretManagerParameters gcpSecretManagerParameters =
+        GcpSecretManagerParametersBuilder.aGcpParameters()
+            .withEnabled(true)
+            .withProjectId(GCP_PROJECT_ID)
+            .withFilter("name:Secret0 OR name:Secret1")
+            .build();
+
+    final SignerConfigurationBuilder configBuilder =
+        new SignerConfigurationBuilder()
+            .withUseConfigFile(useConfigFile)
+            .withMode("eth2")
+            .withGcpParameters(gcpSecretManagerParameters);
+
+    startSigner(configBuilder.build());
+
+    final String healthCheckJsonBody = signer.healthcheck().body().asString();
+    int keysLoaded = getHealtcheckKeysLoaded(healthCheckJsonBody, KEYS_CHECK_GCP_BULK_LOADING);
+
+    assertThat(keysLoaded).isEqualTo(2);
+
+    signer
+        .callApiPublicKeys(KeyType.BLS)
+        .then()
+        .statusCode(200)
+        .contentType(ContentType.JSON)
+        .body(
+            "",
+            containsInAnyOrder(
+                blsKeyPairs.get(0).getPublicKey().toString(),
+                blsKeyPairs.get(1).getPublicKey().toString()),
+            "",
+            hasSize(2));
+  }
+
+  @Test
+  void healthCheckErrorCountWhenInvalidCredentialsAreUsed() {
+    final boolean useConfigFile = false;
+    final GcpSecretManagerParameters invalidGcpParams =
+        GcpSecretManagerParametersBuilder.aGcpParameters()
+            .withEnabled(true)
+            .withProjectId("NON_EXISTING_PROJECT")
+            .build();
+
+    final SignerConfigurationBuilder configBuilder =
+        new SignerConfigurationBuilder()
+            .withUseConfigFile(useConfigFile)
+            .withMode("eth2")
+            .withGcpParameters(invalidGcpParams);
+
+    startSigner(configBuilder.build());
+
+    final String healthCheckJsonBody = signer.healthcheck().body().asString();
+
+    int keysLoaded = getHealtcheckKeysLoaded(healthCheckJsonBody, KEYS_CHECK_GCP_BULK_LOADING);
+    int errorCount = getHealthcheckErrorCount(healthCheckJsonBody, KEYS_CHECK_GCP_BULK_LOADING);
+
+    assertThat(keysLoaded).isEqualTo(0);
+    assertThat(errorCount).isEqualTo(1);
+    assertThat(getHealthcheckStatusValue(healthCheckJsonBody)).isEqualTo("DOWN");
+  }
+
+  @AfterAll
+  void cleanUpAwsResources() {
+    if (gcpSecretManagerUtil != null) {
+      secretNames.forEach(
+          secretName -> {
+            try {
+              gcpSecretManagerUtil.deleteSecret(secretName);
+            } catch (final RuntimeException e) {
+              LOG.warn(
+                  "Unexpected error while deleting key {}{}: {}",
+                  gcpSecretManagerUtil.getSecretsManagerPrefix(),
+                  secretName,
+                  e.getMessage());
+            }
+          });
+      gcpSecretManagerUtil.close();
+    }
+  }
+}