From 7993730f2d15e6a68b907b32d83006b19c9b07cb Mon Sep 17 00:00:00 2001 From: Jason Frame Date: Tue, 12 Sep 2023 13:43:02 +1000 Subject: [PATCH] Aws bulk loading (#889) --- CHANGELOG.md | 5 + .../dsl/signer/SignerConfiguration.java | 8 +- .../signer/SignerConfigurationBuilder.java | 11 +- .../runner/CmdLineParamsConfigFileImpl.java | 119 +++++++-- .../runner/CmdLineParamsDefaultImpl.java | 94 +++++-- .../bulkloading/AwsKmsAcceptanceTest.java | 237 ++++++++++++++++++ .../AwsSecretsManagerAcceptanceTest.java | 22 +- ...ecretsManagerMultiValueAcceptanceTest.java | 10 +- ...cretsManagerPerformanceAcceptanceTest.java | 10 +- .../signing/SecpSigningAcceptanceTest.java | 71 ++---- .../commandline/PicoCliAwsKmsParameters.java | 147 +++++++++++ .../PicoCliAwsSecretsManagerParameters.java | 4 +- .../subcommands/Eth1SubCommand.java | 9 + .../jsonrpcproxy/support/TestEth1Config.java | 15 +- .../pegasys/web3signer/core/Eth1Runner.java | 97 ++++--- .../pegasys/web3signer/core/Eth2Runner.java | 20 +- .../web3signer/core/FilecoinRunner.java | 5 +- .../web3signer/core/config/Eth1Config.java | 3 + ...nerProvider.java => BlsAwsBulkLoader.java} | 6 +- .../bulkloading/SecpAwsBulkLoader.java | 74 ++++++ .../config/AwsSecretsManagerFactory.java | 14 +- ...arameters.java => AwsVaultParameters.java} | 2 +- .../metadata/AwsKeySigningMetadata.java | 4 +- .../signing/secp256k1/aws/AwsKmsClient.java | 104 ++++++++ .../secp256k1/aws/AwsKmsClientKey.java | 30 ++- .../secp256k1/aws/AwsKmsSignerFactory.java | 26 +- .../aws/CachedAwsKmsClientFactory.java | 22 +- .../secp256k1/aws/AwsKmsClientTest.java | 222 ++++++++++++++++ .../secp256k1/aws/AwsKmsSignerTest.java | 53 ++-- .../aws/CachedAwsKmsClientFactoryTest.java | 83 +++--- .../tech/pegasys/web3signer/AwsKmsUtil.java | 91 +++++++ ...er.java => AwsVaultParametersBuilder.java} | 54 ++-- .../DefaultAzureKeyVaultParameters.java | 22 +- 33 files changed, 1366 insertions(+), 328 deletions(-) create mode 100644 acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsKmsAcceptanceTest.java create mode 100644 commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliAwsKmsParameters.java rename signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/{AWSBulkLoadingArtifactSignerProvider.java => BlsAwsBulkLoader.java} (90%) create mode 100644 signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/SecpAwsBulkLoader.java rename signing/src/main/java/tech/pegasys/web3signer/signing/config/{AwsSecretsManagerParameters.java => AwsVaultParameters.java} (96%) create mode 100644 signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClientTest.java create mode 100644 signing/src/testFixtures/java/tech/pegasys/web3signer/AwsKmsUtil.java rename signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/{AwsSecretsManagerParametersBuilder.java => AwsVaultParametersBuilder.java} (75%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d2a26384..c2c1c6bd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Next Release + +### Features Added +- Aws bulk loading for secp256k1 keys in eth1 mode [#889](https://github.com/Consensys/web3signer/pull/889) + ## 23.9.0 ### Features Added diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java index b10ca20ed..37bde8250 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java @@ -16,7 +16,7 @@ import tech.pegasys.web3signer.core.config.client.ClientTlsOptions; import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider; import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; @@ -41,7 +41,7 @@ public class SignerConfiguration { private final List metricsCategories; private final boolean metricsEnabled; private final Optional azureKeyVaultParameters; - private final Optional awsSecretsManagerParameters; + private final Optional awsSecretsManagerParameters; private final Optional keystoresParameters; private final Optional serverTlsOptions; private final Optional overriddenCaTrustStore; @@ -89,7 +89,7 @@ public SignerConfiguration( final List metricsCategories, final boolean metricsEnabled, final Optional azureKeyVaultParameters, - final Optional awsSecretsManagerParameters, + final Optional awsSecretsManagerParameters, final Optional keystoresParameters, final Optional serverTlsOptions, final Optional overriddenCaTrustStore, @@ -221,7 +221,7 @@ public Optional getAzureKeyVaultParameters() { return azureKeyVaultParameters; } - public Optional getAwsSecretsManagerParameters() { + public Optional getAwsParameters() { return awsSecretsManagerParameters; } diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java index 2ed7507b8..23a0286d5 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java @@ -20,7 +20,7 @@ import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider; import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ConfigurationChainId; import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; @@ -50,7 +50,7 @@ public class SignerConfigurationBuilder { private Path slashingProtectionDbPoolConfigurationFile = null; private String mode; private AzureKeyVaultParameters azureKeyVaultParameters; - private AwsSecretsManagerParameters awsSecretsManagerParameters; + private AwsVaultParameters awsVaultParameters; private Map web3SignerEnvironment; private Duration startupTimeout = Boolean.getBoolean("debugSubProcess") ? Duration.ofHours(1) : Duration.ofSeconds(30); @@ -143,9 +143,8 @@ public SignerConfigurationBuilder withAzureKeyVaultParameters( return this; } - public SignerConfigurationBuilder withAwsSecretsManagerParameters( - final AwsSecretsManagerParameters awsSecretsManagerParameters) { - this.awsSecretsManagerParameters = awsSecretsManagerParameters; + public SignerConfigurationBuilder withAwsParameters(final AwsVaultParameters awsVaultParameters) { + this.awsVaultParameters = awsVaultParameters; return this; } @@ -332,7 +331,7 @@ public SignerConfiguration build() { metricsCategories, metricsEnabled, Optional.ofNullable(azureKeyVaultParameters), - Optional.ofNullable(awsSecretsManagerParameters), + Optional.ofNullable(awsVaultParameters), Optional.ofNullable(keystoresParameters), Optional.ofNullable(serverTlsOptions), Optional.ofNullable(overriddenCaTrustStore), diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java index 6667610aa..4a70a5a9a 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java @@ -12,6 +12,13 @@ */ package tech.pegasys.web3signer.dsl.signer.runner; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_ACCESS_KEY_ID_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_AUTH_MODE_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_ENABLED_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_REGION_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_SECRET_ACCESS_KEY_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_TAG_NAMES_FILTER_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_TAG_VALUES_FILTER_OPTION; import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION; import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION; import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION; @@ -31,7 +38,7 @@ import tech.pegasys.web3signer.dsl.signer.SignerConfiguration; import tech.pegasys.web3signer.dsl.signer.WatermarkRepairParameters; import tech.pegasys.web3signer.dsl.utils.DatabaseUtil; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; @@ -143,8 +150,9 @@ public List createCmdLineParams() { } signerConfig - .getAwsSecretsManagerParameters() - .ifPresent(awsParams -> yamlConfig.append(awsBulkLoadingOptions(awsParams))); + .getAwsParameters() + .ifPresent( + awsParams -> yamlConfig.append(awsSecretsManagerBulkLoadingOptions(awsParams))); final CommandArgs subCommandArgs = createSubCommandArgs(); params.addAll(subCommandArgs.params); @@ -160,6 +168,10 @@ public List createCmdLineParams() { signerConfig .getV3KeystoresBulkloadParameters() .ifPresent(setV3KeystoresBulkloadParameters(yamlConfig)); + + signerConfig + .getAwsParameters() + .ifPresent(awsParams -> yamlConfig.append(awsKmsBulkLoadingOptions(awsParams))); } signerConfig @@ -475,71 +487,70 @@ private String createEth2SlashingProtectionArgs() { return yamlConfig.toString(); } - private String awsBulkLoadingOptions( - final AwsSecretsManagerParameters awsSecretsManagerParameters) { + private String awsSecretsManagerBulkLoadingOptions(final AwsVaultParameters awsVaultParameters) { final StringBuilder yamlConfig = new StringBuilder(); yamlConfig.append( String.format( YAML_BOOLEAN_FMT, "eth2." + AWS_SECRETS_ENABLED_OPTION.substring(2), - awsSecretsManagerParameters.isEnabled())); + awsVaultParameters.isEnabled())); yamlConfig.append( String.format( YAML_STRING_FMT, "eth2." + AWS_SECRETS_AUTH_MODE_OPTION.substring(2), - awsSecretsManagerParameters.getAuthenticationMode().name())); + awsVaultParameters.getAuthenticationMode().name())); - if (awsSecretsManagerParameters.getAccessKeyId() != null) { + if (awsVaultParameters.getAccessKeyId() != null) { yamlConfig.append( String.format( YAML_STRING_FMT, "eth2." + AWS_SECRETS_ACCESS_KEY_ID_OPTION.substring(2), - awsSecretsManagerParameters.getAccessKeyId())); + awsVaultParameters.getAccessKeyId())); } - if (awsSecretsManagerParameters.getSecretAccessKey() != null) { + if (awsVaultParameters.getSecretAccessKey() != null) { yamlConfig.append( String.format( YAML_STRING_FMT, "eth2." + AWS_SECRETS_SECRET_ACCESS_KEY_OPTION.substring(2), - awsSecretsManagerParameters.getSecretAccessKey())); + awsVaultParameters.getSecretAccessKey())); } - if (awsSecretsManagerParameters.getRegion() != null) { + if (awsVaultParameters.getRegion() != null) { yamlConfig.append( String.format( YAML_STRING_FMT, "eth2." + AWS_SECRETS_REGION_OPTION.substring(2), - awsSecretsManagerParameters.getRegion())); + awsVaultParameters.getRegion())); } - if (!awsSecretsManagerParameters.getPrefixesFilter().isEmpty()) { + if (!awsVaultParameters.getPrefixesFilter().isEmpty()) { yamlConfig.append( String.format( YAML_STRING_FMT, "eth2." + AWS_SECRETS_PREFIXES_FILTER_OPTION.substring(2), - String.join(",", awsSecretsManagerParameters.getPrefixesFilter()))); + String.join(",", awsVaultParameters.getPrefixesFilter()))); } - if (!awsSecretsManagerParameters.getTagNamesFilter().isEmpty()) { + if (!awsVaultParameters.getTagNamesFilter().isEmpty()) { yamlConfig.append( String.format( YAML_STRING_FMT, "eth2." + AWS_SECRETS_TAG_NAMES_FILTER_OPTION.substring(2), - String.join(",", awsSecretsManagerParameters.getTagNamesFilter()))); + String.join(",", awsVaultParameters.getTagNamesFilter()))); } - if (!awsSecretsManagerParameters.getTagValuesFilter().isEmpty()) { + if (!awsVaultParameters.getTagValuesFilter().isEmpty()) { yamlConfig.append( String.format( YAML_STRING_FMT, "eth2." + AWS_SECRETS_TAG_VALUES_FILTER_OPTION.substring(2), - String.join(",", awsSecretsManagerParameters.getTagValuesFilter()))); + String.join(",", awsVaultParameters.getTagValuesFilter()))); } - awsSecretsManagerParameters + awsVaultParameters .getEndpointOverride() .ifPresent( uri -> @@ -552,6 +563,74 @@ private String awsBulkLoadingOptions( return yamlConfig.toString(); } + private String awsKmsBulkLoadingOptions(final AwsVaultParameters awsVaultParameters) { + final StringBuilder yamlConfig = new StringBuilder(); + + yamlConfig.append( + String.format( + YAML_BOOLEAN_FMT, + "eth1." + AWS_KMS_ENABLED_OPTION.substring(2), + awsVaultParameters.isEnabled())); + + yamlConfig.append( + String.format( + YAML_STRING_FMT, + "eth1." + AWS_KMS_AUTH_MODE_OPTION.substring(2), + awsVaultParameters.getAuthenticationMode().name())); + + if (awsVaultParameters.getAccessKeyId() != null) { + yamlConfig.append( + String.format( + YAML_STRING_FMT, + "eth1." + AWS_KMS_ACCESS_KEY_ID_OPTION.substring(2), + awsVaultParameters.getAccessKeyId())); + } + + if (awsVaultParameters.getSecretAccessKey() != null) { + yamlConfig.append( + String.format( + YAML_STRING_FMT, + "eth1." + AWS_KMS_SECRET_ACCESS_KEY_OPTION.substring(2), + awsVaultParameters.getSecretAccessKey())); + } + + if (awsVaultParameters.getRegion() != null) { + yamlConfig.append( + String.format( + YAML_STRING_FMT, + "eth1." + AWS_KMS_REGION_OPTION.substring(2), + awsVaultParameters.getRegion())); + } + + if (!awsVaultParameters.getTagNamesFilter().isEmpty()) { + yamlConfig.append( + String.format( + YAML_STRING_FMT, + "eth1." + AWS_KMS_TAG_NAMES_FILTER_OPTION.substring(2), + String.join(",", awsVaultParameters.getTagNamesFilter()))); + } + + if (!awsVaultParameters.getTagValuesFilter().isEmpty()) { + yamlConfig.append( + String.format( + YAML_STRING_FMT, + "eth1." + AWS_KMS_TAG_VALUES_FILTER_OPTION.substring(2), + String.join(",", awsVaultParameters.getTagValuesFilter()))); + } + + awsVaultParameters + .getEndpointOverride() + .ifPresent( + uri -> + yamlConfig.append( + String.format( + YAML_STRING_FMT, + "eth1." + AWS_ENDPOINT_OVERRIDE_OPTION.substring(2), + uri))); + + return yamlConfig.toString(); + } + private String formatStringList(final String key, final List stringList) { return stringList.isEmpty() ? String.format("%s: []%n", key) diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java index 299884439..f4007993c 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java @@ -12,6 +12,13 @@ */ package tech.pegasys.web3signer.dsl.signer.runner; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_ACCESS_KEY_ID_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_AUTH_MODE_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_ENABLED_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_REGION_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_SECRET_ACCESS_KEY_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_TAG_NAMES_FILTER_OPTION; +import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_TAG_VALUES_FILTER_OPTION; import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION; import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION; import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION; @@ -31,7 +38,7 @@ import tech.pegasys.web3signer.dsl.signer.SignerConfiguration; import tech.pegasys.web3signer.dsl.signer.WatermarkRepairParameters; import tech.pegasys.web3signer.dsl.utils.DatabaseUtil; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; @@ -120,8 +127,8 @@ public List createCmdLineParams() { } signerConfig - .getAwsSecretsManagerParameters() - .ifPresent(awsParams -> params.addAll(awsBulkLoadingOptions(awsParams))); + .getAwsParameters() + .ifPresent(awsParams -> params.addAll(awsSecretsManagerBulkLoadingOptions(awsParams))); } else if (signerConfig.getMode().equals("eth1")) { params.add("--downstream-http-port"); params.add(Integer.toString(signerConfig.getDownstreamHttpPort())); @@ -136,6 +143,9 @@ public List createCmdLineParams() { signerConfig .getV3KeystoresBulkloadParameters() .ifPresent(setV3KeystoresBulkloadParameters(params)); + signerConfig + .getAwsParameters() + .ifPresent(awsParams -> params.addAll(awsKmsBulkLoadingOptions(awsParams))); } return params; @@ -300,31 +310,31 @@ private Collection createEth2Args() { return params; } - private Collection awsBulkLoadingOptions( - final AwsSecretsManagerParameters awsSecretsManagerParameters) { + private Collection awsSecretsManagerBulkLoadingOptions( + final AwsVaultParameters awsVaultParameters) { final List params = new ArrayList<>(); - params.add(AWS_SECRETS_ENABLED_OPTION + "=" + awsSecretsManagerParameters.isEnabled()); + params.add(AWS_SECRETS_ENABLED_OPTION + "=" + awsVaultParameters.isEnabled()); params.add(AWS_SECRETS_AUTH_MODE_OPTION); - params.add(awsSecretsManagerParameters.getAuthenticationMode().name()); + params.add(awsVaultParameters.getAuthenticationMode().name()); - if (awsSecretsManagerParameters.getAccessKeyId() != null) { + if (awsVaultParameters.getAccessKeyId() != null) { params.add(AWS_SECRETS_ACCESS_KEY_ID_OPTION); - params.add(awsSecretsManagerParameters.getAccessKeyId()); + params.add(awsVaultParameters.getAccessKeyId()); } - if (awsSecretsManagerParameters.getSecretAccessKey() != null) { + if (awsVaultParameters.getSecretAccessKey() != null) { params.add(AWS_SECRETS_SECRET_ACCESS_KEY_OPTION); - params.add(awsSecretsManagerParameters.getSecretAccessKey()); + params.add(awsVaultParameters.getSecretAccessKey()); } - if (awsSecretsManagerParameters.getRegion() != null) { + if (awsVaultParameters.getRegion() != null) { params.add(AWS_SECRETS_REGION_OPTION); - params.add(awsSecretsManagerParameters.getRegion()); + params.add(awsVaultParameters.getRegion()); } - awsSecretsManagerParameters + awsVaultParameters .getEndpointOverride() .ifPresent( uri -> { @@ -332,19 +342,63 @@ private Collection awsBulkLoadingOptions( params.add(uri.toString()); }); - if (!awsSecretsManagerParameters.getPrefixesFilter().isEmpty()) { + if (!awsVaultParameters.getPrefixesFilter().isEmpty()) { params.add(AWS_SECRETS_PREFIXES_FILTER_OPTION); - params.add(String.join(",", awsSecretsManagerParameters.getPrefixesFilter())); + params.add(String.join(",", awsVaultParameters.getPrefixesFilter())); } - if (!awsSecretsManagerParameters.getTagNamesFilter().isEmpty()) { + if (!awsVaultParameters.getTagNamesFilter().isEmpty()) { params.add(AWS_SECRETS_TAG_NAMES_FILTER_OPTION); - params.add(String.join(",", awsSecretsManagerParameters.getTagNamesFilter())); + params.add(String.join(",", awsVaultParameters.getTagNamesFilter())); } - if (!awsSecretsManagerParameters.getTagValuesFilter().isEmpty()) { + if (!awsVaultParameters.getTagValuesFilter().isEmpty()) { params.add(AWS_SECRETS_TAG_VALUES_FILTER_OPTION); - params.add(String.join(",", awsSecretsManagerParameters.getTagValuesFilter())); + params.add(String.join(",", awsVaultParameters.getTagValuesFilter())); + } + + return params; + } + + private Collection awsKmsBulkLoadingOptions(final AwsVaultParameters awsVaultParameters) { + final List params = new ArrayList<>(); + + params.add(AWS_KMS_ENABLED_OPTION + "=" + awsVaultParameters.isEnabled()); + + params.add(AWS_KMS_AUTH_MODE_OPTION); + params.add(awsVaultParameters.getAuthenticationMode().name()); + + if (awsVaultParameters.getAccessKeyId() != null) { + params.add(AWS_KMS_ACCESS_KEY_ID_OPTION); + params.add(awsVaultParameters.getAccessKeyId()); + } + + if (awsVaultParameters.getSecretAccessKey() != null) { + params.add(AWS_KMS_SECRET_ACCESS_KEY_OPTION); + params.add(awsVaultParameters.getSecretAccessKey()); + } + + if (awsVaultParameters.getRegion() != null) { + params.add(AWS_KMS_REGION_OPTION); + params.add(awsVaultParameters.getRegion()); + } + + awsVaultParameters + .getEndpointOverride() + .ifPresent( + uri -> { + params.add(AWS_ENDPOINT_OVERRIDE_OPTION); + params.add(uri.toString()); + }); + + if (!awsVaultParameters.getTagNamesFilter().isEmpty()) { + params.add(AWS_KMS_TAG_NAMES_FILTER_OPTION); + params.add(String.join(",", awsVaultParameters.getTagNamesFilter())); + } + + if (!awsVaultParameters.getTagValuesFilter().isEmpty()) { + params.add(AWS_KMS_TAG_VALUES_FILTER_OPTION); + params.add(String.join(",", awsVaultParameters.getTagValuesFilter())); } return params; diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsKmsAcceptanceTest.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsKmsAcceptanceTest.java new file mode 100644 index 000000000..614ebafe5 --- /dev/null +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsKmsAcceptanceTest.java @@ -0,0 +1,237 @@ +/* + * Copyright 2023 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.tests.bulkloading; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; + +import tech.pegasys.web3signer.AwsKmsUtil; +import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; +import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder; +import tech.pegasys.web3signer.signing.KeyType; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParametersBuilder; +import tech.pegasys.web3signer.signing.secp256k1.EthPublicKeyUtils; +import tech.pegasys.web3signer.tests.AcceptanceTestBase; + +import java.net.URI; +import java.security.interfaces.ECPublicKey; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import io.restassured.http.ContentType; +import io.restassured.response.Response; +import io.vertx.core.json.JsonObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +@EnabledIfEnvironmentVariable( + named = "RW_AWS_ACCESS_KEY_ID", + matches = ".*", + disabledReason = "RW_AWS_ACCESS_KEY_ID env variable is required") +@EnabledIfEnvironmentVariable( + named = "RW_AWS_SECRET_ACCESS_KEY", + matches = ".*", + disabledReason = "RW_AWS_SECRET_ACCESS_KEY env variable is required") +@EnabledIfEnvironmentVariable( + named = "AWS_ACCESS_KEY_ID", + matches = ".*", + disabledReason = "AWS_ACCESS_KEY_ID env variable is required") +@EnabledIfEnvironmentVariable( + named = "AWS_SECRET_ACCESS_KEY", + matches = ".*", + disabledReason = "AWS_SECRET_ACCESS_KEY env variable is required") +@EnabledIfEnvironmentVariable( + named = "AWS_REGION", + matches = ".*", + disabledReason = "AWS_REGION env variable is required") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) // same instance is shared across test methods +public class AwsKmsAcceptanceTest extends AcceptanceTestBase { + private static final Logger LOG = LogManager.getLogger(); + private static final String RW_AWS_ACCESS_KEY_ID = System.getenv("RW_AWS_ACCESS_KEY_ID"); + private static final String RW_AWS_SECRET_ACCESS_KEY = System.getenv("RW_AWS_SECRET_ACCESS_KEY"); + private static final String RO_AWS_ACCESS_KEY_ID = System.getenv("AWS_ACCESS_KEY_ID"); + private static final String RO_AWS_SECRET_ACCESS_KEY = System.getenv("AWS_SECRET_ACCESS_KEY"); + private static final String AWS_REGION = + Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2"); + + // can be pointed to localstack + private final Optional awsEndpointOverride = + System.getenv("AWS_ENDPOINT_OVERRIDE") != null + ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE"))) + : Optional.empty(); + private AwsKmsUtil awsSecretsManagerUtil; + + public record Key(String keyId, String publicKey) {} + + private final List keys = new ArrayList<>(); + + @BeforeAll + void setupAwsResources() { + awsSecretsManagerUtil = + new AwsKmsUtil( + AWS_REGION, + RW_AWS_ACCESS_KEY_ID, + RW_AWS_SECRET_ACCESS_KEY, + Optional.empty(), + awsEndpointOverride); + + for (int i = 0; i < 4; i++) { + final String keyId = awsSecretsManagerUtil.createKey(Map.of("TagName" + i, "TagValue" + i)); + final ECPublicKey publicKey = awsSecretsManagerUtil.publicKey(keyId); + keys.add(new Key(keyId, EthPublicKeyUtils.toHexString(publicKey))); + } + } + + @ParameterizedTest(name = "{index} - Using config file: {0}") + @ValueSource(booleans = {true, false}) + void keysAreLoadedFromAwsKmsAndReportedByPublicApi(final boolean useConfigFile) { + final AwsVaultParameters awsVaultParameters = + AwsVaultParametersBuilder.anAwsParameters() + .withAuthenticationMode(AwsAuthenticationMode.SPECIFIED) + .withRegion(AWS_REGION) + .withAccessKeyId(RO_AWS_ACCESS_KEY_ID) + .withSecretAccessKey(RO_AWS_SECRET_ACCESS_KEY) + .withTagNamesFilter(List.of("TagName0", "TagName1")) + .withTagValuesFilter(List.of("TagValue0", "TagValue1", "TagValue2")) + .withEndpointOverride(awsEndpointOverride) + .build(); + + final SignerConfigurationBuilder configBuilder = + new SignerConfigurationBuilder() + .withUseConfigFile(useConfigFile) + .withMode("eth1") + .withAwsParameters(awsVaultParameters); + + startSigner(configBuilder.build()); + + final Response response = signer.callApiPublicKeys(KeyType.SECP256K1); + response + .then() + .statusCode(200) + .contentType(ContentType.JSON) + .body("", containsInAnyOrder(keys.get(0).publicKey(), keys.get(1).publicKey())); + + final Response healthcheckResponse = signer.healthcheck(); + healthcheckResponse + .then() + .statusCode(200) + .contentType(ContentType.JSON) + .body("status", equalTo("UP")); + + final String jsonBody = healthcheckResponse.body().asString(); + final int keysLoaded = getAwsBulkLoadingData(jsonBody, "keys-loaded"); + assertThat(keysLoaded).isEqualTo(2); + } + + @Test + void healthCheckErrorCountWhenInvalidCredentialsAreUsed() { + final boolean useConfigFile = false; + final AwsVaultParameters invalidCredsParams = + AwsVaultParametersBuilder.anAwsParameters() + .withAuthenticationMode(AwsAuthenticationMode.SPECIFIED) + .withRegion("us-east-2") + .withAccessKeyId("invalid") + .withSecretAccessKey("invalid") + .withPrefixesFilter(List.of("shouldNotExist/")) + .withEndpointOverride(Optional.empty()) + .build(); + + final SignerConfigurationBuilder configBuilder = + new SignerConfigurationBuilder() + .withUseConfigFile(useConfigFile) + .withMode("eth1") + .withAwsParameters(invalidCredsParams); + + startSigner(configBuilder.build()); + + final String healthCheckJsonBody = signer.healthcheck().body().asString(); + + int keysLoaded = getAwsBulkLoadingData(healthCheckJsonBody, "keys-loaded"); + int errorCount = getAwsBulkLoadingData(healthCheckJsonBody, "error-count"); + + assertThat(keysLoaded).isEqualTo(0); + assertThat(errorCount).isEqualTo(1); + assertThat(new JsonObject(healthCheckJsonBody).getString("status")).isEqualTo("DOWN"); + } + + private static int getAwsBulkLoadingData(String healthCheckJsonBody, String dataKey) { + final JsonObject jsonObject = new JsonObject(healthCheckJsonBody); + return jsonObject.getJsonArray("checks").stream() + .map(JsonObject.class::cast) + .filter(check -> "keys-check".equals(check.getString("id"))) + .flatMap(check -> check.getJsonArray("checks").stream()) + .map(JsonObject.class::cast) + .filter(check -> "aws-bulk-loading".equals(check.getString("id"))) + .mapToInt(check -> check.getJsonObject("data").getInteger(dataKey)) + .findFirst() + .orElse(-1); + } + + @ParameterizedTest(name = "{index} - Using config file: {0}") + @ValueSource(booleans = {true, false}) + void keysAreLoadedFromAwsKmsWithEnvironmentAuthModeAndReportedByPublicApi( + final boolean useConfigFile) { + final AwsVaultParameters awsVaultParameters = + AwsVaultParametersBuilder.anAwsParameters() + .withAuthenticationMode(AwsAuthenticationMode.ENVIRONMENT) + .withTagNamesFilter(List.of("TagName2", "TagName3")) + .withTagValuesFilter(List.of("TagValue0", "TagValue2", "TagValue3")) + .withEndpointOverride(awsEndpointOverride) + .build(); + + final SignerConfigurationBuilder configBuilder = + new SignerConfigurationBuilder() + .withUseConfigFile(useConfigFile) + .withMode("eth1") + .withAwsParameters(awsVaultParameters); + + startSigner(configBuilder.build()); + + signer + .callApiPublicKeys(KeyType.SECP256K1) + .then() + .statusCode(200) + .contentType(ContentType.JSON) + .body( + "", + containsInAnyOrder(keys.get(2).publicKey(), keys.get(3).publicKey()), + "", + hasSize(2)); + } + + @AfterAll + void cleanUpAwsResources() { + if (awsSecretsManagerUtil != null) { + keys.forEach( + key -> { + try { + awsSecretsManagerUtil.deleteKey(key.keyId()); + } catch (final RuntimeException e) { + LOG.warn("Unexpected error while deleting key {}: {}", key.keyId(), e.getMessage()); + } + }); + } + } +} diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerAcceptanceTest.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerAcceptanceTest.java index fe6e9f4cf..08a860319 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerAcceptanceTest.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerAcceptanceTest.java @@ -25,8 +25,8 @@ import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder; import tech.pegasys.web3signer.signing.KeyType; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParametersBuilder; import tech.pegasys.web3signer.tests.AcceptanceTestBase; import java.net.URI; @@ -105,8 +105,8 @@ void setupAwsResources() { @ParameterizedTest(name = "{index} - Using config file: {0}") @ValueSource(booleans = {true, false}) void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean useConfigFile) { - final AwsSecretsManagerParameters awsSecretsManagerParameters = - AwsSecretsManagerParametersBuilder.anAwsSecretsManagerParameters() + final AwsVaultParameters awsVaultParameters = + AwsVaultParametersBuilder.anAwsParameters() .withAuthenticationMode(AwsAuthenticationMode.SPECIFIED) .withRegion(AWS_REGION) .withAccessKeyId(RO_AWS_ACCESS_KEY_ID) @@ -121,7 +121,7 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u new SignerConfigurationBuilder() .withUseConfigFile(useConfigFile) .withMode("eth2") - .withAwsSecretsManagerParameters(awsSecretsManagerParameters); + .withAwsParameters(awsVaultParameters); startSigner(configBuilder.build()); @@ -147,8 +147,8 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u @Test void healthCheckErrorCountWhenInvalidCredentialsAreUsed() { final boolean useConfigFile = false; - final AwsSecretsManagerParameters invalidCredsParams = - AwsSecretsManagerParametersBuilder.anAwsSecretsManagerParameters() + final AwsVaultParameters invalidCredsParams = + AwsVaultParametersBuilder.anAwsParameters() .withAuthenticationMode(AwsAuthenticationMode.SPECIFIED) .withRegion("us-east-2") .withAccessKeyId("invalid") @@ -161,7 +161,7 @@ void healthCheckErrorCountWhenInvalidCredentialsAreUsed() { new SignerConfigurationBuilder() .withUseConfigFile(useConfigFile) .withMode("eth2") - .withAwsSecretsManagerParameters(invalidCredsParams); + .withAwsParameters(invalidCredsParams); startSigner(configBuilder.build()); @@ -179,8 +179,8 @@ void healthCheckErrorCountWhenInvalidCredentialsAreUsed() { @ValueSource(booleans = {true, false}) void secretsAreLoadedFromAWSSecretsManagerWithEnvironmentAuthModeAndReportedByPublicApi( final boolean useConfigFile) { - final AwsSecretsManagerParameters awsSecretsManagerParameters = - AwsSecretsManagerParametersBuilder.anAwsSecretsManagerParameters() + final AwsVaultParameters awsVaultParameters = + AwsVaultParametersBuilder.anAwsParameters() .withAuthenticationMode(AwsAuthenticationMode.ENVIRONMENT) .withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix())) .withTagNamesFilter(List.of("TagName2", "TagName3")) @@ -192,7 +192,7 @@ void secretsAreLoadedFromAWSSecretsManagerWithEnvironmentAuthModeAndReportedByPu new SignerConfigurationBuilder() .withUseConfigFile(useConfigFile) .withMode("eth2") - .withAwsSecretsManagerParameters(awsSecretsManagerParameters); + .withAwsParameters(awsVaultParameters); startSigner(configBuilder.build()); diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerMultiValueAcceptanceTest.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerMultiValueAcceptanceTest.java index 44bbb0e00..839d4fe66 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerMultiValueAcceptanceTest.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerMultiValueAcceptanceTest.java @@ -21,8 +21,8 @@ import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder; import tech.pegasys.web3signer.signing.KeyType; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParametersBuilder; import tech.pegasys.web3signer.tests.AcceptanceTestBase; import java.net.URI; @@ -103,8 +103,8 @@ void setupAwsResources() { @ParameterizedTest(name = "{index} -> use config file: {0}") @ValueSource(booleans = {true, false}) void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean useConfigFile) { - final AwsSecretsManagerParameters awsSecretsManagerParameters = - AwsSecretsManagerParametersBuilder.anAwsSecretsManagerParameters() + final AwsVaultParameters awsVaultParameters = + AwsVaultParametersBuilder.anAwsParameters() .withAuthenticationMode(AwsAuthenticationMode.SPECIFIED) .withRegion(AWS_REGION) .withAccessKeyId(RO_AWS_ACCESS_KEY_ID) @@ -118,7 +118,7 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u new SignerConfigurationBuilder() .withUseConfigFile(useConfigFile) .withMode("eth2") - .withAwsSecretsManagerParameters(awsSecretsManagerParameters); + .withAwsParameters(awsVaultParameters); startSigner(configBuilder.build()); diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerPerformanceAcceptanceTest.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerPerformanceAcceptanceTest.java index 1a431aeec..b22d6b772 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerPerformanceAcceptanceTest.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerPerformanceAcceptanceTest.java @@ -19,8 +19,8 @@ import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder; import tech.pegasys.web3signer.signing.KeyType; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParametersBuilder; import tech.pegasys.web3signer.tests.AcceptanceTestBase; import java.net.URI; @@ -120,8 +120,8 @@ void setupAwsResources() { @Test void largeNumberOfKeysAreLoadedSuccessfully() { - final AwsSecretsManagerParameters awsSecretsManagerParameters = - AwsSecretsManagerParametersBuilder.anAwsSecretsManagerParameters() + final AwsVaultParameters awsVaultParameters = + AwsVaultParametersBuilder.anAwsParameters() .withAuthenticationMode(AwsAuthenticationMode.SPECIFIED) .withRegion(AWS_REGION) .withAccessKeyId(RO_AWS_ACCESS_KEY_ID) @@ -133,7 +133,7 @@ void largeNumberOfKeysAreLoadedSuccessfully() { final SignerConfigurationBuilder configBuilder = new SignerConfigurationBuilder() .withMode("eth2") - .withAwsSecretsManagerParameters(awsSecretsManagerParameters) + .withAwsParameters(awsVaultParameters) .withStartupTimeout(STARTUP_TIMEOUT) .withLogLevel(Level.INFO); diff --git a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/SecpSigningAcceptanceTest.java b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/SecpSigningAcceptanceTest.java index 71e39dabd..a4d5769c8 100644 --- a/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/SecpSigningAcceptanceTest.java +++ b/acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/SecpSigningAcceptanceTest.java @@ -17,16 +17,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.web3j.crypto.Sign.signedMessageToKey; -import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; -import tech.pegasys.web3signer.common.config.AwsCredentials; +import tech.pegasys.web3signer.AwsKmsUtil; import tech.pegasys.web3signer.dsl.HashicorpSigningParams; import tech.pegasys.web3signer.dsl.utils.MetadataFileHelpers; import tech.pegasys.web3signer.keystore.hashicorp.dsl.HashicorpNode; import tech.pegasys.web3signer.signing.KeyType; -import tech.pegasys.web3signer.signing.config.AwsCredentialsProviderFactory; import tech.pegasys.web3signer.signing.secp256k1.EthPublicKeyUtils; -import tech.pegasys.web3signer.signing.secp256k1.aws.AwsKmsClient; -import tech.pegasys.web3signer.signing.secp256k1.aws.CachedAwsKmsClientFactory; import java.io.File; import java.math.BigInteger; @@ -35,6 +31,7 @@ import java.nio.file.Path; import java.security.SignatureException; import java.security.interfaces.ECPublicKey; +import java.util.Collections; import java.util.Map; import java.util.Optional; @@ -46,11 +43,6 @@ import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariables; import org.web3j.crypto.Sign.SignatureData; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.services.kms.model.CreateKeyRequest; -import software.amazon.awssdk.services.kms.model.KeySpec; -import software.amazon.awssdk.services.kms.model.KeyUsageType; -import software.amazon.awssdk.services.kms.model.ScheduleKeyDeletionRequest; public class SecpSigningAcceptanceTest extends SigningAcceptanceTestBase { @@ -153,7 +145,14 @@ public void remoteSignWithAwsKMS() { final Optional awsEndpointOverride = Optional.ofNullable(System.getenv("AWS_ENDPOINT_OVERRIDE")).map(URI::create); - final Map.Entry remoteAWSKMSKey = createRemoteAWSKMSKey(); + final AwsKmsUtil awsKmsUtil = + new AwsKmsUtil( + region, + System.getenv("RW_AWS_ACCESS_KEY_ID"), + System.getenv("RW_AWS_SECRET_ACCESS_KEY"), + Optional.ofNullable(System.getenv("AWS_SESSION_TOKEN")), + awsEndpointOverride); + final Map.Entry remoteAWSKMSKey = createRemoteAWSKMSKey(awsKmsUtil); final String awsKeyId = remoteAWSKMSKey.getKey(); final ECPublicKey ecPublicKey = remoteAWSKMSKey.getValue(); @@ -170,7 +169,7 @@ public void remoteSignWithAwsKMS() { signAndVerifySignature(EthPublicKeyUtils.toHexString(ecPublicKey)); } finally { - markAwsKeyForDeletion(region, awsEndpointOverride, awsKeyId); + awsKmsUtil.deleteKey(awsKeyId); } } @@ -206,52 +205,10 @@ private BigInteger recoverPublicKey(final SignatureData signature) { } } - private static Map.Entry createRemoteAWSKMSKey() { - final String region = Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2"); - final Optional awsEndpointOverride = - System.getenv("AWS_ENDPOINT_OVERRIDE") != null - ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE"))) - : Optional.empty(); + private static Map.Entry createRemoteAWSKMSKey(final AwsKmsUtil awsKmsUtil) { + final String testKeyId = awsKmsUtil.createKey(Collections.emptyMap()); - final AwsCredentialsProvider rwAwsCredentialsProvider = - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - AwsAuthenticationMode.SPECIFIED, Optional.of(getAwsCredentialsFromEnvVar())); - final AwsKmsClient rwKmsClient = - new CachedAwsKmsClientFactory(1) - .createKmsClient(rwAwsCredentialsProvider, region, awsEndpointOverride); - // create a test key - final CreateKeyRequest web3SignerTestingKey = - CreateKeyRequest.builder() - .keySpec(KeySpec.ECC_SECG_P256_K1) - .description("Web3Signer Testing Key") - .keyUsage(KeyUsageType.SIGN_VERIFY) - .build(); - - final String testKeyId = rwKmsClient.createKey(web3SignerTestingKey); - final ECPublicKey ecPublicKey = rwKmsClient.getECPublicKey(testKeyId); + final ECPublicKey ecPublicKey = awsKmsUtil.publicKey(testKeyId); return Maps.immutableEntry(testKeyId, ecPublicKey); } - - private static void markAwsKeyForDeletion( - String region, Optional awsEndpointOverride, String awsKeyId) { - // mark aws key for deletion - ScheduleKeyDeletionRequest deletionRequest = - ScheduleKeyDeletionRequest.builder().keyId(awsKeyId).pendingWindowInDays(7).build(); - final AwsCredentialsProvider rwAwsCredentialsProvider = - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - AwsAuthenticationMode.SPECIFIED, Optional.of(getAwsCredentialsFromEnvVar())); - - final AwsKmsClient rwKmsClient = - new CachedAwsKmsClientFactory(1) - .createKmsClient(rwAwsCredentialsProvider, region, awsEndpointOverride); - rwKmsClient.scheduleKeyDeletion(deletionRequest); - } - - private static AwsCredentials getAwsCredentialsFromEnvVar() { - return AwsCredentials.builder() - .withAccessKeyId(System.getenv("RW_AWS_ACCESS_KEY_ID")) - .withSecretAccessKey(System.getenv("RW_AWS_SECRET_ACCESS_KEY")) - .withSessionToken(System.getenv("AWS_SESSION_TOKEN")) - .build(); - } } diff --git a/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliAwsKmsParameters.java b/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliAwsKmsParameters.java new file mode 100644 index 000000000..c9bfcf6ba --- /dev/null +++ b/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliAwsKmsParameters.java @@ -0,0 +1,147 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.commandline; + +import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; + +import java.net.URI; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import picocli.CommandLine.Option; + +public class PicoCliAwsKmsParameters implements AwsVaultParameters { + public static final String AWS_KMS_ENABLED_OPTION = "--aws-kms-enabled"; + public static final String AWS_KMS_AUTH_MODE_OPTION = "--aws-kms-auth-mode"; + public static final String AWS_KMS_ACCESS_KEY_ID_OPTION = "--aws-kms-access-key-id"; + public static final String AWS_KMS_SECRET_ACCESS_KEY_OPTION = "--aws-kms-secret-access-key"; + public static final String AWS_KMS_REGION_OPTION = "--aws-kms-region"; + public static final String AWS_ENDPOINT_OVERRIDE_OPTION = "--aws-endpoint-override"; + public static final String AWS_KMS_TAG_NAMES_FILTER_OPTION = "--aws-kms-tag-names-filter"; + public static final String AWS_KMS_TAG_VALUES_FILTER_OPTION = "--aws-kms-tag-values-filter"; + public static final String AWS_CONNECTION_CACHE_SIZE_OPTION = "--aws-connection-cache-size"; + + @Option( + names = AWS_KMS_ENABLED_OPTION, + description = + "Set to true to enable bulk loading from the AWS KMS service." + + " (Default: ${DEFAULT-VALUE})", + paramLabel = "") + private boolean awsKmsManagerEnabled = false; + + @Option( + names = AWS_KMS_AUTH_MODE_OPTION, + description = + "Authentication mode for AWS KMS service. Valid Values: [${COMPLETION-CANDIDATES}]" + + " (Default: ${DEFAULT-VALUE})", + paramLabel = "") + private AwsAuthenticationMode authenticationMode = AwsAuthenticationMode.SPECIFIED; + + @Option( + names = {AWS_KMS_ACCESS_KEY_ID_OPTION}, + description = + "AWS Access Key Id to authenticate Aws KMS. Required for SPECIFIED authentication mode.", + paramLabel = "") + private String accessKeyId; + + @Option( + names = {AWS_KMS_SECRET_ACCESS_KEY_OPTION}, + description = + "AWS Secret Access Key to authenticate Aws KMS. Required for SPECIFIED authentication mode.", + paramLabel = "") + private String secretAccessKey; + + @Option( + names = {AWS_KMS_REGION_OPTION}, + description = + "AWS region where KMS is available. Required for SPECIFIED authentication mode.", + paramLabel = "") + private String region; + + @Option( + names = {AWS_ENDPOINT_OVERRIDE_OPTION}, + description = "Override AWS endpoint.", + paramLabel = "") + private Optional endpointOverride; + + @Option( + names = AWS_KMS_TAG_NAMES_FILTER_OPTION, + description = + "Optional comma-separated list of tag names filter to apply while fetching key ids from AWS KMS." + + " Applied as AND operation with other filters.", + split = ",") + private List tagNamesFilter = Collections.emptyList(); + + @Option( + names = AWS_KMS_TAG_VALUES_FILTER_OPTION, + description = + "Optional comma-separated list of tag values filter to apply while fetching key ids from AWS KMS." + + " Applied as AND operation with other filters.", + split = ",") + private List tagValuesFilter = Collections.emptyList(); + + @Option( + names = {AWS_CONNECTION_CACHE_SIZE_OPTION}, + paramLabel = "", + description = + "Maximum number of connections to cache to the AWS KMS (default: ${DEFAULT-VALUE})") + private long cacheMaximumSize = 1; + + @Override + public boolean isEnabled() { + return awsKmsManagerEnabled; + } + + @Override + public AwsAuthenticationMode getAuthenticationMode() { + return authenticationMode; + } + + @Override + public String getAccessKeyId() { + return accessKeyId; + } + + @Override + public String getSecretAccessKey() { + return secretAccessKey; + } + + @Override + public String getRegion() { + return region; + } + + @Override + public long getCacheMaximumSize() { + return cacheMaximumSize; + } + + @Override + public Collection getTagNamesFilter() { + return tagNamesFilter; + } + + @Override + public Collection getTagValuesFilter() { + return tagValuesFilter; + } + + @Override + public Optional getEndpointOverride() { + return endpointOverride; + } +} diff --git a/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliAwsSecretsManagerParameters.java b/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliAwsSecretsManagerParameters.java index a3e1f6a63..ae59ad906 100644 --- a/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliAwsSecretsManagerParameters.java +++ b/commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliAwsSecretsManagerParameters.java @@ -13,7 +13,7 @@ package tech.pegasys.web3signer.commandline; import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import java.net.URI; import java.util.Collection; @@ -24,7 +24,7 @@ import picocli.CommandLine; import picocli.CommandLine.Option; -public class PicoCliAwsSecretsManagerParameters implements AwsSecretsManagerParameters { +public class PicoCliAwsSecretsManagerParameters implements AwsVaultParameters { public static final String AWS_SECRETS_ENABLED_OPTION = "--aws-secrets-enabled"; public static final String AWS_SECRETS_AUTH_MODE_OPTION = "--aws-secrets-auth-mode"; public static final String AWS_SECRETS_ACCESS_KEY_ID_OPTION = "--aws-secrets-access-key-id"; diff --git a/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth1SubCommand.java b/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth1SubCommand.java index 7ba4065ee..487f191ff 100644 --- a/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth1SubCommand.java +++ b/commandline/src/main/java/tech/pegasys/web3signer/commandline/subcommands/Eth1SubCommand.java @@ -20,6 +20,7 @@ import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PASSWORDS_PATH; import static tech.pegasys.web3signer.signing.config.KeystoresParameters.KEYSTORES_PASSWORD_FILE; +import tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters; import tech.pegasys.web3signer.commandline.PicoCliEth1AzureKeyVaultParameters; import tech.pegasys.web3signer.commandline.annotations.RequiredOption; import tech.pegasys.web3signer.commandline.config.PicoV3KeystoresBulkloadParameters; @@ -30,6 +31,7 @@ import tech.pegasys.web3signer.core.config.client.ClientTlsOptions; import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider; import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ConfigurationChainId; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; @@ -152,6 +154,8 @@ public void setDownstreamHttpPath(final String path) { @CommandLine.Mixin private PicoV3KeystoresBulkloadParameters picoV3KeystoresBulkloadParameters; + @CommandLine.Mixin private PicoCliAwsKmsParameters awsParameters; + @Override public Runner createRunner() { return new Eth1Runner(config, this); @@ -240,6 +244,11 @@ public AzureKeyVaultParameters getAzureKeyVaultConfig() { return azureKeyVaultParameters; } + @Override + public AwsVaultParameters getAwsVaultParameters() { + return awsParameters; + } + @CommandLine.Option( names = {"--aws-kms-client-cache-size"}, paramLabel = "", diff --git a/core/src/integrationTest/java/tech/pegasys/web3signer/core/jsonrpcproxy/support/TestEth1Config.java b/core/src/integrationTest/java/tech/pegasys/web3signer/core/jsonrpcproxy/support/TestEth1Config.java index 487e5a2ea..8c67d1794 100644 --- a/core/src/integrationTest/java/tech/pegasys/web3signer/core/jsonrpcproxy/support/TestEth1Config.java +++ b/core/src/integrationTest/java/tech/pegasys/web3signer/core/jsonrpcproxy/support/TestEth1Config.java @@ -16,12 +16,15 @@ import tech.pegasys.web3signer.core.config.client.ClientTlsOptions; import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider; import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ConfigurationChainId; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParametersBuilder; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.DefaultAzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; import java.nio.file.Path; import java.time.Duration; +import java.util.Collections; import java.util.Optional; public class TestEth1Config implements Eth1Config { @@ -96,7 +99,17 @@ public ChainIdProvider getChainId() { @Override public AzureKeyVaultParameters getAzureKeyVaultConfig() { - return new DefaultAzureKeyVaultParameters("", "", "", ""); + return new DefaultAzureKeyVaultParameters("", "", "", "", Collections.emptyMap(), 60, false); + } + + @Override + public AwsVaultParameters getAwsVaultParameters() { + return AwsVaultParametersBuilder.anAwsParameters() + .withAccessKeyId("") + .withSecretAccessKey("") + .withRegion("") + .withEnabled(false) + .build(); } @Override diff --git a/core/src/main/java/tech/pegasys/web3signer/core/Eth1Runner.java b/core/src/main/java/tech/pegasys/web3signer/core/Eth1Runner.java index 2e55d53a7..e33bc4e6c 100644 --- a/core/src/main/java/tech/pegasys/web3signer/core/Eth1Runner.java +++ b/core/src/main/java/tech/pegasys/web3signer/core/Eth1Runner.java @@ -12,6 +12,7 @@ */ package tech.pegasys.web3signer.core; +import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_AWS_BULK_LOADING; import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_AZURE_BULK_LOADING; import static tech.pegasys.web3signer.core.config.HealthCheckNames.KEYS_CHECK_V3_KEYSTORES_BULK_LOADING; import static tech.pegasys.web3signer.signing.KeyType.SECP256K1; @@ -45,6 +46,7 @@ import tech.pegasys.web3signer.signing.ArtifactSignerProvider; import tech.pegasys.web3signer.signing.EthSecpArtifactSigner; import tech.pegasys.web3signer.signing.SecpArtifactSignature; +import tech.pegasys.web3signer.signing.bulkloading.SecpAwsBulkLoader; import tech.pegasys.web3signer.signing.bulkloading.SecpAzureBulkLoader; import tech.pegasys.web3signer.signing.bulkloading.SecpV3KeystoresBulkLoader; import tech.pegasys.web3signer.signing.config.AzureKeyVaultFactory; @@ -59,6 +61,7 @@ import tech.pegasys.web3signer.signing.config.metadata.parser.YamlSignerParser; import tech.pegasys.web3signer.signing.config.metadata.yubihsm.YubiHsmOpaqueDataProvider; import tech.pegasys.web3signer.signing.secp256k1.aws.AwsKmsSignerFactory; +import tech.pegasys.web3signer.signing.secp256k1.aws.CachedAwsKmsClientFactory; import tech.pegasys.web3signer.signing.secp256k1.azure.AzureHttpClientFactory; import tech.pegasys.web3signer.signing.secp256k1.azure.AzureKeyVaultSignerFactory; @@ -173,11 +176,21 @@ protected ArtifactSignerProvider createArtifactSignerProvider( registerClose(azureKeyVaultFactory::close); final AzureKeyVaultSignerFactory azureSignerFactory = new AzureKeyVaultSignerFactory(azureKeyVaultFactory, azureHttpClientFactory); - + final CachedAwsKmsClientFactory cachedAwsKmsClientFactory = + new CachedAwsKmsClientFactory(eth1Config.getAwsKmsClientCacheSize()); + final AwsKmsSignerFactory awsKmsSignerFactory = + new AwsKmsSignerFactory(cachedAwsKmsClientFactory, true); + signers.addAll( + loadSignersFromKeyConfigFiles( + vertx, azureKeyVaultFactory, azureSignerFactory, awsKmsSignerFactory) + .getValues()); signers.addAll( - loadSignersFromKeyConfigFiles(vertx, azureKeyVaultFactory, azureSignerFactory) + bulkLoadSigners( + azureKeyVaultFactory, + azureSignerFactory, + cachedAwsKmsClientFactory, + awsKmsSignerFactory) .getValues()); - signers.addAll(bulkLoadSigners(azureKeyVaultFactory, azureSignerFactory).getValues()); return signers; }); } @@ -185,11 +198,9 @@ protected ArtifactSignerProvider createArtifactSignerProvider( private MappedResults loadSignersFromKeyConfigFiles( final Vertx vertx, final AzureKeyVaultFactory azureKeyVaultFactory, - final AzureKeyVaultSignerFactory azureSignerFactory) { + final AzureKeyVaultSignerFactory azureSignerFactory, + final AwsKmsSignerFactory awsKmsSignerFactory) { final HashicorpConnectionFactory hashicorpConnectionFactory = new HashicorpConnectionFactory(); - final boolean applySha3Hash = true; - final AwsKmsSignerFactory awsKmsSignerFactory = - new AwsKmsSignerFactory(eth1Config.getAwsKmsClientCacheSize(), applySha3Hash); try (final InterlockKeyProvider interlockKeyProvider = new InterlockKeyProvider(vertx); final YubiHsmOpaqueDataProvider yubiHsmOpaqueDataProvider = new YubiHsmOpaqueDataProvider()) { @@ -218,29 +229,18 @@ private MappedResults loadSignersFromKeyConfigFiles( private MappedResults bulkLoadSigners( final AzureKeyVaultFactory azureKeyVaultFactory, - final AzureKeyVaultSignerFactory azureSignerFactory) { + final AzureKeyVaultSignerFactory azureSignerFactory, + final CachedAwsKmsClientFactory cachedAwsKmsClientFactory, + final AwsKmsSignerFactory awsKmsSignerFactory) { MappedResults results = MappedResults.newSetInstance(); - final AzureKeyVaultParameters azureKeyVaultConfig = eth1Config.getAzureKeyVaultConfig(); - if (azureKeyVaultConfig.isAzureKeyVaultEnabled()) { - LOG.info("Bulk loading keys from Azure key vault ... "); - final AzureKeyVault azureKeyVault = - azureKeyVaultFactory.createAzureKeyVault( - azureKeyVaultConfig.getClientId(), - azureKeyVaultConfig.getClientSecret(), - azureKeyVaultConfig.getKeyVaultName(), - azureKeyVaultConfig.getTenantId(), - azureKeyVaultConfig.getAuthenticationMode(), - azureKeyVaultConfig.getTimeout()); - final SecpAzureBulkLoader secpAzureBulkLoader = - new SecpAzureBulkLoader(azureKeyVault, azureSignerFactory); - final MappedResults azureResult = - secpAzureBulkLoader.load(azureKeyVaultConfig); - LOG.info( - "Keys loaded from Azure: [{}], with error count: [{}]", - azureResult.getValues().size(), - azureResult.getErrorCount()); - registerSignerLoadingHealthCheck(KEYS_CHECK_AZURE_BULK_LOADING, azureResult); - results = MappedResults.merge(results, azureResult); + if (eth1Config.getAzureKeyVaultConfig().isAzureKeyVaultEnabled()) { + results = + MappedResults.merge(results, bulkLoadAzureKeys(azureKeyVaultFactory, azureSignerFactory)); + } + if (eth1Config.getAwsVaultParameters().isEnabled()) { + results = + MappedResults.merge( + results, bulkLoadAwsKeys(cachedAwsKmsClientFactory, awsKmsSignerFactory)); } // v3 bulk loading @@ -249,6 +249,45 @@ private MappedResults bulkLoadSigners( return results; } + private MappedResults bulkLoadAzureKeys( + AzureKeyVaultFactory azureKeyVaultFactory, AzureKeyVaultSignerFactory azureSignerFactory) { + LOG.info("Bulk loading keys from Azure key vault ... "); + final AzureKeyVaultParameters azureKeyVaultConfig = eth1Config.getAzureKeyVaultConfig(); + final AzureKeyVault azureKeyVault = + azureKeyVaultFactory.createAzureKeyVault( + azureKeyVaultConfig.getClientId(), + azureKeyVaultConfig.getClientSecret(), + azureKeyVaultConfig.getKeyVaultName(), + azureKeyVaultConfig.getTenantId(), + azureKeyVaultConfig.getAuthenticationMode(), + azureKeyVaultConfig.getTimeout()); + final SecpAzureBulkLoader secpAzureBulkLoader = + new SecpAzureBulkLoader(azureKeyVault, azureSignerFactory); + final MappedResults azureResult = secpAzureBulkLoader.load(azureKeyVaultConfig); + LOG.info( + "Keys loaded from Azure: [{}], with error count: [{}]", + azureResult.getValues().size(), + azureResult.getErrorCount()); + registerSignerLoadingHealthCheck(KEYS_CHECK_AZURE_BULK_LOADING, azureResult); + return azureResult; + } + + private MappedResults bulkLoadAwsKeys( + CachedAwsKmsClientFactory cachedAwsKmsClientFactory, + AwsKmsSignerFactory awsKmsSignerFactory) { + LOG.info("Bulk loading keys from AWS KMS key vault ... "); + final SecpAwsBulkLoader secpAwsBulkLoader = + new SecpAwsBulkLoader(cachedAwsKmsClientFactory, awsKmsSignerFactory); + final MappedResults awsResult = + secpAwsBulkLoader.load(eth1Config.getAwsVaultParameters()); + LOG.info( + "Keys loaded from AWS: [{}], with error count: [{}]", + awsResult.getValues().size(), + awsResult.getErrorCount()); + registerSignerLoadingHealthCheck(KEYS_CHECK_AWS_BULK_LOADING, awsResult); + return awsResult; + } + private MappedResults bulkloadV3Keystores() { final KeystoresParameters v3WalletBLParams = eth1Config.getV3KeystoresBulkLoadParameters(); if (!v3WalletBLParams.isEnabled()) { diff --git a/core/src/main/java/tech/pegasys/web3signer/core/Eth2Runner.java b/core/src/main/java/tech/pegasys/web3signer/core/Eth2Runner.java index 04cbd0713..295d3e517 100644 --- a/core/src/main/java/tech/pegasys/web3signer/core/Eth2Runner.java +++ b/core/src/main/java/tech/pegasys/web3signer/core/Eth2Runner.java @@ -43,9 +43,9 @@ import tech.pegasys.web3signer.signing.FileValidatorManager; import tech.pegasys.web3signer.signing.KeystoreFileManager; import tech.pegasys.web3signer.signing.ValidatorManager; -import tech.pegasys.web3signer.signing.bulkloading.AWSBulkLoadingArtifactSignerProvider; +import tech.pegasys.web3signer.signing.bulkloading.BlsAwsBulkLoader; import tech.pegasys.web3signer.signing.bulkloading.BlsKeystoreBulkLoader; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultFactory; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.DefaultArtifactSignerProvider; @@ -94,7 +94,7 @@ public class Eth2Runner extends Runner { private final Optional slashingProtectionContext; private final AzureKeyVaultParameters azureKeyVaultParameters; - private final AwsSecretsManagerParameters awsSecretsManagerParameters; + private final AwsVaultParameters awsVaultParameters; private final SlashingProtectionParameters slashingProtectionParameters; private final boolean pruningEnabled; private final KeystoresParameters keystoresParameters; @@ -106,7 +106,7 @@ public Eth2Runner( final SlashingProtectionParameters slashingProtectionParameters, final AzureKeyVaultParameters azureKeyVaultParameters, final KeystoresParameters keystoresParameters, - final AwsSecretsManagerParameters awsSecretsManagerParameters, + final AwsVaultParameters awsVaultParameters, final Spec eth2Spec, final boolean isKeyManagerApiEnabled) { super(baseConfig); @@ -117,7 +117,7 @@ public Eth2Runner( this.keystoresParameters = keystoresParameters; this.eth2Spec = eth2Spec; this.isKeyManagerApiEnabled = isKeyManagerApiEnabled; - this.awsSecretsManagerParameters = awsSecretsManagerParameters; + this.awsVaultParameters = awsVaultParameters; } private Optional createSlashingProtection( @@ -272,7 +272,7 @@ private MappedResults loadSignersFromKeyConfigFiles( final YubiHsmOpaqueDataProvider yubiHsmOpaqueDataProvider = new YubiHsmOpaqueDataProvider(); final AwsSecretsManagerProvider awsSecretsManagerProvider = - new AwsSecretsManagerProvider(awsSecretsManagerParameters.getCacheMaximumSize()); ) { + new AwsSecretsManagerProvider(awsVaultParameters.getCacheMaximumSize()); ) { final AbstractArtifactSignerFactory artifactSignerFactory = new BlsArtifactSignerFactory( baseConfig.getKeyConfigPath(), @@ -338,13 +338,11 @@ private MappedResults bulkLoadSigners( results = MappedResults.merge(results, keystoreSignersResult); } - if (awsSecretsManagerParameters.isEnabled()) { + if (awsVaultParameters.isEnabled()) { LOG.info("Bulk loading keys from AWS Secrets Manager ... "); - final AWSBulkLoadingArtifactSignerProvider awsBulkLoadingArtifactSignerProvider = - new AWSBulkLoadingArtifactSignerProvider(); + final BlsAwsBulkLoader blsAwsBulkLoader = new BlsAwsBulkLoader(); - final MappedResults awsResult = - awsBulkLoadingArtifactSignerProvider.load(awsSecretsManagerParameters); + final MappedResults awsResult = blsAwsBulkLoader.load(awsVaultParameters); LOG.info( "Keys loaded from AWS Secrets Manager: [{}], with error count: [{}]", awsResult.getValues().size(), diff --git a/core/src/main/java/tech/pegasys/web3signer/core/FilecoinRunner.java b/core/src/main/java/tech/pegasys/web3signer/core/FilecoinRunner.java index cd78b5490..0071b74d9 100644 --- a/core/src/main/java/tech/pegasys/web3signer/core/FilecoinRunner.java +++ b/core/src/main/java/tech/pegasys/web3signer/core/FilecoinRunner.java @@ -39,6 +39,7 @@ import tech.pegasys.web3signer.signing.config.metadata.yubihsm.YubiHsmOpaqueDataProvider; import tech.pegasys.web3signer.signing.filecoin.FilecoinNetwork; import tech.pegasys.web3signer.signing.secp256k1.aws.AwsKmsSignerFactory; +import tech.pegasys.web3signer.signing.secp256k1.aws.CachedAwsKmsClientFactory; import tech.pegasys.web3signer.signing.secp256k1.azure.AzureHttpClientFactory; import tech.pegasys.web3signer.signing.secp256k1.azure.AzureKeyVaultSignerFactory; @@ -117,9 +118,11 @@ protected ArtifactSignerProvider createArtifactSignerProvider( final AzureHttpClientFactory azureHttpClientFactory = new AzureHttpClientFactory(); final AzureKeyVaultSignerFactory azureSignerFactory = new AzureKeyVaultSignerFactory(azureKeyVaultFactory, azureHttpClientFactory); + final CachedAwsKmsClientFactory cachedAwsKmsClientFactory = + new CachedAwsKmsClientFactory(awsKmsClientCacheSize); final boolean applySha3Hash = false; final AwsKmsSignerFactory awsKmsSignerFactory = - new AwsKmsSignerFactory(awsKmsClientCacheSize, applySha3Hash); + new AwsKmsSignerFactory(cachedAwsKmsClientFactory, applySha3Hash); try (final HashicorpConnectionFactory hashicorpConnectionFactory = new HashicorpConnectionFactory(); diff --git a/core/src/main/java/tech/pegasys/web3signer/core/config/Eth1Config.java b/core/src/main/java/tech/pegasys/web3signer/core/config/Eth1Config.java index e6112c0bb..649352a48 100644 --- a/core/src/main/java/tech/pegasys/web3signer/core/config/Eth1Config.java +++ b/core/src/main/java/tech/pegasys/web3signer/core/config/Eth1Config.java @@ -14,6 +14,7 @@ import tech.pegasys.web3signer.core.config.client.ClientTlsOptions; import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters; import tech.pegasys.web3signer.signing.config.KeystoresParameters; @@ -44,6 +45,8 @@ public interface Eth1Config { AzureKeyVaultParameters getAzureKeyVaultConfig(); + AwsVaultParameters getAwsVaultParameters(); + long getAwsKmsClientCacheSize(); KeystoresParameters getV3KeystoresBulkLoadParameters(); diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/AWSBulkLoadingArtifactSignerProvider.java b/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/BlsAwsBulkLoader.java similarity index 90% rename from signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/AWSBulkLoadingArtifactSignerProvider.java rename to signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/BlsAwsBulkLoader.java index f33db9b4a..8ef9368a3 100644 --- a/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/AWSBulkLoadingArtifactSignerProvider.java +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/BlsAwsBulkLoader.java @@ -20,15 +20,15 @@ import tech.pegasys.web3signer.signing.ArtifactSigner; import tech.pegasys.web3signer.signing.BlsArtifactSigner; import tech.pegasys.web3signer.signing.config.AwsSecretsManagerFactory; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import tech.pegasys.web3signer.signing.config.metadata.SignerOrigin; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -public class AWSBulkLoadingArtifactSignerProvider { +public class BlsAwsBulkLoader { - public MappedResults load(final AwsSecretsManagerParameters parameters) { + public MappedResults load(final AwsVaultParameters parameters) { try (final AwsSecretsManagerProvider awsSecretsManagerProvider = new AwsSecretsManagerProvider(parameters.getCacheMaximumSize())) { final AwsSecretsManager awsSecretsManager = diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/SecpAwsBulkLoader.java b/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/SecpAwsBulkLoader.java new file mode 100644 index 000000000..b82364975 --- /dev/null +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/bulkloading/SecpAwsBulkLoader.java @@ -0,0 +1,74 @@ +/* + * Copyright 2022 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.signing.bulkloading; + +import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; +import tech.pegasys.web3signer.common.config.AwsCredentials; +import tech.pegasys.web3signer.keystorage.common.MappedResults; +import tech.pegasys.web3signer.signing.ArtifactSigner; +import tech.pegasys.web3signer.signing.EthSecpArtifactSigner; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; +import tech.pegasys.web3signer.signing.config.metadata.AwsKmsMetadata; +import tech.pegasys.web3signer.signing.secp256k1.aws.AwsKmsClient; +import tech.pegasys.web3signer.signing.secp256k1.aws.AwsKmsSignerFactory; +import tech.pegasys.web3signer.signing.secp256k1.aws.CachedAwsKmsClientFactory; + +import java.util.Optional; + +public class SecpAwsBulkLoader { + private final CachedAwsKmsClientFactory cachedAwsKmsClientFactory; + private final AwsKmsSignerFactory awsKmsSignerFactory; + + public SecpAwsBulkLoader( + final CachedAwsKmsClientFactory cachedAwsKmsClientFactory, + final AwsKmsSignerFactory awsKmsSignerFactory) { + this.cachedAwsKmsClientFactory = cachedAwsKmsClientFactory; + this.awsKmsSignerFactory = awsKmsSignerFactory; + } + + public MappedResults load(final AwsVaultParameters parameters) { + final Optional awsCredentials = + parameters.getAuthenticationMode() == AwsAuthenticationMode.SPECIFIED + ? Optional.of( + AwsCredentials.builder() + .withAccessKeyId(parameters.getAccessKeyId()) + .withSecretAccessKey(parameters.getSecretAccessKey()) + .build()) + : Optional.empty(); + + final AwsKmsClient kmsClient = + cachedAwsKmsClientFactory.createKmsClient( + parameters.getAuthenticationMode(), + awsCredentials, + parameters.getRegion(), + parameters.getEndpointOverride()); + return kmsClient.mapKeyList( + kl -> createSigner(awsCredentials, parameters, kl.keyId()), + parameters.getTagNamesFilter(), + parameters.getTagValuesFilter()); + } + + private EthSecpArtifactSigner createSigner( + final Optional awsCredentials, + final AwsVaultParameters awsVaultParameters, + final String keyId) { + return new EthSecpArtifactSigner( + awsKmsSignerFactory.createSigner( + new AwsKmsMetadata( + awsVaultParameters.getAuthenticationMode(), + awsVaultParameters.getRegion(), + awsCredentials, + keyId, + awsVaultParameters.getEndpointOverride()))); + } +} diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerFactory.java b/signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerFactory.java index b22aefcca..c0c81d302 100644 --- a/signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerFactory.java +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerFactory.java @@ -19,17 +19,17 @@ public class AwsSecretsManagerFactory { public static AwsSecretsManager createAwsSecretsManager( final AwsSecretsManagerProvider awsSecretsManagerProvider, - final AwsSecretsManagerParameters awsSecretsManagerParameters) { - switch (awsSecretsManagerParameters.getAuthenticationMode()) { + final AwsVaultParameters awsVaultParameters) { + switch (awsVaultParameters.getAuthenticationMode()) { case SPECIFIED: return awsSecretsManagerProvider.createAwsSecretsManager( - awsSecretsManagerParameters.getAccessKeyId(), - awsSecretsManagerParameters.getSecretAccessKey(), - awsSecretsManagerParameters.getRegion(), - awsSecretsManagerParameters.getEndpointOverride()); + awsVaultParameters.getAccessKeyId(), + awsVaultParameters.getSecretAccessKey(), + awsVaultParameters.getRegion(), + awsVaultParameters.getEndpointOverride()); default: return awsSecretsManagerProvider.createAwsSecretsManager( - awsSecretsManagerParameters.getEndpointOverride()); + awsVaultParameters.getEndpointOverride()); } } } diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerParameters.java b/signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsVaultParameters.java similarity index 96% rename from signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerParameters.java rename to signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsVaultParameters.java index 99573615a..cc1e5a281 100644 --- a/signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerParameters.java +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/config/AwsVaultParameters.java @@ -19,7 +19,7 @@ import java.util.Collections; import java.util.Optional; -public interface AwsSecretsManagerParameters { +public interface AwsVaultParameters { boolean isEnabled(); AwsAuthenticationMode getAuthenticationMode(); diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/config/metadata/AwsKeySigningMetadata.java b/signing/src/main/java/tech/pegasys/web3signer/signing/config/metadata/AwsKeySigningMetadata.java index 2f81ab2fa..18781c12b 100644 --- a/signing/src/main/java/tech/pegasys/web3signer/signing/config/metadata/AwsKeySigningMetadata.java +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/config/metadata/AwsKeySigningMetadata.java @@ -15,7 +15,7 @@ import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; import tech.pegasys.web3signer.signing.ArtifactSigner; import tech.pegasys.web3signer.signing.KeyType; -import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters; +import tech.pegasys.web3signer.signing.config.AwsVaultParameters; import java.net.URI; import java.util.Optional; @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @JsonDeserialize(using = AwsKeySigningMetadataDeserializer.class) -public class AwsKeySigningMetadata extends SigningMetadata implements AwsSecretsManagerParameters { +public class AwsKeySigningMetadata extends SigningMetadata implements AwsVaultParameters { public static final String TYPE = "aws-secret"; private final AwsAuthenticationMode authenticationMode; private final String region; diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClient.java b/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClient.java index 8780616ff..924dfcb19 100644 --- a/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClient.java +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClient.java @@ -14,25 +14,42 @@ import static com.google.common.base.Preconditions.checkArgument; +import tech.pegasys.web3signer.keystorage.common.MappedResults; + import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; import java.security.Provider; import java.security.interfaces.ECPublicKey; import java.security.spec.InvalidKeySpecException; import java.security.spec.X509EncodedKeySpec; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.bouncycastle.jce.provider.BouncyCastleProvider; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.kms.KmsClient; import software.amazon.awssdk.services.kms.model.CreateKeyRequest; +import software.amazon.awssdk.services.kms.model.DescribeKeyRequest; +import software.amazon.awssdk.services.kms.model.DisableKeyRequest; import software.amazon.awssdk.services.kms.model.GetPublicKeyRequest; import software.amazon.awssdk.services.kms.model.GetPublicKeyResponse; +import software.amazon.awssdk.services.kms.model.KeyListEntry; +import software.amazon.awssdk.services.kms.model.KeyMetadata; import software.amazon.awssdk.services.kms.model.KeySpec; +import software.amazon.awssdk.services.kms.model.KeyState; +import software.amazon.awssdk.services.kms.model.ListResourceTagsRequest; import software.amazon.awssdk.services.kms.model.MessageType; import software.amazon.awssdk.services.kms.model.ScheduleKeyDeletionRequest; import software.amazon.awssdk.services.kms.model.SignRequest; import software.amazon.awssdk.services.kms.model.SigningAlgorithmSpec; +import software.amazon.awssdk.services.kms.model.Tag; /** * Wraps KmsClient to allow the same instance to be cached and re-used. It exposes the methods that @@ -40,6 +57,7 @@ * not implemented close method. */ public class AwsKmsClient { + private static final Logger LOG = LogManager.getLogger(); private static final Provider BC_PROVIDER = new BouncyCastleProvider(); private final KmsClient kmsClient; @@ -82,6 +100,87 @@ public byte[] sign(final String kmsKeyId, final byte[] data) { return kmsClient.sign(signRequest).signature().asByteArray(); } + public MappedResults mapKeyList( + final Function mapper, + final Collection tagKeys, + final Collection tagValues) { + final Set result = ConcurrentHashMap.newKeySet(); + final AtomicInteger errorCount = new AtomicInteger(0); + + try { + kmsClient + .listKeysPaginator() + .iterator() + .forEachRemaining( + listKeysResponse -> + listKeysResponse.keys().parallelStream() + .filter( + keyListEntry -> filterKeys(keyListEntry, tagKeys, tagValues, errorCount)) + .forEach( + keyListEntry -> { + try { + final R value = mapper.apply(keyListEntry); + result.add(value); + } catch (final Exception e) { + LOG.warn( + "Failed to map keyListEntry '{}' to requested object type.", + keyListEntry.keyId(), + e); + errorCount.incrementAndGet(); + } + })); + } catch (Exception e) { + LOG.error("Unexpected error during Aws mapKeyList", e); + errorCount.incrementAndGet(); + } + + return MappedResults.newInstance(result, errorCount.intValue()); + } + + private boolean filterKeys( + final KeyListEntry keyListEntry, + final Collection tagKeys, + final Collection tagValues, + final AtomicInteger errorCount) { + try { + return isEnabledSecp256k1Key(keyListEntry) + && keyMatchesTags(keyListEntry.keyId(), tagKeys, tagValues); + } catch (Exception e) { + LOG.error("Unexpected error during Aws mapKeyList", e); + errorCount.incrementAndGet(); + return false; + } + } + + private boolean isEnabledSecp256k1Key(final KeyListEntry keyListEntry) { + final KeyMetadata keyMetadata = + kmsClient + .describeKey(DescribeKeyRequest.builder().keyId(keyListEntry.keyId()).build()) + .keyMetadata(); + final boolean isEnabled = keyMetadata.keyState() == KeyState.ENABLED; + final boolean isSecp256k1 = keyMetadata.keySpec() == KeySpec.ECC_SECG_P256_K1; + return isEnabled && isSecp256k1; + } + + private boolean keyMatchesTags( + final String keyId, final Collection tagKeys, final Collection tagValues) { + if (tagKeys.isEmpty() && tagValues.isEmpty()) + return true; // we don't want to filter if user-supplied tags map is empty + + final List kmsTags = + kmsClient.listResourceTags(ListResourceTagsRequest.builder().keyId(keyId).build()).tags(); + return matchesTag(kmsTags, tagKeys, Tag::tagKey) + && matchesTag(kmsTags, tagValues, Tag::tagValue); + } + + private boolean matchesTag( + final List kmsTags, + final Collection tags, + final Function tagProperty) { + return tags.isEmpty() + || kmsTags.stream().allMatch(tag -> tags.contains(tagProperty.apply(tag))); + } + @VisibleForTesting public String createKey(CreateKeyRequest createKeyRequest) { return kmsClient.createKey(createKeyRequest).keyMetadata().keyId(); @@ -91,4 +190,9 @@ public String createKey(CreateKeyRequest createKeyRequest) { public void scheduleKeyDeletion(ScheduleKeyDeletionRequest deletionRequest) { kmsClient.scheduleKeyDeletion(deletionRequest); } + + @VisibleForTesting + public void disableKey(final DisableKeyRequest disableKeyRequest) { + kmsClient.disableKey(disableKeyRequest); + } } diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClientKey.java b/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClientKey.java index 4794fac9c..5ff082031 100644 --- a/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClientKey.java +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClientKey.java @@ -12,32 +12,39 @@ */ package tech.pegasys.web3signer.signing.secp256k1.aws; +import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; +import tech.pegasys.web3signer.common.config.AwsCredentials; + import java.net.URI; import java.util.Objects; import java.util.Optional; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; - /** This class acts as a key to identify Aws KmsClient from the cache. */ final class AwsKmsClientKey { - private final AwsCredentialsProvider awsCredentialsProvider; - private final AwsCredentials awsCredentials; + + private final Optional awsCredentials; + private final AwsAuthenticationMode awsAuthenticationMode; private final String region; private final Optional endpointOverride; AwsKmsClientKey( - final AwsCredentialsProvider awsCredentialsProvider, + final AwsAuthenticationMode awsAuthenticationMode, + final Optional awsCredentials, final String region, final Optional endpointOverride) { - this.awsCredentialsProvider = awsCredentialsProvider; - this.awsCredentials = awsCredentialsProvider.resolveCredentials(); + this.awsAuthenticationMode = awsAuthenticationMode; + this.awsCredentials = awsCredentials; + this.region = region; this.endpointOverride = endpointOverride; } - public AwsCredentialsProvider getAwsCredentialsProvider() { - return awsCredentialsProvider; + public Optional getAwsCredentials() { + return awsCredentials; + } + + public AwsAuthenticationMode getAwsAuthenticationMode() { + return awsAuthenticationMode; } public String getRegion() { @@ -54,12 +61,13 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; AwsKmsClientKey that = (AwsKmsClientKey) o; return Objects.equals(awsCredentials, that.awsCredentials) + && awsAuthenticationMode == that.awsAuthenticationMode && Objects.equals(region, that.region) && Objects.equals(endpointOverride, that.endpointOverride); } @Override public int hashCode() { - return Objects.hash(awsCredentials, region, endpointOverride); + return Objects.hash(awsCredentials, awsAuthenticationMode, region, endpointOverride); } } diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsSignerFactory.java b/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsSignerFactory.java index 5f569904d..78b2fef67 100644 --- a/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsSignerFactory.java +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsSignerFactory.java @@ -14,45 +14,37 @@ import static com.google.common.base.Preconditions.checkArgument; -import tech.pegasys.web3signer.signing.config.AwsCredentialsProviderFactory; import tech.pegasys.web3signer.signing.config.metadata.AwsKmsMetadata; import tech.pegasys.web3signer.signing.secp256k1.Signer; import java.security.interfaces.ECPublicKey; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; - /** A Signer factory that create an instance of `Signer` type backed by AWS KMS. */ public class AwsKmsSignerFactory { - private final CachedAwsKmsClientFactory factory; - + private final CachedAwsKmsClientFactory cachedAwsKmsClientFactory; private final boolean applySha3Hash; /** * Construct AwsKmsSignerFactory * - * @param kmsClientCacheSize The cache size of AWS kms clients. This size should be set based on - * the number of credentials/region used. If same set of credentials/region used to access - * kms, set to 1. + * @param cachedAwsKmsClientFactory The cached AWS KMS client factory used to provide cached AWS + * KMS clients. * @param applySha3Hash Set to true for eth1 signing. Set false for filecoin signing. */ - public AwsKmsSignerFactory(final long kmsClientCacheSize, final boolean applySha3Hash) { - checkArgument(kmsClientCacheSize > 0, "Kms client cache Size must be positive."); - factory = new CachedAwsKmsClientFactory(kmsClientCacheSize); + public AwsKmsSignerFactory( + final CachedAwsKmsClientFactory cachedAwsKmsClientFactory, final boolean applySha3Hash) { + this.cachedAwsKmsClientFactory = cachedAwsKmsClientFactory; this.applySha3Hash = applySha3Hash; } public Signer createSigner(final AwsKmsMetadata awsKmsMetadata) { checkArgument(awsKmsMetadata != null, "awsKmsMetadata must not be null"); - final AwsCredentialsProvider awsCredentialsProvider = - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - awsKmsMetadata.getAuthenticationMode(), awsKmsMetadata.getAwsCredentials()); - final AwsKmsClient kmsClient = - factory.createKmsClient( - awsCredentialsProvider, + cachedAwsKmsClientFactory.createKmsClient( + awsKmsMetadata.getAuthenticationMode(), + awsKmsMetadata.getAwsCredentials(), awsKmsMetadata.getRegion(), awsKmsMetadata.getEndpointOverride()); diff --git a/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/CachedAwsKmsClientFactory.java b/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/CachedAwsKmsClientFactory.java index 21d74a430..722ea0db7 100644 --- a/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/CachedAwsKmsClientFactory.java +++ b/signing/src/main/java/tech/pegasys/web3signer/signing/secp256k1/aws/CachedAwsKmsClientFactory.java @@ -14,6 +14,10 @@ import static com.google.common.base.Preconditions.checkArgument; +import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; +import tech.pegasys.web3signer.common.config.AwsCredentials; +import tech.pegasys.web3signer.signing.config.AwsCredentialsProviderFactory; + import java.net.URI; import java.util.Optional; @@ -22,6 +26,7 @@ import com.google.common.cache.LoadingCache; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.kms.KmsClient; import software.amazon.awssdk.services.kms.KmsClientBuilder; @@ -45,11 +50,17 @@ public CachedAwsKmsClientFactory(final long cacheSize) { new CacheLoader<>() { @Override public AwsKmsClient load(final AwsKmsClientKey key) { + final AwsCredentialsProvider awsCredentialsProvider = + AwsCredentialsProviderFactory.createAwsCredentialsProvider( + key.getAwsAuthenticationMode(), key.getAwsCredentials()); + final KmsClientBuilder kmsClientBuilder = KmsClient.builder(); key.getEndpointOverride().ifPresent(kmsClientBuilder::endpointOverride); - kmsClientBuilder - .credentialsProvider(key.getAwsCredentialsProvider()) - .region(Region.of(key.getRegion())); + final Region region = + key.getAwsAuthenticationMode() == AwsAuthenticationMode.SPECIFIED + ? Region.of(key.getRegion()) + : DefaultAwsRegionProviderChain.builder().build().getRegion(); + kmsClientBuilder.credentialsProvider(awsCredentialsProvider).region(region); return new AwsKmsClient(kmsClientBuilder.build()); } @@ -57,10 +68,11 @@ public AwsKmsClient load(final AwsKmsClientKey key) { } public AwsKmsClient createKmsClient( - final AwsCredentialsProvider awsCredentialsProvider, + final AwsAuthenticationMode awsAuthenticationMode, + final Optional awsCredentials, final String region, final Optional endpointOverride) { return cache.getUnchecked( - new AwsKmsClientKey(awsCredentialsProvider, region, endpointOverride)); + new AwsKmsClientKey(awsAuthenticationMode, awsCredentials, region, endpointOverride)); } } diff --git a/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClientTest.java b/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClientTest.java new file mode 100644 index 000000000..254403440 --- /dev/null +++ b/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsClientTest.java @@ -0,0 +1,222 @@ +/* + * Copyright 2023 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer.signing.secp256k1.aws; + +import tech.pegasys.web3signer.AwsKmsUtil; +import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; +import tech.pegasys.web3signer.common.config.AwsCredentials; +import tech.pegasys.web3signer.common.config.AwsCredentials.AwsCredentialsBuilder; +import tech.pegasys.web3signer.keystorage.common.MappedResults; +import tech.pegasys.web3signer.signing.config.AwsCredentialsProviderFactory; + +import java.net.URI; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.kms.KmsClient; +import software.amazon.awssdk.services.kms.KmsClientBuilder; +import software.amazon.awssdk.services.kms.model.KeyListEntry; +import software.amazon.awssdk.services.kms.model.KeySpec; + +@EnabledIfEnvironmentVariable( + named = "RW_AWS_ACCESS_KEY_ID", + matches = ".*", + disabledReason = "RW_AWS_ACCESS_KEY_ID env variable is required") +@EnabledIfEnvironmentVariable( + named = "RW_AWS_SECRET_ACCESS_KEY", + matches = ".*", + disabledReason = "RW_AWS_SECRET_ACCESS_KEY env variable is required") +@EnabledIfEnvironmentVariable( + named = "AWS_ACCESS_KEY_ID", + matches = ".*", + disabledReason = "AWS_ACCESS_KEY_ID env variable is required") +@EnabledIfEnvironmentVariable( + named = "AWS_SECRET_ACCESS_KEY", + matches = ".*", + disabledReason = "AWS_SECRET_ACCESS_KEY env variable is required") +@EnabledIfEnvironmentVariable( + named = "AWS_REGION", + matches = ".*", + disabledReason = "AWS_REGION env variable is required") +public class AwsKmsClientTest { + private static final String AWS_ACCESS_KEY_ID = System.getenv("AWS_ACCESS_KEY_ID"); + private static final String AWS_SECRET_ACCESS_KEY = System.getenv("AWS_SECRET_ACCESS_KEY"); + private static final String AWS_REGION = System.getenv("AWS_REGION"); + private static final String RW_AWS_ACCESS_KEY_ID = System.getenv("RW_AWS_ACCESS_KEY_ID"); + private static final String RW_AWS_SECRET_ACCESS_KEY = System.getenv("RW_AWS_SECRET_ACCESS_KEY"); + private static final Optional AWS_SESSION_TOKEN = + Optional.ofNullable(System.getenv("AWS_SESSION_TOKEN")); + private static final Optional ENDPOINT_OVERRIDE = + Optional.ofNullable(System.getenv("AWS_ENDPOINT_OVERRIDE")).map(URI::create); + + private static String testKeyId; + private static String testWithTagKeyId; + private static String testWithDisabledKeyId; + private static String testWithNistSecpKeyId; + private static AwsKmsUtil awsKmsUtil; + private static AwsKmsClient awsKmsClient; + + @BeforeAll + static void init() { + awsKmsUtil = + new AwsKmsUtil( + AWS_REGION, + RW_AWS_ACCESS_KEY_ID, + RW_AWS_SECRET_ACCESS_KEY, + AWS_SESSION_TOKEN, + ENDPOINT_OVERRIDE); + testKeyId = awsKmsUtil.createKey(Collections.emptyMap()); + testWithTagKeyId = awsKmsUtil.createKey(Map.of("name", "tagged")); + testWithDisabledKeyId = awsKmsUtil.createKey(Map.of("name", "disabled")); + awsKmsUtil.disableKey(testWithDisabledKeyId); + testWithNistSecpKeyId = awsKmsUtil.createKey(Map.of("name", "nist"), KeySpec.ECC_NIST_P256); + + final AwsCredentialsBuilder awsCredentialsBuilder = AwsCredentials.builder(); + awsCredentialsBuilder + .withAccessKeyId(AWS_ACCESS_KEY_ID) + .withSecretAccessKey(AWS_SECRET_ACCESS_KEY); + AWS_SESSION_TOKEN.ifPresent(awsCredentialsBuilder::withSessionToken); + + final AwsCredentialsProvider awsCredentialsProvider = + AwsCredentialsProviderFactory.createAwsCredentialsProvider( + AwsAuthenticationMode.SPECIFIED, Optional.of(awsCredentialsBuilder.build())); + + final KmsClientBuilder kmsClientBuilder = KmsClient.builder(); + kmsClientBuilder.credentialsProvider(awsCredentialsProvider).region(Region.of(AWS_REGION)); + ENDPOINT_OVERRIDE.ifPresent(kmsClientBuilder::endpointOverride); + + awsKmsClient = new AwsKmsClient(kmsClientBuilder.build()); + } + + @AfterAll + static void cleanup() { + if (awsKmsUtil == null) { + return; + } + // delete key + awsKmsUtil.deleteKey(testKeyId); + awsKmsUtil.deleteKey(testWithTagKeyId); + } + + @Test + void keyListCanBeMappedUsingCustomMappingFunction() { + final MappedResults result = + awsKmsClient.mapKeyList( + KeyListEntry::keyId, Collections.emptyList(), Collections.emptyList()); + + final Optional testKeyEntry = + result.getValues().stream().filter(e -> e.equals(testKeyId)).findAny(); + Assertions.assertThat(testKeyEntry).isPresent(); + Assertions.assertThat(testKeyEntry.get()).isEqualTo(testKeyId); + Assertions.assertThat(result.getErrorCount()).isZero(); + } + + @Test + void mapKeyListThrowsAwayObjectsWhichFailMapper() { + final MappedResults result = + awsKmsClient.mapKeyList( + kl -> { + if (kl.keyId().equals(testKeyId)) { + throw new IllegalStateException("Failed mapper"); + } else { + return kl.keyId(); + } + }, + Collections.emptyList(), + Collections.emptyList()); + + final Optional testKeyEntry = + result.getValues().stream().filter(e -> e.equals(testKeyId)).findAny(); + Assertions.assertThat(testKeyEntry).isEmpty(); + Assertions.assertThat(result.getErrorCount()).isOne(); + } + + @Test + void mapKeyListUsingTagsKey() { + final MappedResults result = + awsKmsClient.mapKeyList(KeyListEntry::keyId, List.of("name"), Collections.emptyList()); + + final Optional testKeyEntry = + result.getValues().stream().filter(e -> e.equals(testWithTagKeyId)).findAny(); + Assertions.assertThat(testKeyEntry).isPresent(); + Assertions.assertThat(testKeyEntry.get()).isEqualTo(testWithTagKeyId); + Assertions.assertThat(result.getErrorCount()).isZero(); + } + + @Test + void mapKeyListUsingTagsValue() { + final MappedResults result = + awsKmsClient.mapKeyList(KeyListEntry::keyId, Collections.emptyList(), List.of("tagged")); + + final Optional testKeyEntry = + result.getValues().stream().filter(e -> e.equals(testWithTagKeyId)).findAny(); + Assertions.assertThat(testKeyEntry).isPresent(); + Assertions.assertThat(testKeyEntry.get()).isEqualTo(testWithTagKeyId); + Assertions.assertThat(result.getErrorCount()).isZero(); + } + + @Test + void mapKeyListUsingTagsKeyAndValue() { + final MappedResults result = + awsKmsClient.mapKeyList(KeyListEntry::keyId, List.of("name"), List.of("tagged")); + + final Optional testKeyEntry = + result.getValues().stream().filter(e -> e.equals(testWithTagKeyId)).findAny(); + Assertions.assertThat(testKeyEntry).isPresent(); + Assertions.assertThat(testKeyEntry.get()).isEqualTo(testWithTagKeyId); + Assertions.assertThat(result.getErrorCount()).isZero(); + } + + @Test + void mapKeyListWhenTagDoesNotExist() { + final MappedResults result = + awsKmsClient.mapKeyList( + KeyListEntry::keyId, List.of("unknownKey"), List.of("unknownValue")); + + final Optional testKeyEntry = + result.getValues().stream().filter(e -> e.equals(testWithTagKeyId)).findAny(); + Assertions.assertThat(testKeyEntry).isEmpty(); + Assertions.assertThat(result.getErrorCount()).isZero(); + } + + @Test + void mapKeyListIgnoresDisabledKeys() { + final MappedResults result = + awsKmsClient.mapKeyList(KeyListEntry::keyId, List.of("name"), List.of("disabled")); + + final Optional disableTestKeyEntry = + result.getValues().stream().filter(e -> e.equals(testWithDisabledKeyId)).findAny(); + Assertions.assertThat(disableTestKeyEntry).isEmpty(); + Assertions.assertThat(result.getErrorCount()).isZero(); + } + + @Test + void mapKeyListIgnoresNonSecpKeys() { + final MappedResults result = + awsKmsClient.mapKeyList(KeyListEntry::keyId, List.of("name"), List.of("nist")); + + final Optional disableTestKeyEntry = + result.getValues().stream().filter(e -> e.equals(testWithNistSecpKeyId)).findAny(); + Assertions.assertThat(disableTestKeyEntry).isEmpty(); + Assertions.assertThat(result.getErrorCount()).isZero(); + } +} diff --git a/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsSignerTest.java b/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsSignerTest.java index 2d0b8d1cd..527c3e854 100644 --- a/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsSignerTest.java +++ b/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/AwsKmsSignerTest.java @@ -15,9 +15,9 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; +import tech.pegasys.web3signer.AwsKmsUtil; import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; import tech.pegasys.web3signer.common.config.AwsCredentials; -import tech.pegasys.web3signer.signing.config.AwsCredentialsProviderFactory; import tech.pegasys.web3signer.signing.config.metadata.AwsKmsMetadata; import tech.pegasys.web3signer.signing.secp256k1.EthPublicKeyUtils; import tech.pegasys.web3signer.signing.secp256k1.Signature; @@ -26,6 +26,7 @@ import java.math.BigInteger; import java.net.URI; import java.security.SignatureException; +import java.util.Collections; import java.util.Optional; import org.junit.jupiter.api.AfterAll; @@ -36,11 +37,6 @@ import org.web3j.crypto.Hash; import org.web3j.crypto.Sign; import org.web3j.utils.Numeric; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.services.kms.model.CreateKeyRequest; -import software.amazon.awssdk.services.kms.model.KeySpec; -import software.amazon.awssdk.services.kms.model.KeyUsageType; -import software.amazon.awssdk.services.kms.model.ScheduleKeyDeletionRequest; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @EnabledIfEnvironmentVariable( @@ -73,13 +69,6 @@ public class AwsKmsSignerTest { private static final Optional ENDPOINT_OVERRIDE = Optional.ofNullable(System.getenv("AWS_ENDPOINT_OVERRIDE")).map(URI::create); - private static final AwsCredentials AWS_RW_CREDENTIALS = - AwsCredentials.builder() - .withAccessKeyId(RW_AWS_ACCESS_KEY_ID) - .withSecretAccessKey(RW_AWS_SECRET_ACCESS_KEY) - .withSessionToken(AWS_SESSION_TOKEN) - .build(); - private static final AwsCredentials AWS_CREDENTIALS = AwsCredentials.builder() .withAccessKeyId(AWS_ACCESS_KEY_ID) @@ -87,39 +76,28 @@ public class AwsKmsSignerTest { .withSessionToken(AWS_SESSION_TOKEN) .build(); - private static final CachedAwsKmsClientFactory KMS_CLIENT_FACTORY = - new CachedAwsKmsClientFactory(1); - private static AwsKmsClient awsKMSClient; private static String testKeyId; + private static AwsKmsUtil awsKmsUtil; @BeforeAll static void init() { - AwsCredentialsProvider awsCredentialsProvider = - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - AwsAuthenticationMode.SPECIFIED, Optional.of(AWS_RW_CREDENTIALS)); - awsKMSClient = - KMS_CLIENT_FACTORY.createKmsClient(awsCredentialsProvider, AWS_REGION, ENDPOINT_OVERRIDE); - - // create a test key - final CreateKeyRequest web3SignerTestingKey = - CreateKeyRequest.builder() - .keySpec(KeySpec.ECC_SECG_P256_K1) - .description("Web3Signer Testing Key") - .keyUsage(KeyUsageType.SIGN_VERIFY) - .build(); - testKeyId = awsKMSClient.createKey(web3SignerTestingKey); + awsKmsUtil = + new AwsKmsUtil( + AWS_REGION, + RW_AWS_ACCESS_KEY_ID, + RW_AWS_SECRET_ACCESS_KEY, + Optional.ofNullable(AWS_SESSION_TOKEN), + ENDPOINT_OVERRIDE); + testKeyId = awsKmsUtil.createKey(Collections.emptyMap()); assertThat(testKeyId).isNotEmpty(); } @AfterAll static void cleanup() { - if (awsKMSClient == null) { + if (awsKmsUtil == null) { return; } - // delete key - ScheduleKeyDeletionRequest deletionRequest = - ScheduleKeyDeletionRequest.builder().keyId(testKeyId).pendingWindowInDays(7).build(); - awsKMSClient.scheduleKeyDeletion(deletionRequest); + awsKmsUtil.deleteKey(testKeyId); } @Test @@ -133,8 +111,11 @@ void awsSignatureCanBeVerified() throws SignatureException { ENDPOINT_OVERRIDE); final long kmsClientCacheSize = 1; final boolean applySha3Hash = true; + final CachedAwsKmsClientFactory cachedAwsKmsClientFactory = + new CachedAwsKmsClientFactory(kmsClientCacheSize); final Signer signer = - new AwsKmsSignerFactory(kmsClientCacheSize, applySha3Hash).createSigner(awsKmsMetadata); + new AwsKmsSignerFactory(cachedAwsKmsClientFactory, applySha3Hash) + .createSigner(awsKmsMetadata); final BigInteger publicKey = Numeric.toBigInt(EthPublicKeyUtils.toByteArray(signer.getPublicKey())); diff --git a/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/CachedAwsKmsClientFactoryTest.java b/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/CachedAwsKmsClientFactoryTest.java index dab8c34bd..d3c232736 100644 --- a/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/CachedAwsKmsClientFactoryTest.java +++ b/signing/src/test/java/tech/pegasys/web3signer/signing/secp256k1/aws/CachedAwsKmsClientFactoryTest.java @@ -16,7 +16,6 @@ import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; import tech.pegasys.web3signer.common.config.AwsCredentials; -import tech.pegasys.web3signer.signing.config.AwsCredentialsProviderFactory; import java.util.Optional; @@ -35,49 +34,45 @@ void init() { void cachedInstanceOfKmsClientIsReturnedForSpecifiedCredentials() { final AwsKmsClient kmsClient_1 = cachedAwsKmsClientFactory.createKmsClient( - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - AwsAuthenticationMode.SPECIFIED, - Optional.of( - AwsCredentials.builder() - .withAccessKeyId("test") - .withSecretAccessKey("test") - .build())), + AwsAuthenticationMode.SPECIFIED, + Optional.of( + AwsCredentials.builder() + .withAccessKeyId("test") + .withSecretAccessKey("test") + .build()), "us-east-2", Optional.empty()); final AwsKmsClient kmsClient_2 = cachedAwsKmsClientFactory.createKmsClient( - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - AwsAuthenticationMode.SPECIFIED, - Optional.of( - AwsCredentials.builder() - .withAccessKeyId("test3") - .withSecretAccessKey("test3") - .build())), + AwsAuthenticationMode.SPECIFIED, + Optional.of( + AwsCredentials.builder() + .withAccessKeyId("test3") + .withSecretAccessKey("test3") + .build()), "us-east-2", Optional.empty()); final AwsKmsClient kmsClient_3 = cachedAwsKmsClientFactory.createKmsClient( - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - AwsAuthenticationMode.SPECIFIED, - Optional.of( - AwsCredentials.builder() - .withAccessKeyId("test") - .withSecretAccessKey("test") - .build())), + AwsAuthenticationMode.SPECIFIED, + Optional.of( + AwsCredentials.builder() + .withAccessKeyId("test") + .withSecretAccessKey("test") + .build()), "us-east-2", Optional.empty()); final AwsKmsClient kmsClient_4 = cachedAwsKmsClientFactory.createKmsClient( - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - AwsAuthenticationMode.SPECIFIED, - Optional.of( - AwsCredentials.builder() - .withAccessKeyId("test3") - .withSecretAccessKey("test3") - .build())), + AwsAuthenticationMode.SPECIFIED, + Optional.of( + AwsCredentials.builder() + .withAccessKeyId("test3") + .withSecretAccessKey("test3") + .build()), "us-east-2", Optional.empty()); @@ -91,27 +86,25 @@ void cachedInstanceOfKmsClientIsReturnedForSpecifiedCredentials() { void cachedInstanceOfKmsClientIsReturnedForSpecifiedCredentialsWithSessionToken() { final AwsKmsClient kmsClient_1 = cachedAwsKmsClientFactory.createKmsClient( - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - AwsAuthenticationMode.SPECIFIED, - Optional.of( - AwsCredentials.builder() - .withAccessKeyId("test") - .withSecretAccessKey("test") - .withSessionToken("test") - .build())), + AwsAuthenticationMode.SPECIFIED, + Optional.of( + AwsCredentials.builder() + .withAccessKeyId("test") + .withSecretAccessKey("test") + .withSessionToken("test") + .build()), "us-east-2", Optional.empty()); final AwsKmsClient kmsClient_2 = cachedAwsKmsClientFactory.createKmsClient( - AwsCredentialsProviderFactory.createAwsCredentialsProvider( - AwsAuthenticationMode.SPECIFIED, - Optional.of( - AwsCredentials.builder() - .withAccessKeyId("test") - .withSecretAccessKey("test") - .withSessionToken("test") - .build())), + AwsAuthenticationMode.SPECIFIED, + Optional.of( + AwsCredentials.builder() + .withAccessKeyId("test") + .withSecretAccessKey("test") + .withSessionToken("test") + .build()), "us-east-2", Optional.empty()); diff --git a/signing/src/testFixtures/java/tech/pegasys/web3signer/AwsKmsUtil.java b/signing/src/testFixtures/java/tech/pegasys/web3signer/AwsKmsUtil.java new file mode 100644 index 000000000..29214b523 --- /dev/null +++ b/signing/src/testFixtures/java/tech/pegasys/web3signer/AwsKmsUtil.java @@ -0,0 +1,91 @@ +/* + * Copyright 2023 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.web3signer; + +import tech.pegasys.web3signer.common.config.AwsAuthenticationMode; +import tech.pegasys.web3signer.common.config.AwsCredentials; +import tech.pegasys.web3signer.common.config.AwsCredentials.AwsCredentialsBuilder; +import tech.pegasys.web3signer.signing.secp256k1.aws.AwsKmsClient; +import tech.pegasys.web3signer.signing.secp256k1.aws.CachedAwsKmsClientFactory; + +import java.net.URI; +import java.security.interfaces.ECPublicKey; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import software.amazon.awssdk.services.kms.model.CreateKeyRequest; +import software.amazon.awssdk.services.kms.model.DisableKeyRequest; +import software.amazon.awssdk.services.kms.model.KeySpec; +import software.amazon.awssdk.services.kms.model.KeyUsageType; +import software.amazon.awssdk.services.kms.model.ScheduleKeyDeletionRequest; +import software.amazon.awssdk.services.kms.model.Tag; + +public class AwsKmsUtil { + + private final AwsKmsClient awsKMSClient; + + public AwsKmsUtil( + final String region, + final String accessKeyId, + final String secretAccessKey, + final Optional sessionToken, + Optional awsEndpointOverride) { + final AwsCredentialsBuilder awsCredentialsBuilder = AwsCredentials.builder(); + awsCredentialsBuilder.withAccessKeyId(accessKeyId).withSecretAccessKey(secretAccessKey); + sessionToken.ifPresent(awsCredentialsBuilder::withSessionToken); + + final CachedAwsKmsClientFactory cachedAwsKmsClientFactory = new CachedAwsKmsClientFactory(1); + awsKMSClient = + cachedAwsKmsClientFactory.createKmsClient( + AwsAuthenticationMode.SPECIFIED, + Optional.of(awsCredentialsBuilder.build()), + region, + awsEndpointOverride); + } + + public String createKey(final Map tags, final KeySpec keySpec) { + final CreateKeyRequest.Builder keyRequestBuilder = + CreateKeyRequest.builder() + .keySpec(keySpec) + .description("Web3Signer Testing Key") + .keyUsage(KeyUsageType.SIGN_VERIFY); + final List awsTags = + tags.entrySet().stream() + .map(e -> Tag.builder().tagKey(e.getKey()).tagValue(e.getValue()).build()) + .toList(); + if (!awsTags.isEmpty()) { + keyRequestBuilder.tags(awsTags); + } + return awsKMSClient.createKey(keyRequestBuilder.build()); + } + + public String createKey(final Map tags) { + return createKey(tags, KeySpec.ECC_SECG_P256_K1); + } + + public void deleteKey(final String keyId) { + final ScheduleKeyDeletionRequest deletionRequest = + ScheduleKeyDeletionRequest.builder().keyId(keyId).pendingWindowInDays(7).build(); + awsKMSClient.scheduleKeyDeletion(deletionRequest); + } + + public void disableKey(final String keyId) { + final DisableKeyRequest disableKeyRequest = DisableKeyRequest.builder().keyId(keyId).build(); + awsKMSClient.disableKey(disableKeyRequest); + } + + public ECPublicKey publicKey(final String keyId) { + return awsKMSClient.getECPublicKey(keyId); + } +} diff --git a/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerParametersBuilder.java b/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/AwsVaultParametersBuilder.java similarity index 75% rename from signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerParametersBuilder.java rename to signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/AwsVaultParametersBuilder.java index 897da217f..a74b576d9 100644 --- a/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/AwsSecretsManagerParametersBuilder.java +++ b/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/AwsVaultParametersBuilder.java @@ -19,7 +19,7 @@ import java.util.Collections; import java.util.Optional; -public final class AwsSecretsManagerParametersBuilder { +public final class AwsVaultParametersBuilder { private AwsAuthenticationMode authenticationMode = AwsAuthenticationMode.SPECIFIED; private String accessKeyId; private String secretAccessKey; @@ -30,64 +30,66 @@ public final class AwsSecretsManagerParametersBuilder { private long cacheMaximumSize = 1; private Optional endpointURI = Optional.empty(); + private boolean enabled; - private AwsSecretsManagerParametersBuilder() {} + private AwsVaultParametersBuilder() {} - public static AwsSecretsManagerParametersBuilder anAwsSecretsManagerParameters() { - return new AwsSecretsManagerParametersBuilder(); + public static AwsVaultParametersBuilder anAwsParameters() { + return new AwsVaultParametersBuilder(); } - public AwsSecretsManagerParametersBuilder withAuthenticationMode( + public AwsVaultParametersBuilder withAuthenticationMode( final AwsAuthenticationMode authenticationMode) { this.authenticationMode = authenticationMode; return this; } - public AwsSecretsManagerParametersBuilder withAccessKeyId(final String accessKeyId) { + public AwsVaultParametersBuilder withAccessKeyId(final String accessKeyId) { this.accessKeyId = accessKeyId; return this; } - public AwsSecretsManagerParametersBuilder withSecretAccessKey(final String secretAccessKey) { + public AwsVaultParametersBuilder withSecretAccessKey(final String secretAccessKey) { this.secretAccessKey = secretAccessKey; return this; } - public AwsSecretsManagerParametersBuilder withRegion(final String region) { + public AwsVaultParametersBuilder withRegion(final String region) { this.region = region; return this; } - public AwsSecretsManagerParametersBuilder withPrefixesFilter( - final Collection prefixesFilter) { + public AwsVaultParametersBuilder withPrefixesFilter(final Collection prefixesFilter) { this.prefixesFilter = prefixesFilter; return this; } - public AwsSecretsManagerParametersBuilder withTagNamesFilter( - final Collection tagNameFilters) { + public AwsVaultParametersBuilder withTagNamesFilter(final Collection tagNameFilters) { this.tagNamesFilter = tagNameFilters; return this; } - public AwsSecretsManagerParametersBuilder withTagValuesFilter( - final Collection tagValuesFilter) { + public AwsVaultParametersBuilder withTagValuesFilter(final Collection tagValuesFilter) { this.tagValuesFilter = tagValuesFilter; return this; } - public AwsSecretsManagerParametersBuilder withCacheMaximumSize(final long cacheMaximumSize) { + public AwsVaultParametersBuilder withCacheMaximumSize(final long cacheMaximumSize) { this.cacheMaximumSize = cacheMaximumSize; return this; } - public AwsSecretsManagerParametersBuilder withEndpointOverride( - final Optional endpointOverride) { + public AwsVaultParametersBuilder withEndpointOverride(final Optional endpointOverride) { this.endpointURI = endpointOverride; return this; } - public AwsSecretsManagerParameters build() { + public AwsVaultParametersBuilder withEnabled(final boolean enabled) { + this.enabled = enabled; + return this; + } + + public AwsVaultParameters build() { if (authenticationMode == AwsAuthenticationMode.SPECIFIED) { if (accessKeyId == null) { throw new IllegalArgumentException("accessKeyId is required"); @@ -102,7 +104,7 @@ public AwsSecretsManagerParameters build() { } } - return new TestAwsSecretsManagerParameters( + return new TestAwsVaultParameters( authenticationMode, accessKeyId, secretAccessKey, @@ -111,10 +113,11 @@ public AwsSecretsManagerParameters build() { tagNamesFilter, tagValuesFilter, cacheMaximumSize, - endpointURI); + endpointURI, + enabled); } - private static class TestAwsSecretsManagerParameters implements AwsSecretsManagerParameters { + private static class TestAwsVaultParameters implements AwsVaultParameters { private final AwsAuthenticationMode authenticationMode; private final String accessKeyId; private final String secretAccessKey; @@ -124,8 +127,9 @@ private static class TestAwsSecretsManagerParameters implements AwsSecretsManage private final Collection tagValuesFilter; private final long cacheMaximumSize; private final Optional endpointOverride; + private final boolean enabled; - TestAwsSecretsManagerParameters( + TestAwsVaultParameters( final AwsAuthenticationMode authenticationMode, final String accessKeyId, final String secretAccessKey, @@ -134,7 +138,8 @@ private static class TestAwsSecretsManagerParameters implements AwsSecretsManage final Collection tagNamesFilter, final Collection tagValuesFilter, final long cacheMaximumSize, - final Optional endpointOverride) { + final Optional endpointOverride, + final boolean enabled) { this.authenticationMode = authenticationMode; this.accessKeyId = accessKeyId; this.secretAccessKey = secretAccessKey; @@ -144,11 +149,12 @@ private static class TestAwsSecretsManagerParameters implements AwsSecretsManage this.tagValuesFilter = tagValuesFilter; this.cacheMaximumSize = cacheMaximumSize; this.endpointOverride = endpointOverride; + this.enabled = enabled; } @Override public boolean isEnabled() { - return true; + return enabled; } @Override diff --git a/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/DefaultAzureKeyVaultParameters.java b/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/DefaultAzureKeyVaultParameters.java index 349dad8c4..ffae9ffff 100644 --- a/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/DefaultAzureKeyVaultParameters.java +++ b/signing/src/testFixtures/java/tech/pegasys/web3signer/signing/config/DefaultAzureKeyVaultParameters.java @@ -18,7 +18,8 @@ public class DefaultAzureKeyVaultParameters implements AzureKeyVaultParameters { - private static long AZURE_DEFAULT_TIMEOUT = 60; + private static final long AZURE_DEFAULT_TIMEOUT = 60; + private static final boolean AZURE_DEFAULT_ENABLED = true; private final String keyVaultName; private final AzureAuthenticationMode authenticationMode; @@ -27,6 +28,7 @@ public class DefaultAzureKeyVaultParameters implements AzureKeyVaultParameters { private final String clientSecret; private final Map tags = new HashMap<>(); private final long timeout; + private final boolean enabled; public DefaultAzureKeyVaultParameters( final String keyVaultName, @@ -39,7 +41,8 @@ public DefaultAzureKeyVaultParameters( tenantId, clientSecret, Collections.emptyMap(), - AZURE_DEFAULT_TIMEOUT); + AZURE_DEFAULT_TIMEOUT, + AZURE_DEFAULT_ENABLED); } public DefaultAzureKeyVaultParameters( @@ -48,7 +51,14 @@ public DefaultAzureKeyVaultParameters( final String tenantId, final String clientSecret, final Map tags) { - this(keyVaultName, clientId, tenantId, clientSecret, tags, AZURE_DEFAULT_TIMEOUT); + this( + keyVaultName, + clientId, + tenantId, + clientSecret, + tags, + AZURE_DEFAULT_TIMEOUT, + AZURE_DEFAULT_ENABLED); } public DefaultAzureKeyVaultParameters( @@ -57,7 +67,8 @@ public DefaultAzureKeyVaultParameters( final String tenantId, final String clientSecret, final Map tags, - final long timeout) { + final long timeout, + final boolean enabled) { this.keyVaultName = keyVaultName; this.clientId = clientId; this.tenantId = tenantId; @@ -65,6 +76,7 @@ public DefaultAzureKeyVaultParameters( this.authenticationMode = AzureAuthenticationMode.CLIENT_SECRET; this.tags.putAll(tags); this.timeout = timeout; + this.enabled = enabled; } @Override @@ -89,7 +101,7 @@ public String getClientSecret() { @Override public boolean isAzureKeyVaultEnabled() { - return true; + return enabled; } @Override