Skip to content

Commit

Permalink
Aws bulk loading (#889)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jframe
jframe authored Sep 12, 2023
1 parent f98bae7 commit 7993730
Show file tree
Hide file tree
Showing 33 changed files with 1,366 additions and 328 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Changelog

## Next Release

### Features Added
- Aws bulk loading for secp256k1 keys in eth1 mode [#889](https://github.com/Consensys/web3signer/pull/889)

## 23.9.0

### Features Added
Expand Down
8 changes: 4 additions & 4 deletions acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import tech.pegasys.web3signer.core.config.client.ClientTlsOptions;
import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider;
import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition;
import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
import tech.pegasys.web3signer.signing.config.KeystoresParameters;

Expand All @@ -41,7 +41,7 @@ public class SignerConfiguration {
private final List<String> metricsCategories;
private final boolean metricsEnabled;
private final Optional<AzureKeyVaultParameters> azureKeyVaultParameters;
private final Optional<AwsSecretsManagerParameters> awsSecretsManagerParameters;
private final Optional<AwsVaultParameters> awsSecretsManagerParameters;
private final Optional<KeystoresParameters> keystoresParameters;
private final Optional<TlsOptions> serverTlsOptions;
private final Optional<TlsCertificateDefinition> overriddenCaTrustStore;
Expand Down Expand Up @@ -89,7 +89,7 @@ public SignerConfiguration(
final List<String> metricsCategories,
final boolean metricsEnabled,
final Optional<AzureKeyVaultParameters> azureKeyVaultParameters,
final Optional<AwsSecretsManagerParameters> awsSecretsManagerParameters,
final Optional<AwsVaultParameters> awsSecretsManagerParameters,
final Optional<KeystoresParameters> keystoresParameters,
final Optional<TlsOptions> serverTlsOptions,
final Optional<TlsCertificateDefinition> overriddenCaTrustStore,
Expand Down Expand Up @@ -221,7 +221,7 @@ public Optional<AzureKeyVaultParameters> getAzureKeyVaultParameters() {
return azureKeyVaultParameters;
}

public Optional<AwsSecretsManagerParameters> getAwsSecretsManagerParameters() {
public Optional<AwsVaultParameters> getAwsParameters() {
return awsSecretsManagerParameters;
}

Expand Down
11 changes: 5 additions & 6 deletions ...ce-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ChainIdProvider;
import tech.pegasys.web3signer.core.service.jsonrpc.handlers.signing.ConfigurationChainId;
import tech.pegasys.web3signer.dsl.tls.TlsCertificateDefinition;
import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
import tech.pegasys.web3signer.signing.config.KeystoresParameters;

Expand Down Expand Up @@ -50,7 +50,7 @@ public class SignerConfigurationBuilder {
private Path slashingProtectionDbPoolConfigurationFile = null;
private String mode;
private AzureKeyVaultParameters azureKeyVaultParameters;
private AwsSecretsManagerParameters awsSecretsManagerParameters;
private AwsVaultParameters awsVaultParameters;
private Map<String, String> web3SignerEnvironment;
private Duration startupTimeout =
Boolean.getBoolean("debugSubProcess") ? Duration.ofHours(1) : Duration.ofSeconds(30);
Expand Down Expand Up @@ -143,9 +143,8 @@ public SignerConfigurationBuilder withAzureKeyVaultParameters(
return this;
}

public SignerConfigurationBuilder withAwsSecretsManagerParameters(
final AwsSecretsManagerParameters awsSecretsManagerParameters) {
this.awsSecretsManagerParameters = awsSecretsManagerParameters;
public SignerConfigurationBuilder withAwsParameters(final AwsVaultParameters awsVaultParameters) {
this.awsVaultParameters = awsVaultParameters;
return this;
}

Expand Down Expand Up @@ -332,7 +331,7 @@ public SignerConfiguration build() {
metricsCategories,
metricsEnabled,
Optional.ofNullable(azureKeyVaultParameters),
Optional.ofNullable(awsSecretsManagerParameters),
Optional.ofNullable(awsVaultParameters),
Optional.ofNullable(keystoresParameters),
Optional.ofNullable(serverTlsOptions),
Optional.ofNullable(overriddenCaTrustStore),
Expand Down
119 changes: 99 additions & 20 deletions .../src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,13 @@
*/
package tech.pegasys.web3signer.dsl.signer.runner;

import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_ACCESS_KEY_ID_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_AUTH_MODE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_ENABLED_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_REGION_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_SECRET_ACCESS_KEY_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_TAG_NAMES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsKmsParameters.AWS_KMS_TAG_VALUES_FILTER_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION;
Expand All @@ -31,7 +38,7 @@
import tech.pegasys.web3signer.dsl.signer.SignerConfiguration;
import tech.pegasys.web3signer.dsl.signer.WatermarkRepairParameters;
import tech.pegasys.web3signer.dsl.utils.DatabaseUtil;
import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
import tech.pegasys.web3signer.signing.config.AwsVaultParameters;
import tech.pegasys.web3signer.signing.config.AzureKeyVaultParameters;
import tech.pegasys.web3signer.signing.config.KeystoresParameters;

Expand Down Expand Up @@ -143,8 +150,9 @@ public List<String> createCmdLineParams() {
}

signerConfig
.getAwsSecretsManagerParameters()
.ifPresent(awsParams -> yamlConfig.append(awsBulkLoadingOptions(awsParams)));
.getAwsParameters()
.ifPresent(
awsParams -> yamlConfig.append(awsSecretsManagerBulkLoadingOptions(awsParams)));

final CommandArgs subCommandArgs = createSubCommandArgs();
params.addAll(subCommandArgs.params);
Expand All @@ -160,6 +168,10 @@ public List<String> createCmdLineParams() {
signerConfig
.getV3KeystoresBulkloadParameters()
.ifPresent(setV3KeystoresBulkloadParameters(yamlConfig));

signerConfig
.getAwsParameters()
.ifPresent(awsParams -> yamlConfig.append(awsKmsBulkLoadingOptions(awsParams)));
}

signerConfig
Expand Down Expand Up @@ -475,71 +487,70 @@ private String createEth2SlashingProtectionArgs() {
return yamlConfig.toString();
}

private String awsBulkLoadingOptions(
final AwsSecretsManagerParameters awsSecretsManagerParameters) {
private String awsSecretsManagerBulkLoadingOptions(final AwsVaultParameters awsVaultParameters) {
final StringBuilder yamlConfig = new StringBuilder();

yamlConfig.append(
String.format(
YAML_BOOLEAN_FMT,
"eth2." + AWS_SECRETS_ENABLED_OPTION.substring(2),
awsSecretsManagerParameters.isEnabled()));
awsVaultParameters.isEnabled()));

yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_AUTH_MODE_OPTION.substring(2),
awsSecretsManagerParameters.getAuthenticationMode().name()));
awsVaultParameters.getAuthenticationMode().name()));

if (awsSecretsManagerParameters.getAccessKeyId() != null) {
if (awsVaultParameters.getAccessKeyId() != null) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_ACCESS_KEY_ID_OPTION.substring(2),
awsSecretsManagerParameters.getAccessKeyId()));
awsVaultParameters.getAccessKeyId()));
}

if (awsSecretsManagerParameters.getSecretAccessKey() != null) {
if (awsVaultParameters.getSecretAccessKey() != null) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_SECRET_ACCESS_KEY_OPTION.substring(2),
awsSecretsManagerParameters.getSecretAccessKey()));
awsVaultParameters.getSecretAccessKey()));
}

if (awsSecretsManagerParameters.getRegion() != null) {
if (awsVaultParameters.getRegion() != null) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_REGION_OPTION.substring(2),
awsSecretsManagerParameters.getRegion()));
awsVaultParameters.getRegion()));
}

if (!awsSecretsManagerParameters.getPrefixesFilter().isEmpty()) {
if (!awsVaultParameters.getPrefixesFilter().isEmpty()) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_PREFIXES_FILTER_OPTION.substring(2),
String.join(",", awsSecretsManagerParameters.getPrefixesFilter())));
String.join(",", awsVaultParameters.getPrefixesFilter())));
}

if (!awsSecretsManagerParameters.getTagNamesFilter().isEmpty()) {
if (!awsVaultParameters.getTagNamesFilter().isEmpty()) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_TAG_NAMES_FILTER_OPTION.substring(2),
String.join(",", awsSecretsManagerParameters.getTagNamesFilter())));
String.join(",", awsVaultParameters.getTagNamesFilter())));
}

if (!awsSecretsManagerParameters.getTagValuesFilter().isEmpty()) {
if (!awsVaultParameters.getTagValuesFilter().isEmpty()) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth2." + AWS_SECRETS_TAG_VALUES_FILTER_OPTION.substring(2),
String.join(",", awsSecretsManagerParameters.getTagValuesFilter())));
String.join(",", awsVaultParameters.getTagValuesFilter())));
}

awsSecretsManagerParameters
awsVaultParameters
.getEndpointOverride()
.ifPresent(
uri ->
Expand All @@ -552,6 +563,74 @@ private String awsBulkLoadingOptions(
return yamlConfig.toString();
}

private String awsKmsBulkLoadingOptions(final AwsVaultParameters awsVaultParameters) {
final StringBuilder yamlConfig = new StringBuilder();

yamlConfig.append(
String.format(
YAML_BOOLEAN_FMT,
"eth1." + AWS_KMS_ENABLED_OPTION.substring(2),
awsVaultParameters.isEnabled()));

yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth1." + AWS_KMS_AUTH_MODE_OPTION.substring(2),
awsVaultParameters.getAuthenticationMode().name()));

if (awsVaultParameters.getAccessKeyId() != null) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth1." + AWS_KMS_ACCESS_KEY_ID_OPTION.substring(2),
awsVaultParameters.getAccessKeyId()));
}

if (awsVaultParameters.getSecretAccessKey() != null) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth1." + AWS_KMS_SECRET_ACCESS_KEY_OPTION.substring(2),
awsVaultParameters.getSecretAccessKey()));
}

if (awsVaultParameters.getRegion() != null) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth1." + AWS_KMS_REGION_OPTION.substring(2),
awsVaultParameters.getRegion()));
}

if (!awsVaultParameters.getTagNamesFilter().isEmpty()) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth1." + AWS_KMS_TAG_NAMES_FILTER_OPTION.substring(2),
String.join(",", awsVaultParameters.getTagNamesFilter())));
}

if (!awsVaultParameters.getTagValuesFilter().isEmpty()) {
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth1." + AWS_KMS_TAG_VALUES_FILTER_OPTION.substring(2),
String.join(",", awsVaultParameters.getTagValuesFilter())));
}

awsVaultParameters
.getEndpointOverride()
.ifPresent(
uri ->
yamlConfig.append(
String.format(
YAML_STRING_FMT,
"eth1." + AWS_ENDPOINT_OVERRIDE_OPTION.substring(2),
uri)));

return yamlConfig.toString();
}

private String formatStringList(final String key, final List<String> stringList) {
return stringList.isEmpty()
? String.format("%s: []%n", key)
Expand Down
Loading

0 comments on commit 7993730

Please sign in to comment.