From 011fad2ae2526b5436fc3e72d781998543995285 Mon Sep 17 00:00:00 2001 From: Carlos Bermudez Porto <43155355+cbermudez97@users.noreply.github.com> Date: Thu, 4 Jul 2024 08:52:17 -0400 Subject: [PATCH 1/2] feat: Allow keystores creation with eth1 withdrawal credentials (#235) * feat: allow keystores creation with eth1 withdrawal credentials * fix: error when passing empty withdrawal eth1 address * chore: Add devcontainer config * fix: Withdrawal credentials prefix zeros in deposit data. * feat: Add json schema and logic to validate deposit_data.json * test: Add e2e test for keys gen with eth1 withdrawal cred * feat: Support installing deps on debian 12 * fix: fix e2e tests error in Windows * fix: e2e sedge keys test for Windows --------- Co-authored-by: Miguel Tenorio <46824157+AntiD2ta@users.noreply.github.com> Co-authored-by: AntiD2ta Co-authored-by: Haneen Khalifa --- .devcontainer/devcontainer.json | 45 ++++++ cli/keys.go | 32 +++- configs/errors.go | 137 +++++++++--------- configs/messages.go | 75 +++++----- e2e/e2e.go | 21 ++- e2e/generate_test.go | 4 +- e2e/keys_test.go | 84 +++++++++++ .../pkg/keystores/keystore_external_utils.go | 32 ++-- .../keystore_schema_deposit_data.json | 73 ++++++++++ internal/pkg/keystores/types.go | 1 + internal/pkg/keystores/validate.go | 48 ++++-- templates/setup/linux/docker/debian_12.sh | 24 +++ 12 files changed, 431 insertions(+), 145 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 e2e/keys_test.go create mode 100644 internal/pkg/keystores/keystore_schema_deposit_data.json create mode 100644 templates/setup/linux/docker/debian_12.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..ab9c306f --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,45 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/go +{ + "name": "Go", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/go:1-1.22-bookworm", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "ghcr.io/devcontainers-contrib/features/curl-apt-get:1": {} + }, + "customizations": { + "vscode": { + "extensions": [ + "GitHub.copilot", + "streetsidesoftware.code-spell-checker", + "ms-azuretools.vscode-docker", + "dbaeumer.vscode-eslint", + "donjayamanne.githistory", + "GitHub.copilot-chat", + "GitHub.vscode-pull-request-github", + "golang.go", + "yzhang.markdown-all-in-one", + "DavidAnson.vscode-markdownlint", + "unifiedjs.vscode-mdx", + "redhat.vscode-yaml", + "eamodio.gitlens" + ] + } + } + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "go version", + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/cli/keys.go b/cli/keys.go index 3d711776..3b9d4ea9 100644 --- a/cli/keys.go +++ b/cli/keys.go @@ -24,6 +24,7 @@ import ( "github.com/NethermindEth/sedge/internal/pkg/commands" "github.com/NethermindEth/sedge/internal/pkg/keystores" "github.com/NethermindEth/sedge/internal/ui" + "github.com/NethermindEth/sedge/internal/utils" eth2 "github.com/protolambda/zrnt/eth2/configs" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -83,6 +84,10 @@ func KeysCmd(cmdRunner commands.CommandRunner, p ui.Prompter) *cobra.Command { if err := configs.NetworkCheck(flags.network); err != nil { log.Fatal(err.Error()) } + // Validate fee recipient + if flags.eth1WithdrawalAddress != "" && !utils.IsAddress(flags.eth1WithdrawalAddress) { + log.Fatal(configs.ErrInvalidWithdrawalAddr) + } // Ensure that path is absolute log.Debugf("Path to keystore folder: %s", flags.path) absPath, err := filepath.Abs(flags.path) @@ -93,7 +98,10 @@ func KeysCmd(cmdRunner commands.CommandRunner, p ui.Prompter) *cobra.Command { return nil }, Run: func(cmd *cobra.Command, args []string) { - // TODO: allow usage of withdrawal address + // Warn about withdrawal address + if flags.eth1WithdrawalAddress != "" { + log.Warn(configs.WithdrawalAddressDefinedWarning) + } // Get keystore passphrase if !flags.randomPassphrase && flags.passphrasePath != "" { content, err := readFileContent(flags.passphrasePath) @@ -153,14 +161,22 @@ func KeysCmd(cmdRunner commands.CommandRunner, p ui.Prompter) *cobra.Command { flags.numberVal = numberVal } + keystorePath := filepath.Join(flags.path, "keystore") + + var withdrawalAddress string + if flags.eth1WithdrawalAddress != "" { + withdrawalAddress = flags.eth1WithdrawalAddress[2:] + } + data := keystores.ValidatorKeysGenData{ - Mnemonic: mnemonic, - Passphrase: passphrase, - OutputPath: keystorePath, - MinIndex: uint64(flags.existingVal), - MaxIndex: uint64(flags.existingVal) + uint64(flags.numberVal), - NetworkName: flags.network, - ForkVersion: configs.NetworksConfigs()[flags.network].GenesisForkVersion, + Mnemonic: mnemonic, + Passphrase: passphrase, + OutputPath: keystorePath, + MinIndex: uint64(flags.existingVal), + MaxIndex: uint64(flags.existingVal) + uint64(flags.numberVal), + NetworkName: flags.network, + ForkVersion: configs.NetworksConfigs()[flags.network].GenesisForkVersion, + WithdrawalAddress: withdrawalAddress, // Constants UseUniquePassphrase: true, Insecure: false, diff --git a/configs/errors.go b/configs/errors.go index abdedcae..ca1de8c1 100644 --- a/configs/errors.go +++ b/configs/errors.go @@ -17,71 +17,74 @@ package configs // All the strings that are needed for error logging. const ( - ReadingInstructionError = "failed to read instructions from file %s" - IncorrectClientError = "incorrect %s client name \"%s\". Please provide correct client name. Use 'clients' command to see the list of supported clients" - ClosingFileError = "failed to close file %s" - ScriptIsNotRunningError = "services of docker-compose script provided are not running. Error: %v" - GettingLogsError = "failed to get logs for services %s. Error: %v" - DockerComposePsReturnedEmptyError = "'docker compose ps --services' returned empty string" - InvalidVolumePathError = "invalid path provided: %s. If you intended to pass a host directory, use absolute path" - ZipError = "all lists must have the same size" - CommandError = "command '%s' throws error: %v" - DistroInfoError = "failed to get linux distribution info. Error: %v" - EmptyClientMapError = "is not possible to select a random element from an empty collection" - NoSupportedClientsError = "collection of clients given for random choice doesn't have any supported client. Check the target network (flag --network). Use 'clients' command to see the list of supported clients for every supported network" - NetworkValidationFailedError = "'network' flag validation failed. Error: %v" - UnknownNetworkError = "unknown network \"%s\". Please provide correct network name. Use 'networks' command to see the list of supported networks" - GenerateJWTSecretError = "JWT secret generation failed. Error: %v" - GetPWDError = "something failed trying to get current working directory. Error: %v" - EmptyFeeRecipientError = "you should provide an Ethereum address for the Fee Recipient" - KeystorePasswordError = "keystore password must have more than 8 characters" - PortOccupationError = "port occupation check failed. Error: %v" - DefaultPortInvalidError = "default %s can not be zero" - PrintFileError = "error printing file content: %v" - CleaningEnvFileError = "error cleaning env file: %v" - CleaningDCFileError = "error cleaning docker compose file: %v" - PassphraseReadFileError = "error reading passphrase file: %v" - MnemonicReadFileError = "error reading passphrase file: %v" - MnemonicGenerationError = "error creating mnemonic: %v" - KeyEntryGenerationError = "error generating keystore: could not read sufficient secure random bytes" - AESParamsCreationError = "failed to create AES128CTR params: %w" - SecretEncryptionError = "failed to encrypt secret: %w" - KeystoreOutputExistingError = "output folder for keystores already exists" - KeystoreGenerationError = "error generating keystores: %v" - KeystoreDerivationError = "keystore %s cannot be derived, continuing to next keystore" - KeystoreExistingInWalletError = "keystore with name \"%s\" already exists" - KeystoreImportingError = "failed to import keystore with pubkey %s into output wallet: %v" - InvalidMnemonicError = "mnemonic is not valid" - BadMnemonicError = "bad mnemonic: %v" - ForkVersionDecodeError = "cannot decode fork version: %v" - DepositFileWriteError = "cannot write deposit file: %v" - KeystoreSecretKeyCreationError = "failed to create validator private key for path %q: %v" - WithdrawalSecretKeyCreationError = "failed to create withdrawal private key for path %q: %v" - KeystoreSecretKeyConvertionError = "cannot convert validator priv key: %v" - DepositDataEncodingError = "could not encode deposit data to json: %v" - InvalidLoggingFlag = "bad logging flag: %v" - CannotGenerateSecret = "cannot generate 32 bytes long secret" - ShowMnemonicError = "error displaying mnemonic: %v" - InvalidFilePathOrUrl = "invalid filepath or url: %s" - CannotGetUrlContent = "cannot get url %s content: %v" - CannotReadFileContent = "cannot read file %s content: %v" - ErrorCheckingFile = "error checking file %s: %v" - DestFileAlreadyExist = "destiny file %s already exist" - ErrorCreatingFile = "error creating file %s: %v" - ErrorDownloadingFile = "error downloading file from %s: %v" - ErrorCopyingFile = "error copying file from %s: %v" - ErrorWritingDeployBlockFile = "error writing custom deploy block file %s: %v" - InvalidUrlFlagError = "invalid %s url %s. URL must be in the format http(s)://:///... or http://///" - InvalidEnodeError = "invalid enode %s. Bootnode must be in the format enode://@:" - InvalidEnrError = "invalid enr %s. ENR must be in the format enr:" - InvalidService = "provided service %s is invalid" - SetupContainersErr = "error setting up service containers: %w" - StartingContainersErr = "error starting service containers: %w" - ReadingYmlErr = "error reading yml file: %w" - ParsingYmlErr = "error parsing yml file, it seems is not a valid docker-compose script: %w" - ServicesNotFoundErr = "services not found in the docker-compose script" - InvalidComposeErr = "provided docker-compose script is invalid: %w" - ErrDuplicatedBootNode = "duplicated boot node" - ErrGraffitiLength = "graffiti must have 16 characters at most. Provided graffiti %s has %d characters" - ErrCMDArgsNotSupported = "command %s does not support arguments. Please use flags instead" + ReadingInstructionError = "failed to read instructions from file %s" + IncorrectClientError = "incorrect %s client name \"%s\". Please provide correct client name. Use 'clients' command to see the list of supported clients" + ClosingFileError = "failed to close file %s" + ScriptIsNotRunningError = "services of docker-compose script provided are not running. Error: %v" + GettingLogsError = "failed to get logs for services %s. Error: %v" + DockerComposePsReturnedEmptyError = "'docker compose ps --services' returned empty string" + InvalidVolumePathError = "invalid path provided: %s. If you intended to pass a host directory, use absolute path" + ZipError = "all lists must have the same size" + CommandError = "command '%s' throws error: %v" + DistroInfoError = "failed to get linux distribution info. Error: %v" + EmptyClientMapError = "is not possible to select a random element from an empty collection" + NoSupportedClientsError = "collection of clients given for random choice doesn't have any supported client. Check the target network (flag --network). Use 'clients' command to see the list of supported clients for every supported network" + NetworkValidationFailedError = "'network' flag validation failed. Error: %v" + UnknownNetworkError = "unknown network \"%s\". Please provide correct network name. Use 'networks' command to see the list of supported networks" + GenerateJWTSecretError = "JWT secret generation failed. Error: %v" + GetPWDError = "something failed trying to get current working directory. Error: %v" + EmptyFeeRecipientError = "you should provide an Ethereum address for the Fee Recipient" + KeystorePasswordError = "keystore password must have more than 8 characters" + PortOccupationError = "port occupation check failed. Error: %v" + DefaultPortInvalidError = "default %s can not be zero" + PrintFileError = "error printing file content: %v" + CleaningEnvFileError = "error cleaning env file: %v" + CleaningDCFileError = "error cleaning docker compose file: %v" + PassphraseReadFileError = "error reading passphrase file: %v" + MnemonicReadFileError = "error reading passphrase file: %v" + MnemonicGenerationError = "error creating mnemonic: %v" + KeyEntryGenerationError = "error generating keystore: could not read sufficient secure random bytes" + AESParamsCreationError = "failed to create AES128CTR params: %w" + SecretEncryptionError = "failed to encrypt secret: %w" + KeystoreOutputExistingError = "output folder for keystores already exists" + KeystoreGenerationError = "error generating keystores: %v" + KeystoreDerivationError = "keystore %s cannot be derived, continuing to next keystore" + KeystoreExistingInWalletError = "keystore with name \"%s\" already exists" + KeystoreImportingError = "failed to import keystore with pubkey %s into output wallet: %v" + InvalidMnemonicError = "mnemonic is not valid" + BadMnemonicError = "bad mnemonic: %v" + ForkVersionDecodeError = "cannot decode fork version: %v" + DepositFileWriteError = "cannot write deposit file: %v" + KeystoreSecretKeyCreationError = "failed to create validator private key for path %q: %v" + WithdrawalSecretKeyCreationError = "failed to create withdrawal private key for path %q: %v" + KeystoreSecretKeyConvertionError = "cannot convert validator priv key: %v" + DepositDataEncodingError = "could not encode deposit data to json: %v" + InvalidLoggingFlag = "bad logging flag: %v" + CannotGenerateSecret = "cannot generate 32 bytes long secret" + ShowMnemonicError = "error displaying mnemonic: %v" + InvalidFilePathOrUrl = "invalid filepath or url: %s" + CannotGetUrlContent = "cannot get url %s content: %v" + CannotReadFileContent = "cannot read file %s content: %v" + ErrorCheckingFile = "error checking file %s: %v" + DestFileAlreadyExist = "destiny file %s already exist" + ErrorCreatingFile = "error creating file %s: %v" + ErrorDownloadingFile = "error downloading file from %s: %v" + ErrorCopyingFile = "error copying file from %s: %v" + ErrorWritingDeployBlockFile = "error writing custom deploy block file %s: %v" + InvalidUrlFlagError = "invalid %s url %s. URL must be in the format http(s)://:///... or http://///" + InvalidEnodeError = "invalid enode %s. Bootnode must be in the format enode://@:" + InvalidEnrError = "invalid enr %s. ENR must be in the format enr:" + InvalidService = "provided service %s is invalid" + SetupContainersErr = "error setting up service containers: %w" + StartingContainersErr = "error starting service containers: %w" + ReadingYmlErr = "error reading yml file: %w" + ParsingYmlErr = "error parsing yml file, it seems is not a valid docker-compose script: %w" + ServicesNotFoundErr = "services not found in the docker-compose script" + InvalidComposeErr = "provided docker-compose script is invalid: %w" + ErrDuplicatedBootNode = "duplicated boot node" + ErrGraffitiLength = "graffiti must have 16 characters at most. Provided graffiti %s has %d characters" + ErrCMDArgsNotSupported = "command %s does not support arguments. Please use flags instead" + ErrWithdrawalEth1SecretKeyCreation = "failed to create withdrawal private key for address 0x%s: %v" + ErrWithdrawalBLSSecretKeyCreation = "failed to create withdrawal private key for path %q: %v" + ErrInvalidWithdrawalAddr = "provided withdrawal address is not a valid Ethereum address" ) diff --git a/configs/messages.go b/configs/messages.go index ebd6391f..f57f2177 100644 --- a/configs/messages.go +++ b/configs/messages.go @@ -23,43 +23,44 @@ import ( // All the strings that are needed for debugging and info logging, and constant strings. const ( - DefaultMevBoostEndpoint = "http://mev-boost" - DefaultEnvFileName = ".env" - CheckingDependencies = "Checking dependencies: %s" - DependenciesPending = "pending dependencies: %s" - DependenciesOK = "All dependencies are installed on host machine" - GeneratingDockerComposeScript = "Generating docker-compose script for current selection of clients" - GeneratingEnvFile = "Generating environment file for current selection of clients" - GeneratedDockerComposeScript = "Generated docker-compose script for current selection of clients" - GeneratedEnvFile = "Generated environment file for current selection of clients" - CleaningGeneratedFiles = "Cleaning generated docker-compose and environment files" - CleanedGeneratedFiles = "Cleaned generated files" - GenerationEnd = "Generation of files successfully, happy staking! You can use now 'sedge run' to start the setup." - Exiting = "Exiting..." - InstructionsFor = "Instructions for %s" - OSNotSupported = "installation not supported for %s" - ProvideClients = "Please provide both execution client and consensus client" - CreatedFile = "Created file %s" - DefaultSedgeDataFolderName = "sedge-data" - ClientNotSupported = "client %s is not supported. Please use 'clients' command to see the list of supported clients" - PrintingFile = "File %s:" - SupportedClients = "Supported clients of type %s: %v" - ConfigClientsMsg = "Provided clients of type %s in configuration file: %v" - RunningDockerCompose = "Running docker-compose script" - Component = "component" - RunningCommand = "Running command: %s" - UnableToProceed = "Unable to proceed. Please check the logs for more details" - CheckingDockerEngine = "Checking if docker engine is on" - DepositCLIDockerImageUrl = "nethermindeth/staking-deposit-cli" //"github.com/ethereum/staking-deposit-cli" - DepositCLIDockerImageName = "nethermindeth/staking-deposit-cli" //"deposit-cli:local" - GeneratingKeystores = "Generating keystores..." - GeneratingKeystoresLegacy = "Generating keystore folder" - KeystoresGenerated = "Keystores generated successfully" - GeneratingDepositData = "Generating deposit data..." - DepositDataGenerated = "Deposit data generated successfully" - KeysFoundAt = "If everything went well, your keys can be found at: %s" - ImageNotFoundBuilding = "Image %s not found, building it" - ImageNotFoundPulling = "Image %s not found, pulling it" + DefaultMevBoostEndpoint = "http://mev-boost" + DefaultEnvFileName = ".env" + CheckingDependencies = "Checking dependencies: %s" + DependenciesPending = "pending dependencies: %s" + DependenciesOK = "All dependencies are installed on host machine" + GeneratingDockerComposeScript = "Generating docker-compose script for current selection of clients" + GeneratingEnvFile = "Generating environment file for current selection of clients" + GeneratedDockerComposeScript = "Generated docker-compose script for current selection of clients" + GeneratedEnvFile = "Generated environment file for current selection of clients" + CleaningGeneratedFiles = "Cleaning generated docker-compose and environment files" + CleanedGeneratedFiles = "Cleaned generated files" + GenerationEnd = "Generation of files successfully, happy staking! You can use now 'sedge run' to start the setup." + Exiting = "Exiting..." + InstructionsFor = "Instructions for %s" + OSNotSupported = "installation not supported for %s" + ProvideClients = "Please provide both execution client and consensus client" + CreatedFile = "Created file %s" + DefaultSedgeDataFolderName = "sedge-data" + ClientNotSupported = "client %s is not supported. Please use 'clients' command to see the list of supported clients" + PrintingFile = "File %s:" + SupportedClients = "Supported clients of type %s: %v" + ConfigClientsMsg = "Provided clients of type %s in configuration file: %v" + RunningDockerCompose = "Running docker-compose script" + Component = "component" + RunningCommand = "Running command: %s" + UnableToProceed = "Unable to proceed. Please check the logs for more details" + CheckingDockerEngine = "Checking if docker engine is on" + DepositCLIDockerImageUrl = "nethermindeth/staking-deposit-cli" //"github.com/ethereum/staking-deposit-cli" + DepositCLIDockerImageName = "nethermindeth/staking-deposit-cli" //"deposit-cli:local" + GeneratingKeystores = "Generating keystores..." + GeneratingKeystoresLegacy = "Generating keystore folder" + KeystoresGenerated = "Keystores generated successfully" + GeneratingDepositData = "Generating deposit data..." + DepositDataGenerated = "Deposit data generated successfully" + KeysFoundAt = "If everything went well, your keys can be found at: %s" + ImageNotFoundBuilding = "Image %s not found, building it" + ImageNotFoundPulling = "Image %s not found, pulling it" + WithdrawalAddressDefinedWarning = "You have defined a withdrawal address for your validators. Make sure this is intended. Deposits made with this withdrawal address cannot be changed. If you want to change the withdrawal address, you will have to exit your validators and create new validators with the new withdrawal address." ReviewKeystorePath = "In case you used custom paths for the 'cli' or the 'keys' commands, please review if the keystore path in the generated .env file points to the generated keystore folder (the .env key should be KEYSTORE_DIR). If not, change the path in the .env file to the correct one." NodesSynced = "Execution and Consensus clients are synced, proceeding to start validator node" diff --git a/e2e/e2e.go b/e2e/e2e.go index 0d1bab57..9aa2af11 100644 --- a/e2e/e2e.go +++ b/e2e/e2e.go @@ -4,13 +4,14 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "testing" ) type ( e2eArranger func(t *testing.T, sedgePath string) error - e2eAct func(t *testing.T, sedgePath string) - e2eAssert func(t *testing.T) + e2eAct func(t *testing.T, sedgePath, dataDirPath string) + e2eAssert func(t *testing.T, dataDirPath string) ) type e2eTestCase struct { @@ -54,15 +55,19 @@ func (e *e2eTestCase) run() { } } if e.act != nil { - e.act(e.t, e.BinaryPath()) + e.act(e.t, e.BinaryPath(), e.dataDirPath()) } if e.assert != nil { - e.assert(e.t) + e.assert(e.t, e.dataDirPath()) } } func (e *e2eTestCase) BinaryPath() string { - return filepath.Join(e.testDir, "sedge") + binaryName := "sedge" + if runtime.GOOS == "windows" { + binaryName += ".exe" + } + return filepath.Join(e.testDir, binaryName) } func (e *e2eTestCase) Cleanup() { @@ -98,7 +103,11 @@ func (e *e2eTestCase) installGoModules() { func (e *e2eTestCase) build() { e.t.Helper() - outPath := filepath.Join(e.testDir, "sedge") + binaryName := "sedge" + if runtime.GOOS == "windows" { + binaryName += ".exe" + } + outPath := filepath.Join(e.testDir, binaryName) e.t.Logf("Building binary to %s", outPath) err := exec.Command("go", "build", "-o", outPath, filepath.Join(e.repoPath, "cmd", "sedge", "main.go")).Run() if err != nil { diff --git a/e2e/generate_test.go b/e2e/generate_test.go index 3a2a7dcd..08c28970 100644 --- a/e2e/generate_test.go +++ b/e2e/generate_test.go @@ -17,11 +17,11 @@ func TestGenerate_FullNode_GoerliNotSupported(t *testing.T) { // Arrange nil, // Act - func(t *testing.T, binaryPath string) { + func(t *testing.T, binaryPath, dataDirPath string) { runErr = runSedge(t, binaryPath, "generate", "full-node", "--network", "goerli") }, // Assert - func(t *testing.T) { + func(t *testing.T, dataDirPath string) { assert.Error(t, runErr, "generate command should fail without arguments") }, ) diff --git a/e2e/keys_test.go b/e2e/keys_test.go new file mode 100644 index 00000000..619891ed --- /dev/null +++ b/e2e/keys_test.go @@ -0,0 +1,84 @@ +package e2e + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/NethermindEth/sedge/internal/pkg/keystores" + "github.com/stretchr/testify/assert" +) + +type depositDataKey struct { + Account string `json:"account"` + Amount int `json:"amount"` + DepositCliVersion string `json:"deposit_cli_version"` + DepositDataRoot string `json:"deposit_data_root"` + DepositMessageRoot string `json:"deposit_message_root"` + ForkVersion string `json:"fork_version"` + NetworkName string `json:"network_name"` + Pubkey string `json:"pubkey"` + Signature string `json:"signature"` + Version int `json:"version"` + WithdrawalCredentials string `json:"withdrawal_credentials"` +} + +func TestKeys_Eth1_Withdrawal_Keys_Mainnet(t *testing.T) { + // Test context + var ( + runErr error + ) + // Build test case + e2eTest := newE2ETestCase( + t, + // Arrange + func(t *testing.T, binaryPath string) error { + mnemonicPathFile := filepath.Join(filepath.Dir(binaryPath), "mnemonic.txt") + file, err := os.Create(mnemonicPathFile) + if err != nil { + return err + } + defer file.Close() + mnemonicText := "science ill robust clump oxygen intact barely horror athlete eyebrow cave target hero input entry citizen wink affair entire alert sick flight gossip refuse" + _, err = file.WriteString(mnemonicText) + return err + }, + // Act + func(t *testing.T, binaryPath, dataDirPath string) { + mnemonicPathFile := filepath.Join(filepath.Dir(binaryPath), "mnemonic.txt") + runErr = runSedge(t, binaryPath, "keys", + "--eth1-withdrawal-address", "0xb794f5ea0ba39494ce839613fffba74279579268", + "--network", "mainnet", + "--num-validators", "10", + "--mnemonic-path", mnemonicPathFile, + "--existing", "0", + "--random-passphrase", + "--path", dataDirPath) + }, + // Assert + func(t *testing.T, dataDirPath string) { + assert.NoError(t, runErr, "keys command should not fail") + // Check if the deposit_data.json was created + depositDataFilePath := filepath.Join(dataDirPath, "keystore", keystores.DepositDataFileName) + assert.FileExists(t, depositDataFilePath, "deposit_data.json should be created") + + // Check if the deposit_data.json is valid + var keys []depositDataKey + jsonData, err := os.ReadFile(depositDataFilePath) + assert.NoError(t, err, "error reading deposit_data.json") + err = json.Unmarshal([]byte(jsonData), &keys) + assert.NoError(t, err, "error unmarshalling json") + + pattern := `^010000000000000000000000[a-fA-F0-9]{40}$` + regex := regexp.MustCompile(pattern) + for _, key := range keys { + assert.Regexp(t, regex, key.WithdrawalCredentials, "withdrawal_credentials should match the pattern") + assert.Equal(t, key.NetworkName, "mainnet", "network_name should be mainnet") + } + }, + ) + // Run test case + e2eTest.run() +} diff --git a/internal/pkg/keystores/keystore_external_utils.go b/internal/pkg/keystores/keystore_external_utils.go index 3ee4ab00..dfcc7de4 100644 --- a/internal/pkg/keystores/keystore_external_utils.go +++ b/internal/pkg/keystores/keystore_external_utils.go @@ -311,24 +311,38 @@ func CreateDepositData( depositData.WriteString("[") } for i := vkgd.MinIndex; i < vkgd.MaxIndex; i++ { + // Validator credentials valAccPath := fmt.Sprintf("m/12381/3600/%d/0/0", i) val, err := util.PrivateKeyFromSeedAndPath(valSeed, valAccPath) if err != nil { return fmt.Errorf(configs.KeystoreSecretKeyCreationError, valAccPath, err) } - withdrAccPath := fmt.Sprintf("m/12381/3600/%d/0", i) - withdr, err := util.PrivateKeyFromSeedAndPath(withdrSeed, withdrAccPath) - if err != nil { - return fmt.Errorf(configs.WithdrawalSecretKeyCreationError, withdrAccPath, err) - } - var pub common.BLSPubkey copy(pub[:], val.PublicKey().Marshal()) + // Withdrawal credentials var withdrPub common.BLSPubkey - copy(withdrPub[:], withdr.PublicKey().Marshal()) - withdrCreds := hashing.Hash(withdrPub[:]) - withdrCreds[0] = common.BLS_WITHDRAWAL_PREFIX + var withdrPrefix byte + var withdrCreds [32]byte + if vkgd.WithdrawalAddress != "" { + withdrPrefix = common.ETH1_ADDRESS_WITHDRAWAL_PREFIX + eth1Addr, err := hex.DecodeString(vkgd.WithdrawalAddress) + if err != nil { + return fmt.Errorf(configs.ErrWithdrawalEth1SecretKeyCreation, vkgd.WithdrawalAddress, err) + } + // eleven zero bytes, then the address + copy(withdrCreds[12:], eth1Addr) + } else { + withdrPrefix = common.BLS_WITHDRAWAL_PREFIX + withdrAccPath := fmt.Sprintf("m/12381/3600/%d/0", i) + withdr, err := util.PrivateKeyFromSeedAndPath(withdrSeed, withdrAccPath) + if err != nil { + return fmt.Errorf(configs.ErrWithdrawalBLSSecretKeyCreation, withdrAccPath, err) + } + copy(withdrPub[:], withdr.PublicKey().Marshal()) + withdrCreds = hashing.Hash(withdrPub[:]) + } + withdrCreds[0] = withdrPrefix data := common.DepositData{ Pubkey: pub, diff --git a/internal/pkg/keystores/keystore_schema_deposit_data.json b/internal/pkg/keystores/keystore_schema_deposit_data.json new file mode 100644 index 00000000..0954f0cb --- /dev/null +++ b/internal/pkg/keystores/keystore_schema_deposit_data.json @@ -0,0 +1,73 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "account": { + "type": "string", + "pattern": "^m/12381/3600/\\d+/0/0$" + }, + "amount": { + "type": "integer", + "minimum": 0 + }, + "deposit_cli_version": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+$" + }, + "deposit_data_root": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "deposit_message_root": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "fork_version": { + "type": "string", + "pattern": "^[a-f0-9]{8}$" + }, + "network_name": { + "type": "string", + "enum": ["mainnet", "holesky", "sepolia"] + }, + "pubkey": { + "type": "string", + "pattern": "^[a-f0-9]{96}$" + }, + "signature": { + "type": "string", + "pattern": "^[a-f0-9]{192}$" + }, + "version": { + "type": "integer", + "enum": [1] + }, + "withdrawal_credentials": { + "type": "string", + "oneOf": [ + { + "pattern": "^[a-fA-F0-9]{64}$" + }, + { + "pattern": "^010000000000000000000000[a-fA-F0-9]{40}$" + } + ] + } + }, + "required": [ + "account", + "amount", + "deposit_cli_version", + "deposit_data_root", + "deposit_message_root", + "fork_version", + "network_name", + "pubkey", + "signature", + "version", + "withdrawal_credentials" + ] + } + } \ No newline at end of file diff --git a/internal/pkg/keystores/types.go b/internal/pkg/keystores/types.go index b0c6da52..1265ef7e 100644 --- a/internal/pkg/keystores/types.go +++ b/internal/pkg/keystores/types.go @@ -27,4 +27,5 @@ type ValidatorKeysGenData struct { ForkVersion string AmountGwei uint64 AsJsonList bool + WithdrawalAddress string } diff --git a/internal/pkg/keystores/validate.go b/internal/pkg/keystores/validate.go index 03cce90a..8521f101 100644 --- a/internal/pkg/keystores/validate.go +++ b/internal/pkg/keystores/validate.go @@ -21,9 +21,10 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "os" "path/filepath" + "regexp" "github.com/santhosh-tekuri/jsonschema/v5" ) @@ -31,13 +32,18 @@ import ( //go:embed keystore_schema_eip2335.json var keystoreJSONSchema []byte +//go:embed keystore_schema_deposit_data.json +var depositDataJSONSchema []byte + var ( ErrDepositDataNotFound = errors.New("deposit_data.json not found") ErrKeystorePasswordNotFound = errors.New("keystore_password.txt not found") ErrValidatorKeysDirNotFound = errors.New("validator_keys directory not found") ErrValidatorKeysDirWithoutKeystores = errors.New("validator_keys directory does not contain any keystores") ErrInvalidKeystoreFile = errors.New("invalid keystore file") + ErrInvalidKeystoreFileName = errors.New("invalid keystore file name") ErrInvalidKeystoreFileSchema = errors.New("file does not match keystore schema (EIP-2335)") + ErrInvalidDepositDataFileSchema = errors.New("file does not match deposit data schema") ) const ( @@ -59,15 +65,6 @@ func ValidateKeystoreDir(dir string) (errors []error) { return } -func validateDepositDataFile(keystoreDirPath string) error { - depositDataFile, err := os.Stat(filepath.Join(keystoreDirPath, DepositDataFileName)) - if err != nil || depositDataFile.IsDir() { - return ErrDepositDataNotFound - } - // TODO check deposit_data.json is valid json - return nil -} - func validateKeystorePasswordFile(keystoreDirPath string) error { keystorePasswordFileName, err := os.Stat(filepath.Join(keystoreDirPath, KeystorePasswordFileName)) if err != nil || keystorePasswordFileName.IsDir() { @@ -101,19 +98,38 @@ func validateValidatorKeysFolder(keystoreDirPath string) error { } func validateKeystoreFile(path string) error { - // TODO validate file name + pattern := `^keystore-m_12381_3600_\d+_0_0\.json$` + regex := regexp.MustCompile(pattern) + if !regex.MatchString(filepath.Base(path)) { + return ErrInvalidKeystoreFileName + } + + return validateFileWithSchema(path, "keystore_schema", keystoreJSONSchema, ErrInvalidKeystoreFileSchema) +} + +func validateDepositDataFile(path string) error { + depositDataFile, err := os.Stat(filepath.Join(path, DepositDataFileName)) + if err != nil || depositDataFile.IsDir() { + return ErrDepositDataNotFound + } + + return nil + return validateFileWithSchema(path, "deposit_data_schema", depositDataJSONSchema, ErrInvalidDepositDataFileSchema) +} + +func validateFileWithSchema(path, schemaName string, schema []byte, schemaErr error) error { compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("keystore_schema", bytes.NewReader(keystoreJSONSchema)); err != nil { + if err := compiler.AddResource(schemaName, bytes.NewReader(schema)); err != nil { return err } - schema := compiler.MustCompile("keystore_schema") + schemaCmp := compiler.MustCompile(schemaName) f, err := os.Open(path) if err != nil { return err } defer f.Close() - data, err := ioutil.ReadAll(f) + data, err := io.ReadAll(f) if err != nil { return err } @@ -121,8 +137,8 @@ func validateKeystoreFile(path string) error { if err := json.Unmarshal(data, &value); err != nil { return err } - if err := schema.Validate(value); err != nil { - return ErrInvalidKeystoreFileSchema + if err := schemaCmp.Validate(value); err != nil { + return schemaErr } return nil } diff --git a/templates/setup/linux/docker/debian_12.sh b/templates/setup/linux/docker/debian_12.sh new file mode 100644 index 00000000..9e43b08d --- /dev/null +++ b/templates/setup/linux/docker/debian_12.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Update the apt package index and install packages to allow apt to use a repository over HTTPS +sudo apt-get update +sudo apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + lsb-release + +# Add Docker’s official GPG key +sudo mkdir -p /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + +# Set up the Docker repository +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Update the apt package index again +sudo apt-get update + +# Install Docker Engine, containerd, and Docker Compose +sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin From 74604f57c2f771440050a5176b0ffd1713c2eb91 Mon Sep 17 00:00:00 2001 From: Haneen Khalifa <124837763+khalifaa55@users.noreply.github.com> Date: Thu, 4 Jul 2024 17:05:07 +0300 Subject: [PATCH 2/2] feat: Add support for mev-boost on Holesky (#380) * feat: add mev RelayUrls for Holesky * feat: update holesky/envs_base.tmpl * test: add test case in TestSupportMEVBoost * docs: update docs, README.md, and CHANGELOG.md * feat: update TestCli/consensus_node_holesky * feat: update TestCli/consensus_node_holesky --------- Co-authored-by: Marcos Antonio Maceo <35319980+stdevMac@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ README.md | 12 ++++++------ cli/cli_test.go | 2 ++ configs/init.go | 8 +++++++- configs/networks_test.go | 5 +++++ docs/docs/networks/holesky.mdx | 8 ++++++++ templates/envs/holesky/env_base.tmpl | 4 ++-- 7 files changed, 34 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed4afa0c..304b02a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add support for MEV-boost on Holesky. + ### Changed - Update client images to Dencun-ready versions. diff --git a/README.md b/README.md index b4146c0e..d5f2a320 100644 --- a/README.md +++ b/README.md @@ -174,12 +174,12 @@ Users acknowledge that no warranty is being made of a successful installation. S ### CL clients with Mev-Boost -| Client | Mev-Boost | Networks | -| ---------- | --------- |--------------------------| -| Lighthouse | yes | Mainnet, Sepolia | -| Lodestar | yes | Mainnet, Sepolia | -| Prysm | yes | Mainnet, Sepolia | -| Teku | yes | Mainnet, Sepolia | +| Client | Mev-Boost | Networks | +| ---------- | --------- |---------------------------| +| Lighthouse | yes | Mainnet, Sepolia, Holesky | +| Lodestar | yes | Mainnet, Sepolia, Holesky | +| Prysm | yes | Mainnet, Sepolia, Holesky | +| Teku | yes | Mainnet, Sepolia, Holesky | ## Supported Linux flavours for dependency installation diff --git a/cli/cli_test.go b/cli/cli_test.go index 922b05a9..a65f1168 100644 --- a/cli/cli_test.go +++ b/cli/cli_test.go @@ -372,6 +372,7 @@ func TestCli(t *testing.T) { MapAllPorts: false, ExecutionApiUrl: "http://execution:5051", ExecutionAuthUrl: "http://execution:5051", + MevBoostEndpoint: "http://mev-boost:3030", ContainerTag: "tag", JWTSecretPath: filepath.Join(generationPath, "jwtsecret"), } @@ -383,6 +384,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().Input("Container tag, sedge will add to each container and the network, a suffix with the tag", "", false, nil).Return("tag", nil), prompter.EXPECT().Select("Select consensus client", "", ETHClients["consensus"]).Return(3, nil), prompter.EXPECT().InputURL("Checkpoint sync URL", configs.NetworksConfigs()[genData.Network].CheckpointSyncURL, false).Return("https://checkpoint-sync.holesky.ethpandaops.io/", nil), + prompter.EXPECT().InputURL("Mev-Boost endpoint", "", false).Return("http://mev-boost:3030", nil), prompter.EXPECT().InputURL("Execution API URL", "", true).Return("http://execution:5051", nil), prompter.EXPECT().InputURL("Execution Auth API URL", "", true).Return("http://execution:5051", nil), prompter.EXPECT().EthAddress("Please enter the Fee Recipient address (press enter to skip it)", "", false).Return("0x2d07a21ebadde0c13e8b91022a7e5732eb6bf5d5", nil), diff --git a/configs/init.go b/configs/init.go index 8a483df3..db5dd533 100644 --- a/configs/init.go +++ b/configs/init.go @@ -59,8 +59,14 @@ var networksConfigs map[string]NetworkConfig = map[string]NetworkConfig{ Name: NetworkHolesky, NetworkService: "merge", GenesisForkVersion: "0x00017000", - SupportsMEVBoost: false, + SupportsMEVBoost: true, CheckpointSyncURL: "https://checkpoint-sync.holesky.ethpandaops.io/", + RelayURLs: []string{ + "https://0xafa4c6985aa049fb79dd37010438cfebeb0f2bd42b115b89dd678dab0670c1de38da0c4e9138c9290a398ecd9a0b3110@boost-relay-holesky.flashbots.net", + "https://0x8db06236d88cf080e541f894507f6c933d40333405c36c8ea00158c165628ea57ad59b024467fe7d4d31113fadc0e187@holesky.agnostic-relay.net", + "https://0xab78bf8c781c58078c3beb5710c57940874dd96aef2835e7742c866b4c7c0406754376c2c8285a36c630346aa5c5f833@holesky.aestus.live", + "https://0xaa58208899c6105603b74396734a6263cc7d947f444f396a90f7b7d3e65d102aec7e5e5291b27e08d02c50a050825c2f@holesky.titanrelay.xyz", + }, }, NetworkCustom: { Name: NetworkCustom, diff --git a/configs/networks_test.go b/configs/networks_test.go index 332ed735..d11f995f 100644 --- a/configs/networks_test.go +++ b/configs/networks_test.go @@ -74,6 +74,11 @@ func TestSupportMEVBoost(t *testing.T) { network: "sepolia", want: true, }, + { + name: "Valid network, holesky", + network: "holesky", + want: true, + }, { name: "Valid network, gnosis", network: "gnosis", diff --git a/docs/docs/networks/holesky.mdx b/docs/docs/networks/holesky.mdx index 862fbd7a..51e732dc 100644 --- a/docs/docs/networks/holesky.mdx +++ b/docs/docs/networks/holesky.mdx @@ -25,6 +25,14 @@ Holesky is Ethereum's public testnet that serves as a technical experimentation - [Prysm](https://docs.prylabs.network/docs/getting-started/) - [Teku](https://docs.teku.consensys.net/en/latest/) +## MEV-Boost + +We support [mev-boost](https://github.com/flashbots/mev-boost) on holesky by default, in case you want to remove it, +run `sedge generate full-node` using the `--no-mev-boost` flag, for example: + +``` +sedge generate full-node --network=holesky --no-mev-boost=true +``` ## Generating a Full Node diff --git a/templates/envs/holesky/env_base.tmpl b/templates/envs/holesky/env_base.tmpl index 95c0b8c7..fbecca6d 100644 --- a/templates/envs/holesky/env_base.tmpl +++ b/templates/envs/holesky/env_base.tmpl @@ -1,8 +1,8 @@ {{/* docker-compose_base.tmpl */}} {{ define "env" }} # --- Global configuration --- -NETWORK=holesky -{{if .FeeRecipient}} +NETWORK=holesky{{if .WithMevBoostClient}} +RELAY_URLS={{.RelayURLs}}{{end}}{{if .FeeRecipient}} FEE_RECIPIENT={{.FeeRecipient}}{{end}} {{template "execution" .}} {{template "consensus" .}}