diff --git a/.gitignore b/.gitignore index 1f55638..9c07107 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ .idea /ethereum/data/geth/ /storage/ -node_modules \ No newline at end of file +node_modules +vagrant-box +.vagrant +*.log \ No newline at end of file diff --git a/README.adoc b/README.adoc index d848bf6..fc185fc 100644 --- a/README.adoc +++ b/README.adoc @@ -1,148 +1,35 @@ = KEEP & tBTC local setup -== Setting up development environment -If you’re on macOS, install Homebrew and https://github.com/keep-network/keep-core/blob/master/scripts/macos-setup.sh[macos-setup.sh] script to setup all necessary tools. +This repository provide a bunch of scripts which allow to install tBTC and +Keep locally for development and test purposes. It also contains and +enables execution of automated end-to-end system tests for tBTC. -If you are not on macOS or you don't have Homebrew, you need to install: +Basically, the entire setup operates on several component submodules: -- Go compiler, at least 1.14 -- Geth (Ethereum client) 1.9.9-stable -- Solidity, at least 0.5.17 -- Truffle, at least 5.1.18 -- Protobuf compiler, at least 3.11.4 -- `protoc-gen-gogoslick` toolchain -- `jq` +- https://github.com/keep-network/keep-core[`keep-core`] +- https://github.com/keep-network/keep-ecdsa[`keep-ecdsa`] +- https://github.com/keep-network/tbtc[`tbtc`] +- https://github.com/keep-network/tbtc.js[`tbtc.js`] +- https://github.com/keep-network/tbtc-dapp[`tbtc-dapp`] +- https://github.com/keep-network/relays[`relays`] -Regardless you use `macos-setup.sh` or not, you should also install: +It also uses some auxiliary software needed by tBTC and Keep: -- https://github.com/pyenv/pyenv[pyenv], at least 1.2.18 -- https://github.com/pypa/pipenv[pipenv], at least 2018.11.26 -- https://docs.docker.com/get-docker[Docker] & https://docs.docker.com/compose/install/[Docker Compose] +- https://github.com/ethereum/go-ethereum[Geth] +- https://bitcoincore.org[Bitcoin Core] +- https://electrumx.readthedocs.io/en/latest[ElectrumX] -== Node.js version notice +== Installation -Please use Node `v11.15.0` for everything but E2E test script. -To run the test script, please use Node `v14.3.0`. +The installation can be performed in two different ways described below: -We recommend using https://github.com/nvm-sh/nvm[nvm] to manage node versions easily. -You can install both versions by doing: -``` -nvm install 11.15.0 -nvm install 14.3.0 -``` -And use the specific version: -``` -nvm use 11.15.0 -nvm use 14.3.0 -``` +=== Automatic installation on separate virtual machine -== Installing the system -. Initialize local `geth` using: -+ -``` -./initialize-geth.sh -``` -+ -You can skip this step if your local geth is already initialized. This script clears all Ethereum client data, initilizes genesis block, and funds five accounts. All client data are kept in the working directory of this project so all the data used for other projects stay untouched. +To install everything automatically on a separate VirtualBox machine using Packer and +Vagrant, please follow the <<./docs/auto-setup.adoc#title, automatic setup guide>>. -. Run local `geth` node using: -+ -``` -./run-geth.sh -``` -. Run local Bitcoin Core node and ElectrumX using: -+ -``` -./run-bitcoin.sh -``` -. Run Keep & tBTC installation script: -+ -``` -./install.sh -``` -+ -This script will fetch `keep-core`, `keep-ecdsa`, and `tbtc` source code, deploy contracts of `keep-core`, `keep-ecdsa`, and `tbtc`. It will also build `keep-core` and `keep-ecdsa` off-chain clients. -+ -Later on, if you decide to update `tbtc`, you can run just `install-tbtc.sh`. -+ -Keep in mind, `tbtc` depends on `keep-ecdsa` and `keep-ecdsa` depends on `keep-core` so if you decide to update `keep-ecdsa`, you can run `install-keep-ecdsa.sh` followed by `install-tbtc.sh`. -+ -If you decide to update `keep-core`, you have to run `install-keep-core.sh` followed by `install-keep-ecdsa.sh` followed by `install-tbtc.sh` or just the entire `install.sh` again. +=== Manual installation on your local machine -== Run clients -The above installation script will configure: +To install everything on your local machine by hand, +please follow the <<./docs/manual-setup.adoc#title, manual setup guide>>. -- 1 `keep-core` client -- 3 `keep-ecdsa` clients - -To run the `keep-core` client use: -``` -./run-core-1.sh -``` - -It is enough to run one `keep-core` client to generate a group and produce relay entries. Setting up more than one client locally is possible but consumes more resources. - -To run `keep-ecdsa` clients use: -``` -./run-ecdsa-1.sh -``` -``` -./run-ecdsa-2.sh -``` -``` -./run-ecdsa-3.sh -``` - -There are at least 3 `keep-ecdsa` clients needed to open a keep. Setting up more than three clients locally is possible but consumes more resources. - -== Beacon genesis - -Before the beacon is able to produce a first relay entry, genesis needs to happen. Genesis triggers the first random beacon group selection. - -Genesis should be triggered after `keep-core` client started with: -``` -cd keep-core -KEEP_ETHEREUM_PASSWORD="password" ./keep-core --config configs/config.local.1.toml relay genesis -``` - -Bonded ECDSA keep factory from `keep-ecdsa` contracts requests for new relay entry to reseed after each signer selection but it is also possible to request for a new relay entry manually with: -``` -cd keep-core -KEEP_ETHEREUM_PASSWORD="password" ./keep-core --config configs/config.local.1.toml relay request -``` - -= How to interact with the system - -You can interact with the system through the tBTC dApp or automated -end-to-end tests. Before you start interacting, make sure you: - -- Installed all system components using `install.sh` script -- Have a local Geth instance (`run-geth.sh`) working -- Have local Bitcoin Core and ElectrumX instances (`run-bitcoin.sh`) working -- Have 1 `keep-core` and 3 `keep-ecdsa` clients up and running - -== Keep Dashboard dApp - -To run the Keep Dashboard dApp invoke: -``` -./run-keep-dashboard.sh -``` - -== tBTC dApp - -To run the tBTC dApp against the local Bitcoin network, -make sure the tBTC dApp is configured to work with the local ElectrumX instance. -You can configure this in `./tbtc-dapp/src/wrappers/web3.js` file, -where `TBTC.withConfig` line occurs. -Then, you can invoke: -``` -./run-tbtc-dapp.sh -``` -The application will be available on `http://localhost:3000`. - -== E2E tests - -To run the automated end-to-end scenario invoke: -``` -./run-e2e-test.sh -``` diff --git a/deployments/local-setup-environment/packerfile.json b/deployments/local-setup-environment/packerfile.json new file mode 100644 index 0000000..257843a --- /dev/null +++ b/deployments/local-setup-environment/packerfile.json @@ -0,0 +1,67 @@ +{ + "builders": [ + { + "type": "vagrant", + "source_path": "ubuntu/xenial64", + "provider": "virtualbox", + "communicator": "ssh", + "output_dir": "vagrant-box" + }, + { + "type": "docker", + "image": "ubuntu:xenial", + "commit": true, + "run_command": [ + "-d", "-i", "-t", + "-w=/root", + "--entrypoint=/bin/bash", + "--", + "{{.Image}}" + ], + "changes": [ + "WORKDIR /root", + "ENTRYPOINT /bin/bash -l" + ] + } + ], + "provisioners": [ + { + "type": "shell", + "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} bash -l {{ .Path }}", + "inline": ["apt-get update && apt-get -y install sudo"], + "only": ["docker"] + }, + { + "type": "shell", + "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} bash -l {{ .Path }}", + "scripts": [ + "./provisioning/install-commons.sh", + "./provisioning/install-nodejs.sh", + "./provisioning/install-go.sh", + "./provisioning/install-geth.sh", + "./provisioning/install-solidity.sh", + "./provisioning/install-protobuf.sh", + "./provisioning/install-docker.sh", + "./provisioning/install-docker-compose.sh" + ] + }, + { + "type": "shell", + "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} bash -l {{ .Path }}", + "inline": ["npm -g config set user root"], + "only": ["docker"] + }, + { + "type": "shell", + "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} bash -l {{ .Path }}", + "script": "./provisioning/install-truffle.sh" + } + ], + "post-processors": [ + { + "type": "docker-tag", + "repository": "local-setup-environment", + "only": ["docker"] + } + ] +} \ No newline at end of file diff --git a/deployments/local-setup-environment/provisioning/install-commons.sh b/deployments/local-setup-environment/provisioning/install-commons.sh new file mode 100755 index 0000000..5d538bc --- /dev/null +++ b/deployments/local-setup-environment/provisioning/install-commons.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +echo "Installing common tools..." + +sudo apt-get update + +sudo apt-get install -y \ + curl \ + wget \ + git \ + unzip \ + jq \ + python \ + build-essential + +if ! [ -x "$(command -v curl)" ]; then echo "curl installation failed"; exit 1; fi +if ! [ -x "$(command -v wget)" ]; then echo "wget installation failed"; exit 1; fi +if ! [ -x "$(command -v git)" ]; then echo "git installation failed"; exit 1; fi +if ! [ -x "$(command -v unzip)" ]; then echo "unzip installation failed"; exit 1; fi +if ! [ -x "$(command -v jq)" ]; then echo "jq installation failed"; exit 1; fi +if ! [ -x "$(command -v python)" ]; then echo "python installation failed"; exit 1; fi +if ! [ -x "$(command -v make)" ]; then echo "build-essential installation failed"; exit 1; fi + +echo "Common tools have been installed successfully!" \ No newline at end of file diff --git a/deployments/local-setup-environment/provisioning/install-docker-compose.sh b/deployments/local-setup-environment/provisioning/install-docker-compose.sh new file mode 100755 index 0000000..c892980 --- /dev/null +++ b/deployments/local-setup-environment/provisioning/install-docker-compose.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +echo "Installing Docker Compose..." + +sudo curl -L "https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m)" \ + -o /usr/local/bin/docker-compose + +sudo chmod +x /usr/local/bin/docker-compose + +if ! [ -x "$(command -v docker-compose)" ]; then echo "Docker compose installation failed"; exit 1; fi + +echo "Docker Compose has been installed successfully!" \ No newline at end of file diff --git a/deployments/local-setup-environment/provisioning/install-docker.sh b/deployments/local-setup-environment/provisioning/install-docker.sh new file mode 100755 index 0000000..6678092 --- /dev/null +++ b/deployments/local-setup-environment/provisioning/install-docker.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +echo "Installing Docker..." + +sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + +sudo apt-get update +sudo apt-get install -y docker-ce docker-ce-cli containerd.io + +if ! [ -x "$(command -v docker)" ]; then echo "Docker installation failed"; exit 1; fi + +echo "Docker has been installed successfully!" \ No newline at end of file diff --git a/deployments/local-setup-environment/provisioning/install-geth.sh b/deployments/local-setup-environment/provisioning/install-geth.sh new file mode 100755 index 0000000..5364858 --- /dev/null +++ b/deployments/local-setup-environment/provisioning/install-geth.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +echo "Installing go-ethereum..." + +GETH_PACKAGE=geth-alltools-linux-amd64-1.9.9-01744997.tar.gz + +curl -O https://gethstore.blob.core.windows.net/builds/$GETH_PACKAGE + +tar -xvf $GETH_PACKAGE +mkdir ./go-ethereum && tar -xzf $GETH_PACKAGE -C ./go-ethereum --strip-components=1 +sudo chown -R root:root ./go-ethereum +sudo mv ./go-ethereum/* /usr/local/bin + +if ! [ -x "$(command -v geth)" ]; then echo "go-ethereum installation failed"; exit 1; fi + +echo "go-ethereum has been installed successfully!" \ No newline at end of file diff --git a/deployments/local-setup-environment/provisioning/install-go.sh b/deployments/local-setup-environment/provisioning/install-go.sh new file mode 100755 index 0000000..4b7fc42 --- /dev/null +++ b/deployments/local-setup-environment/provisioning/install-go.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +echo "Installing Go..." + +GOLANG_PACKAGE=go1.13.4.linux-amd64.tar.gz + +curl -O https://storage.googleapis.com/golang/$GOLANG_PACKAGE + +tar -xvf $GOLANG_PACKAGE +sudo chown -R root:root ./go +sudo mv go /usr/local + +echo 'GOPATH="$HOME/go"' >> ~/.profile +echo 'PATH="$PATH:/usr/local/go/bin:$GOPATH/bin"' >> ~/.profile +source ~/.profile + +if ! [ -x "$(command -v go)" ]; then echo "Go installation failed"; exit 1; fi + +echo "Go has been installed successfully!" \ No newline at end of file diff --git a/deployments/local-setup-environment/provisioning/install-nodejs.sh b/deployments/local-setup-environment/provisioning/install-nodejs.sh new file mode 100755 index 0000000..351b199 --- /dev/null +++ b/deployments/local-setup-environment/provisioning/install-nodejs.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +echo "Installing Node.js and NPM..." + +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash + +echo 'export NVM_DIR="$HOME/.nvm"' >> ~/.profile +echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> ~/.profile +source ~/.profile + +nvm install 14.3.0 +nvm install 11.15.0 +nvm alias default 11.15.0 +nvm use default + +if ! [ -x "$(command -v node)" ]; then echo "Node installation failed"; exit 1; fi +if ! [ -x "$(command -v npm)" ]; then echo "NPM installation failed"; exit 1; fi + +echo "Node.js and NPM have been installed successfully!" \ No newline at end of file diff --git a/deployments/local-setup-environment/provisioning/install-protobuf.sh b/deployments/local-setup-environment/provisioning/install-protobuf.sh new file mode 100755 index 0000000..85421ca --- /dev/null +++ b/deployments/local-setup-environment/provisioning/install-protobuf.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +echo "Installing Protobuf..." + +PROTOC_VERSION=3.11.4 +PROTOC_PACKAGE=protoc-$PROTOC_VERSION-linux-x86_64.zip + +wget https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_PACKAGE + +mkdir ./protoc && unzip $PROTOC_PACKAGE -d ./protoc +chmod 755 -R ./protoc +sudo mv protoc/bin/protoc /usr/local/bin +sudo mv protoc/include/* /usr/local/include + +go get -u github.com/gogo/protobuf/protoc-gen-gogoslick + +if ! [ -x "$(command -v protoc)" ]; then echo "protoc installation failed"; exit 1; fi +if ! [ -x "$(command -v protoc-gen-gogoslick)" ]; then echo "protoc-gen-gogoslick installation failed"; exit 1; fi + +echo "Protobuf has been installed successfully!" \ No newline at end of file diff --git a/deployments/local-setup-environment/provisioning/install-solidity.sh b/deployments/local-setup-environment/provisioning/install-solidity.sh new file mode 100755 index 0000000..d8b0b48 --- /dev/null +++ b/deployments/local-setup-environment/provisioning/install-solidity.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +echo "Installing Solidity..." + +SOLC_VERSION=v0.5.17 + +wget https://github.com/ethereum/solidity/releases/download/$SOLC_VERSION/solc-static-linux + +chmod 755 solc-static-linux +sudo mv solc-static-linux /usr/local/bin +sudo ln -s -f /usr/local/bin/solc-static-linux /usr/local/bin/solc + +if ! [ -x "$(command -v solc)" ]; then echo "Solidity installation failed"; exit 1; fi + +echo "Solidity has been installed successfully!" \ No newline at end of file diff --git a/deployments/local-setup-environment/provisioning/install-truffle.sh b/deployments/local-setup-environment/provisioning/install-truffle.sh new file mode 100755 index 0000000..1595ce0 --- /dev/null +++ b/deployments/local-setup-environment/provisioning/install-truffle.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +echo "Installing Truffle..." + +npm install -g truffle@5.0.30 + +if ! [ -x "$(command -v truffle)" ]; then echo "Truffle installation failed"; exit 1; fi + +echo "Truffle has been installed successfully!" \ No newline at end of file diff --git a/deployments/local-setup-instance/Vagrantfile b/deployments/local-setup-instance/Vagrantfile new file mode 100644 index 0000000..31ced34 --- /dev/null +++ b/deployments/local-setup-instance/Vagrantfile @@ -0,0 +1,29 @@ +def total_cpus + require 'etc' + Etc.nprocessors +end + +# Install vagrant-disksize to allow resizing the vagrant box disk. +unless Vagrant.has_plugin?("vagrant-disksize") + raise Vagrant::Errors::VagrantError.new, "vagrant-disksize plugin is missing. Please install it using 'vagrant plugin install vagrant-disksize' and rerun the command" +end + +Vagrant.configure("2") do |config| + config.vm.box = "local-setup-environment" + config.vm.synced_folder ".", "/vagrant", disabled: true + config.disksize.size = '15GB' + + config.vm.provider "virtualbox" do |v| + v.name = "local-setup-vm" + v.memory = "4096" + v.cpus = total_cpus - 2 + v.customize ["modifyvm", :id, "--vram", "256"] + end + + config.vm.provision "shell", path: "./provisioning/clone-repository.sh", privileged: false + config.vm.provision "shell", path: "./provisioning/initialize-geth.sh", privileged: false + config.vm.provision "shell", path: "./provisioning/run-geth.sh", privileged: false, run: "always" + config.vm.provision "shell", path: "./provisioning/run-bitcoin.sh", privileged: false, run: "always" + config.vm.provision "shell", path: "./provisioning/run-install.sh", privileged: false + config.vm.provision "shell", path: "./provisioning/run-clients.sh", privileged: false, run: "always" +end \ No newline at end of file diff --git a/deployments/local-setup-instance/provisioning/clone-repository.sh b/deployments/local-setup-instance/provisioning/clone-repository.sh new file mode 100755 index 0000000..e01f0ba --- /dev/null +++ b/deployments/local-setup-instance/provisioning/clone-repository.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +echo "Cloning repository..." + +if [ -d local-setup ]; then sudo rm -Rf local-setup; fi + +git clone https://github.com/keep-network/local-setup.git + +echo "Repository has been cloned successfully!" + diff --git a/deployments/local-setup-instance/provisioning/initialize-geth.sh b/deployments/local-setup-instance/provisioning/initialize-geth.sh new file mode 100755 index 0000000..b48a059 --- /dev/null +++ b/deployments/local-setup-instance/provisioning/initialize-geth.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +echo "Initializing Geth..." + +cd local-setup + +./initialize-geth.sh + +echo "Geth has been initialized successfully!" \ No newline at end of file diff --git a/deployments/local-setup-instance/provisioning/run-bitcoin.sh b/deployments/local-setup-instance/provisioning/run-bitcoin.sh new file mode 100755 index 0000000..0d3c934 --- /dev/null +++ b/deployments/local-setup-instance/provisioning/run-bitcoin.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +echo "Running Bitcoin Core and ElectrumX as transient system service..." + +cd local-setup + +WORKDIR=$(pwd) + +sudo systemd-run --unit=bitcoin -p WorkingDirectory=$WORKDIR ./run-bitcoin.sh + +echo "Bitcoin Core and ElectrumX have been run as transient system service successfully!" \ No newline at end of file diff --git a/deployments/local-setup-instance/provisioning/run-clients.sh b/deployments/local-setup-instance/provisioning/run-clients.sh new file mode 100755 index 0000000..9064fb7 --- /dev/null +++ b/deployments/local-setup-instance/provisioning/run-clients.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +echo "Running Core and ECDSA clients..." + +cd local-setup + +WORKDIR=$(pwd) + +sudo systemd-run --unit=keep-core-1 -p WorkingDirectory=$WORKDIR ./run-core-1.sh + +sudo systemd-run --unit=keep-ecdsa-1 -p WorkingDirectory=$WORKDIR ./run-ecdsa-1.sh +sudo systemd-run --unit=keep-ecdsa-2 -p WorkingDirectory=$WORKDIR ./run-ecdsa-2.sh +sudo systemd-run --unit=keep-ecdsa-3 -p WorkingDirectory=$WORKDIR ./run-ecdsa-3.sh + +echo "Core and ECDSA have been run successfully!" \ No newline at end of file diff --git a/deployments/local-setup-instance/provisioning/run-geth.sh b/deployments/local-setup-instance/provisioning/run-geth.sh new file mode 100755 index 0000000..6e02636 --- /dev/null +++ b/deployments/local-setup-instance/provisioning/run-geth.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +echo "Running Geth as transient system service..." + +cd local-setup + +WORKDIR=$(pwd) + +sudo systemd-run --unit=geth -p WorkingDirectory=$WORKDIR ./run-geth.sh + +echo "Geth has been run as transient system service successfully!" \ No newline at end of file diff --git a/deployments/local-setup-instance/provisioning/run-install.sh b/deployments/local-setup-instance/provisioning/run-install.sh new file mode 100755 index 0000000..0975a26 --- /dev/null +++ b/deployments/local-setup-instance/provisioning/run-install.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -e + +cd local-setup + +while true; do + echo "Checking Geth state..." + + ETH_BLOCK=$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":67}' http://localhost:8545 | jq -r '.result') + + if [ -z "$ETH_BLOCK" ] || [ "$ETH_BLOCK" == "0x0" ]; then + echo "Geth still not initialized - waiting..." + sleep 30 + continue; + fi + + echo "Geth initialized!" + + break +done + +while true; do + echo "Checking Bitcoin Core state..." + + BTC_BLOCK=$(curl -s --user user:password --data-binary '{"jsonrpc": "1.0", "id": "curltest", "method": "getblockcount", "params": []}' -H 'content-type: text/plain;' http://127.0.0.1:18332 | jq -r '.result') + + if [ -z "$BTC_BLOCK" ] || [ "$BTC_BLOCK" == "0" ]; then + echo "Bitcoin Core still not initialized - waiting..." + sleep 30 + continue; + fi + + echo "Bitcoin Core initialized!" + + break +done + +echo "Running install script..." + +./install.sh + +echo "Install script executed successfully!" + + + + diff --git a/docs/auto-setup.adoc b/docs/auto-setup.adoc new file mode 100644 index 0000000..e2711f1 --- /dev/null +++ b/docs/auto-setup.adoc @@ -0,0 +1,130 @@ += Automatic setup on virtual machine + +== Prerequisites + +You need to install: + +- https://www.packer.io/downloads[Packer] +- https://www.vagrantup.com/downloads[Vagrant] +- https://www.virtualbox.org/wiki/Downloads[VirtualBox] + +[TIP] +VirtualBox installation may fail on macOS Catalina. If it's your case, please look +at https://apple.stackexchange.com/questions/372492/virtualbox-installation-failed-on-macos-catalina[this issue]. + +== Installing the system + +First, go to the `deployments/local-setup-environment` directory and build the environment image: +``` +packer build -only=vagrant packerfile.json +``` +An output directory named `vagrant-box` containing `package.box` file should be created. + +Then, switch the directory to `deployments/local-setup-instance` and import +the image: +``` +vagrant box add --force --name=local-setup-environment ./../local-setup-environment/vagrant-box/package.box +``` +Also, install the required `vagrant-disksize` plugin: +``` +vagrant plugin install vagrant-disksize +``` +Finally, you can run the virtual machine by doing: +``` +vagrant up +``` +A new virtual machine will be created and provisioned accordingly. When +this command terminates successfully, the machine should be ready to work +and have all the auxiliary software like Geth, Bitcoin Core, ElectrumX +and Core/ECDSA clients running. Worth noting, `keep-core` and `keep-ecdsa` client +executables are always built from their recent master versions. + +== How to interact with the system + +You can interact with the system through the tBTC dApp or automated +end-to-end tests. Before you start interacting, you have to connect +to the virtual machine. First, switch to the `deployments/local-setup-instance` +directory and do: +``` +vagrant ssh +``` + +You will be moved to the virtual machine home directory where you can find +the `local-setup` directory where all installed things live. + +=== Keep Dashboard dApp + +To run the Keep Dashboard dApp invoke: +``` +./run-keep-dashboard.sh +``` + +=== tBTC dApp + +To run the tBTC dApp against the local Bitcoin network, +make sure the tBTC dApp is configured to work with the local ElectrumX instance. +You can configure this in `./tbtc-dapp/src/wrappers/web3.js` file, +where `TBTC.withConfig` line occurs. Change it as follows: +``` +const tbtc = await TBTC.withConfig({ + web3: web3, + bitcoinNetwork: "regtest", + electrum: { + testnetWS: { + server: "127.0.0.1", + port: 50003, + protocol: "ws" + } + } +}) +``` + +Then, you can invoke: +``` +./run-tbtc-dapp.sh +``` +The application will be available on `http://localhost:3000`. + +=== E2E tests + +To run the automated end-to-end scenario switch to Node 14.3.0: +``` +nvm use 14.3.0 +``` +Then invoke: +``` +./run-e2e-test.sh +``` + +== Virtual machine lifecycle + +You can manage the VM lifecycle by switching to the `deployments/local-setup-instance` +directory. + +To stop the machine do: +``` +vagrant halt +``` + +To start it again, just make: +``` +vagrant up +``` + +To restart the machine, invoke: +``` +vagrant reload +``` + +Everytime you start the machine, Geth, btcd, ElectrumX and Core/ECDSA clients +will be started as well but the configured `local-setup` directory won't be touched. +However, if you want to deploy everything from scratch +you can reload the machine with provisioning option enabled: +``` +vagrant reload --provision +``` +or, destroy the machine: +``` +vagrant destroy +``` +and run it again. diff --git a/docs/manual-setup.adoc b/docs/manual-setup.adoc new file mode 100644 index 0000000..37bb74f --- /dev/null +++ b/docs/manual-setup.adoc @@ -0,0 +1,166 @@ += Manual setup on local machine + +== Prerequisites +If you’re on macOS, install Homebrew and https://github.com/keep-network/keep-core/blob/master/scripts/macos-setup.sh[macos-setup.sh] script to setup all necessary tools. + +If you are not on macOS or you don't have Homebrew, you need to install: + +- Go compiler, at least 1.14 +- Geth (Ethereum client) 1.9.9-stable +- Solidity, at least 0.5.17 +- Truffle, at least 5.1.18 +- Protobuf compiler, at least 3.11.4 +- `protoc-gen-gogoslick` toolchain +- `jq` + +Regardless you use `macos-setup.sh` or not, you should also install: + +- https://docs.docker.com/get-docker[Docker] & https://docs.docker.com/compose/install/[Docker Compose] +- https://github.com/pyenv/pyenv[pyenv], at least 1.2.18 (optional - only if testnet relay is used) +- https://github.com/pypa/pipenv[pipenv], at least 2018.11.26 (optional - only if testnet relay is used) + +== Node.js version notice + +Please use Node `v11.15.0` for everything but E2E test script. +To run the test script, please use Node `v14.3.0`. + +We recommend using https://github.com/nvm-sh/nvm[nvm] to manage node versions easily. +You can install both versions by doing: +``` +nvm install 11.15.0 +nvm install 14.3.0 +``` +And use the specific version: +``` +nvm use 11.15.0 +nvm use 14.3.0 +``` + +== Installing the system +. Initialize local `geth` using: ++ +``` +./initialize-geth.sh +``` ++ +You can skip this step if your local geth is already initialized. This script clears all Ethereum client data, initilizes genesis block, and funds five accounts. All client data are kept in the working directory of this project so all the data used for other projects stay untouched. + +. Run local `geth` node using: ++ +``` +./run-geth.sh +``` +. Run local Bitcoin Core node and ElectrumX using: ++ +``` +./run-bitcoin.sh +``` +. Run Keep & tBTC installation script: ++ +``` +./install.sh +``` ++ +This script will fetch `keep-core`, `keep-ecdsa`, and `tbtc` source code, deploy contracts of `keep-core`, `keep-ecdsa`, and `tbtc`. It will also build `keep-core` and `keep-ecdsa` off-chain clients. ++ +Later on, if you decide to update `tbtc`, you can run just `install-tbtc.sh`. ++ +Keep in mind, `tbtc` depends on `keep-ecdsa` and `keep-ecdsa` depends on `keep-core` so if you decide to update `keep-ecdsa`, you can run `install-keep-ecdsa.sh` followed by `install-tbtc.sh`. ++ +If you decide to update `keep-core`, you have to run `install-keep-core.sh` followed by `install-keep-ecdsa.sh` followed by `install-tbtc.sh` or just the entire `install.sh` again. + +== Run clients +The above installation script will configure: + +- 1 `keep-core` client +- 3 `keep-ecdsa` clients + +To run the `keep-core` client use: +``` +./run-core-1.sh +``` + +It is enough to run one `keep-core` client to generate a group and produce relay entries. Setting up more than one client locally is possible but consumes more resources. + +To run `keep-ecdsa` clients use: +``` +./run-ecdsa-1.sh +``` +``` +./run-ecdsa-2.sh +``` +``` +./run-ecdsa-3.sh +``` + +There are at least 3 `keep-ecdsa` clients needed to open a keep. Setting up more than three clients locally is possible but consumes more resources. + +== Beacon genesis + +Before the beacon is able to produce a first relay entry, genesis needs to happen. Genesis triggers the first random beacon group selection. + +Genesis should be triggered after `keep-core` client started with: +``` +cd keep-core +KEEP_ETHEREUM_PASSWORD="password" ./keep-core --config configs/config.local.1.toml relay genesis +``` + +Bonded ECDSA keep factory from `keep-ecdsa` contracts requests for new relay entry to reseed after each signer selection but it is also possible to request for a new relay entry manually with: +``` +cd keep-core +KEEP_ETHEREUM_PASSWORD="password" ./keep-core --config configs/config.local.1.toml relay request +``` + +== How to interact with the system + +You can interact with the system through the tBTC dApp or automated +end-to-end tests. Before you start interacting, make sure you: + +- Installed all system components using `install.sh` script +- Have a local Geth instance (`run-geth.sh`) working +- Have local Bitcoin Core and ElectrumX instances (`run-bitcoin.sh`) working +- Have 1 `keep-core` and 3 `keep-ecdsa` clients up and running + +=== Keep Dashboard dApp + +To run the Keep Dashboard dApp invoke: +``` +./run-keep-dashboard.sh +``` + +=== tBTC dApp + +To run the tBTC dApp against the local Bitcoin network, +make sure the tBTC dApp is configured to work with the local ElectrumX instance. +You can configure this in `./tbtc-dapp/src/wrappers/web3.js` file, +where `TBTC.withConfig` line occurs. Change it as follows: +``` +const tbtc = await TBTC.withConfig({ + web3: web3, + bitcoinNetwork: "regtest", + electrum: { + testnetWS: { + server: "127.0.0.1", + port: 50003, + protocol: "ws" + } + } +}) +``` + +Then, you can invoke: +``` +./run-tbtc-dapp.sh +``` +The application will be available on `http://localhost:3000`. + +=== E2E tests + +To run the automated end-to-end scenario switch to Node 14.3.0: +``` +nvm use 14.3.0 +``` +Then invoke: +``` +./run-e2e-test.sh +``` diff --git a/install-keep-core.sh b/install-keep-core.sh index 422f80c..c740376 100755 --- a/install-keep-core.sh +++ b/install-keep-core.sh @@ -19,8 +19,9 @@ cp -R configs/keep-core/. keep-core/configs/ cd keep-core/configs # Fill absolute paths in config file with actual working directory. -sed -i .OLD 's:WORKDIR:'$WORKDIR':' config.local.1.toml -rm *.OLD +TMP_FILE=$(mktemp /tmp/config.local.1.toml.XXXXXXXXXX) +sed 's:WORKDIR:'$WORKDIR':' config.local.1.toml > $TMP_FILE +mv $TMP_FILE config.local.1.toml printf "${LOG_START}Creating storage directories...${LOG_END}" diff --git a/install-keep-ecdsa.sh b/install-keep-ecdsa.sh index 2cb62d2..5e4de46 100755 --- a/install-keep-ecdsa.sh +++ b/install-keep-ecdsa.sh @@ -19,10 +19,17 @@ cp -R configs/keep-ecdsa/. keep-ecdsa/configs/ cd keep-ecdsa/configs # Fill absolute paths in config files with actual working directory. -sed -i .OLD 's:WORKDIR:'$WORKDIR':' config.local.1.toml -sed -i .OLD 's:WORKDIR:'$WORKDIR':' config.local.2.toml -sed -i .OLD 's:WORKDIR:'$WORKDIR':' config.local.3.toml -rm *.OLD +TMP_FILE=$(mktemp /tmp/config.local.1.toml.XXXXXXXXXX) +sed 's:WORKDIR:'$WORKDIR':' config.local.1.toml > $TMP_FILE +mv $TMP_FILE config.local.1.toml + +TMP_FILE=$(mktemp /tmp/config.local.2.toml.XXXXXXXXXX) +sed 's:WORKDIR:'$WORKDIR':' config.local.2.toml > $TMP_FILE +mv $TMP_FILE config.local.2.toml + +TMP_FILE=$(mktemp /tmp/config.local.3.toml.XXXXXXXXXX) +sed 's:WORKDIR:'$WORKDIR':' config.local.3.toml > $TMP_FILE +mv $TMP_FILE config.local.3.toml printf "${LOG_START}Creating storage directories...${LOG_END}" @@ -37,8 +44,9 @@ printf "${LOG_START}Updating keep-ecdsa configuration...${LOG_END}" # Set correct Geth WS port. cd keep-ecdsa/solidity -sed -i "" 's/\port\:.*/\port\: '8546,'/g' truffle.js -sed -i "" 's/\websockets\:.*/\websockets\: 'true,'/g' truffle.js +TMP_FILE=$(mktemp /tmp/truffle.js.XXXXXXXXXX) +sed -e 's/\port\:.*/\port\: '8546,'/g;s/\websockets\:.*/\websockets\: 'true,'/g' truffle.js > $TMP_FILE +mv $TMP_FILE truffle.js cd .. printf "${LOG_START}Running install script...${LOG_END}" diff --git a/install-repositories.sh b/install-repositories.sh index a8679c2..2c7781e 100755 --- a/install-repositories.sh +++ b/install-repositories.sh @@ -7,5 +7,4 @@ LOG_END='\n\e[0m' # new line + reset color printf "${LOG_START}Initializing submodules...${LOG_END}" -git submodule init -git submodule update \ No newline at end of file +git submodule update --init --recursive --remote --rebase --force \ No newline at end of file diff --git a/install-testnet-relay.sh b/install-testnet-relay.sh index 2598167..e039589 100755 --- a/install-testnet-relay.sh +++ b/install-testnet-relay.sh @@ -32,8 +32,9 @@ cp -R configs/relays/. relays/maintainer/maintainer/config/ cd "$WORKDIR/relays/maintainer/maintainer/config" # Fill SUMMA_RELAY_CONTRACT env in config file with their actual value. -sed -i .OLD 's:RELAYCONTRACT:'$TESTNET_RELAY_CONTRACT_ADDRESS':' .my_env_file.env -rm .my_env_file.env.OLD +TMP_FILE=$(mktemp /tmp/.my_env_file.env.XXXXXXXXXX) +sed 's:RELAYCONTRACT:'$TESTNET_RELAY_CONTRACT_ADDRESS':' .my_env_file.env > $TMP_FILE +mv $TMP_FILE .my_env_file.env cd $WORKDIR diff --git a/install.sh b/install.sh index 7dd458d..68d2689 100755 --- a/install.sh +++ b/install.sh @@ -20,5 +20,4 @@ set -e # Install tBTC dApp. ./install-tbtc-dapp.sh -# Install relay-maintainer. -./install-testnet-relay.sh +echo "Installation script executed successfully with versions of submodules:\n$(git submodule)" \ No newline at end of file