support QA deploy

.env

@@ -5,8 +5,8 @@ KEYPASS="0123456789"
 INIT_HOLDER="0x04d63aBCd2b9b1baa327f2Dda0f873F197ccd186"
 INIT_HOLDER_PRV="59ba8068eb256d520179e903f43dacf6d8d57d72bd306e1bd603fdb8c8da10e8"
 RPC_URL="http://127.0.0.1:8545"
-PASSED_FORK_DELAY=40
-LAST_FORK_MORE_DELAY=10
+PASSED_FORK_DELAY=100
+LAST_FORK_MORE_DELAY=300
 FullImmutabilityThreshold=512
 MinBlocksForBlobRequests=576
 DefaultExtraReserveForBlobRequests=32

.gitignore

@@ -6,4 +6,6 @@
 
 .local/
 
-.idea/
+.idea/
+
+/qa-env-resource/machines_meta.sh

qa-env-resource/bsc.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=bsc
+After=network.target
+
+[Service]
+Type=simple
+User=root
+Group=root
+ExecStart=/server/validator/chaind.sh -start
+ExecReload=/server/validator/chaind.sh -restart
+ExecStop=/server/validator/chaind.sh -stop
+PrivateTmp=true
+Restart=always
+LimitNOFILE=10000
+RestartSec=5
+StartLimitInterval=0
+
+[Install]
+WantedBy=multi-user.target

qa-env-resource/chaind.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+export GOGC=200
+# default values
+FullImmutabilityThreshold=90000
+MinBlocksForBlobRequests=524288
+DefaultExtraReserveForBlobRequests=28800
+BreatheBlockInterval=600
+FixedTurnLength=1
+LAST_FORK_MORE_DELAY=1800
+
+function startChaind() {
+    workspace=/server/validator
+
+    PassedForkTime=`cat ${workspace}/hardforkTime.txt|grep passedHardforkTime|awk -F" " '{print $NF}'`
+    LastHardforkTime=$(expr ${PassedForkTime} + ${LAST_FORK_MORE_DELAY})
+    initLog=${workspace}/init.log
+    rialtoHash=`cat ${initLog}|grep "database=lightchaindata"|awk -F"=" '{print $NF}'|awk -F'"' '{print $1}'`
+
+    ip=`ifconfig eth0|grep inet|grep -v inet6 |awk '{ print $2 }'`
+    sed -i -e "s?FileRoot = \"\"?FileRoot = \"/mnt/efs/validator/${ip}/\"?g" /server/validator/config.toml
+    mkdir -p /mnt/efs/validator/${ip}
+    ${workspace}/bsc --config ${workspace}/config.toml \
+        --datadir ${workspace} \
+        --password ${workspace}/password.txt \
+        --blspassword ${workspace}/password.txt \
+        --unlock {{validatorAddr}} --miner.etherbase {{validatorAddr}} --rpc.allow-unprotected-txs --allow-insecure-unlock \
+        --ws --ws.port 8545 --ws.addr ${ip} --http.addr 0.0.0.0 --http.corsdomain "*" \
+        --metrics --metrics.addr 0.0.0.0 \
+        --pprof --pprof.port 6061 \
+        --syncmode snap --mine --vote --monitor.maliciousvote \
+        --cache 10480 --light.serve 50 \
+        --rialtohash ${rialtoHash} --override.passedforktime ${PassedForkTime} --override.bohr ${LastHardforkTime} \
+        --override.immutabilitythreshold ${FullImmutabilityThreshold} --override.breatheblockinterval ${BreatheBlockInterval} \
+        --override.minforblobrequest ${MinBlocksForBlobRequests} --override.defaultextrareserve ${DefaultExtraReserveForBlobRequests} \
+        `# --override.fixedturnlength ${FixedTurnLength}` \
+        >> /mnt/efs/validator/${ip}/bscnode.log 2>&1
+}
+
+function stopChaind() {
+    pid=`ps -ef | grep /server/validator/bsc | grep -v grep | awk '{print $2}'`
+    if [ -n "$pid" ]; then
+        for((i=1;i<=4;i++));
+        do
+            kill $pid
+            sleep 5
+            pid=`ps -ef | grep /server/validator/bsc | grep -v grep | awk '{print $2}'`
+            if [ -z "$pid" ]; then
+                break
+            elif [ $i -eq 4 ]; then
+                kill -9 $kid
+            fi
+        done
+    fi
+}
+
+CMD=$1
+
+case $CMD in
+-start)
+    echo "start"
+    startChaind
+    ;;
+-stop)
+    echo "stop"
+    stopChaind
+    ;;
+-restart)
+    stopChaind
+    sleep 3
+    startChaind
+    ;;
+*)
+    echo "Usage: chaind.sh -start | -stop | -restart .Or use systemctl start | stop | restart bsc.service "
+    ;;
+esac

qa-env-resource/config.toml

@@ -0,0 +1,65 @@
+[Eth]
+NetworkId = 714
+SyncMode = "full"
+NoPruning = false
+NoPrefetch = false
+LightPeers = 100
+TrieTimeout = 100000000000
+DatabaseCache = 512
+DatabaseFreezer = ""
+TriesInMemory = 128
+TrieCleanCache = 256
+TrieDirtyCache = 256
+EnablePreimageRecording = false
+
+[Eth.Miner]
+GasFloor = 500000000
+GasCeil = 500000000
+GasPrice = 1000000000
+Recommit = 10000000000
+
+[Eth.TxPool]
+Locals = []
+NoLocals = true
+Journal = "transactions.rlp"
+Rejournal = 3600000000000
+PriceLimit = 1000000000
+PriceBump = 10
+AccountSlots = 512
+GlobalSlots = 10000
+AccountQueue = 256
+GlobalQueue = 5000
+Lifetime = 10800000000000
+
+[Eth.GPO]
+Blocks = 20
+Percentile = 60
+OracleThreshold = 1000
+
+[Node]
+IPCPath = "geth.ipc"
+HTTPHost = "localhost"
+NoUSB = true
+InsecureUnlockAllowed = false
+HTTPPort = 8545
+HTTPVirtualHosts = ["localhost"]
+HTTPModules = ["eth", "net", "web3", "txpool", "parlia", "miner"]
+WSPort = 8546
+WSModules = ["net", "web3", "eth"]
+
+[Node.P2P]
+MaxPeers = 30
+NoDiscovery = false
+ListenAddr = ":30311"
+EnableMsgEvents = false
+
+[Node.HTTPTimeouts]
+ReadTimeout = 30000000000
+WriteTimeout = 30000000000
+IdleTimeout = 120000000000
+
+[Node.LogConfig]
+FilePath = "bsc.log"
+MaxBytesSize = 10485760
+Level = "info"
+FileRoot = ""

qa-env-resource/init.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+workspace=$(cd `dirname $0`; pwd)
+
+cp ${workspace}/bsc.service /usr/lib/systemd/system/
+chmod +x ${workspace}/start.sh ${workspace}/chaind.sh ${workspace}/bsc
+
+service bsc stop
+rm -rf /server/validator
+mv  ${workspace} /server/validator
+
+/server/validator/start.sh

qa-env-resource/start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+systemctl daemon-reload
+chkconfig bsc on
+service bsc restart

qa-env-resource/upgrade-single.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+service bsc stop
+cp /tmp/bsc /server/validator/bsc
+
+#cd  /server/validator/geth/
+#rm -rf  chaindata  les.server  nodes  triecache
+
+#sed -i -e 's/sleep 5/sleep 40/' /server/validator/chaind.sh
+#sed -i -e 's/FixedTurnLength=1/FixedTurnLength=4/' /server/validator/chaind.sh
+
+#/server/validator/bsc init --datadir /server/validator/ /server/validator/genesis.json
+
+#sed -i -e 's/HTTPModules = \[/HTTPModules = \["debug",/g' /server/validator/config.toml
+#sed -i -e 's/GlobalSlots = 10000/GlobalSlots = 10000/g' /server/validator/config.toml
+#sed -i -e 's/GlobalQueue = 5000/GlobalQueue = 5000/g' /server/validator/config.toml
+#sed -i -e 's/Level = "info"/Level = "debug"/g' /server/validator/config.toml
+
+service bsc start

start_cluster.sh

@@ -9,13 +9,14 @@ basedir=$(
 )
 workspace=${basedir}
 source ${workspace}/.env
-size=$((BSC_CLUSTER_SIZE))
+source ${workspace}/qa-env-resource/machines_meta.sh # including machine ips and ids, don't upload!!!
+size=${#ips2ids[@]}
 stateScheme="hash"
 dbEngine="leveldb"
 gcmode="full"
 epoch=200
 blockInterval=3
-needRegister=false
+needRegister=true
 sleepBeforeStart=10
 
 # stop geth client
@@ -47,14 +48,13 @@ function reset_genesis() {
         mv genesis-template.json.bk genesis-template.json
     fi
 
-    poetry install --no-root
+    # poetry install --no-root
     npm install
     rm -rf lib/forge-std
     forge install --no-git --no-commit foundry-rs/[email protected]
     cd lib/forge-std/lib
     rm -rf ds-test
     git clone https://github.com/dapphub/ds-test
-
 }
 
 function prepare_config() {
@@ -73,6 +73,8 @@ function prepare_config() {
         mkdir -p ${workspace}/.local/bsc/node${i}
         cp ${workspace}/keys/password.txt ${workspace}/.local/bsc/node${i}/
         cp ${workspace}/.local/bsc/hardforkTime.txt ${workspace}/.local/bsc/node${i}/
+        cp ${workspace}/qa-env-resource/* ${workspace}/.local/bsc/node${i}/
+        sed -i -e "s/{{validatorAddr}}/${cons_addr}/g" ${workspace}/.local/bsc/node${i}/chaind.sh
         bbcfee_addrs=${fee_addr}
         powers="0x000001d1a94a2000" #2000000000000
         mv ${workspace}/.local/bsc/bls${i}/bls ${workspace}/.local/bsc/node${i}/ && rm -rf ${workspace}/.local/bsc/bls${i}
@@ -89,10 +91,10 @@ function prepare_config() {
         sed -i -e '/registeredContractChannelMap\[VALIDATOR_CONTRACT_ADDR\]\[STAKING_CHANNELID\]/d' ${workspace}/genesis/contracts/CrossChain.sol
         sed -i -e  's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 30) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
     fi
-    poetry run python -m scripts.generate generate-validators
-    poetry run python -m scripts.generate generate-init-holders "${initHolders}"
+    python3 -m scripts.generate generate-validators
+    python3 -m scripts.generate generate-init-holders "${initHolders}"
     quorum=$(($BSC_CLUSTER_SIZE*2/3+1))
-    poetry run python -m scripts.generate dev --dev-chain-id ${BSC_CHAIN_ID} --whitelist-1 "${INIT_HOLDER}" \
+    python3 -m scripts.generate dev --dev-chain-id ${BSC_CHAIN_ID} --whitelist-1 "${INIT_HOLDER}" \
       --epoch ${epoch} \
       --init-felony-slash-scope "60" \
       --breathe-block-interval "10 minutes" \
@@ -117,13 +119,14 @@ function initNetwork() {
         mkdir ${workspace}/.local/bsc/node${i}/geth
         cp ${workspace}/keys/nodekey${i} ${workspace}/.local/bsc/node${i}/geth/nodekey
     done
-    ${workspace}/bin/geth init-network --init.dir ${workspace}/.local/bsc --init.size=${size} --config ${workspace}/config.toml ${workspace}/genesis/genesis.json
+    ${workspace}/bin/geth init-network --init.dir ${workspace}/.local/bsc --init.size=${size} --init.ips "${validator_ips_comma}" --config ${workspace}/qa-env-resource/config.toml ${workspace}/genesis/genesis.json
     rm -rf ${workspace}/*bsc.log*
     for ((i = 0; i < size; i++)); do
         sed -i -e '/"<nil>"/d' ${workspace}/.local/bsc/node${i}/config.toml
         mv ${workspace}/.local/bsc/validator${i}/keystore ${workspace}/.local/bsc/node${i}/ && rm -rf ${workspace}/.local/bsc/validator${i}
 
-        cp ${workspace}/bin/geth ${workspace}/.local/bsc/node${i}/geth${i}
+        #cp ${workspace}/bin/geth ${workspace}/.local/bsc/node${i}/geth${i}
+        cp ${workspace}/bin/geth ${workspace}/.local/bsc/node${i}/bsc
         # init genesis
         initLog=${workspace}/.local/bsc/node${i}/init.log
         if  [ $i -eq 0 ] ; then
@@ -181,15 +184,34 @@ function native_start() {
 
 function register_stakehub(){
     if ${needRegister};then
-        echo "sleep 45s to wait feynman enable"
-        sleep 45
+        echo "sleep 100s to wait feynman enable"
+        sleep 100
         for ((i = 0; i < size; i++));do
             ${workspace}/create-validator/create-validator --consensus-key-dir ${workspace}/keys/validator${i} --vote-key-dir ${workspace}/keys/bls${i} \
                 --password-path ${workspace}/keys/password.txt --amount 20001 --validator-desc Val${i} --rpc-url ${RPC_URL}
         done
     fi
 }
 
+function remote_start() {
+    rm -rf /mnt/efs/bsc-qa/clusterNetwork
+    cp -r ${workspace}/.local/bsc /mnt/efs/bsc-qa/clusterNetwork
+    ips=(${validator_ips_comma//,/ })
+    for ((i=0;i<${#ips[@]};i++));do
+        dst_id=${ips2ids[${ips[i]}]}
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo bash -x /mnt/efs/bsc-qa/clusterNetwork/node${i}/init.sh"
+    done
+}
+
+function remote_upgrade() {
+    cp ${workspace}/bin/geth /mnt/efs/bsc-qa/clusterNetwork/
+    cp ${workspace}/qa-env-resource/upgrade-single.sh /mnt/efs/bsc-qa/clusterNetwork/
+    for dst_id in ${ips2ids[@]}; do
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
+            --parameters commands="sudo cp /mnt/efs/bsc-qa/clusterNetwork/geth /tmp/bsc && sudo cp /mnt/efs/bsc-qa/clusterNetwork/upgrade-single.sh /tmp/ && sudo bash -x /tmp/upgrade-single.sh"
+    done
+}
+
 ## docker relate begin
 function generate_static_peers() {
     tool=${workspace}/bin/bootnode
@@ -298,6 +320,17 @@ restart)
     exit_previous $ValidatorIdx
     native_start $ValidatorIdx
     ;;
+remote_reset)
+    create_validator
+    reset_genesis
+    prepare_config
+    initNetwork
+    remote_start
+    register_stakehub
+    ;;
+remote_upgrade)
+    remote_upgrade
+    ;;
 install_k8s)
     create_validator
     reset_genesis
@@ -310,6 +343,6 @@ uninstall_k8s)
     uninstall_k8s
     ;;
 *)
-    echo "Usage: start_cluster.sh | reset | stop | start | restart"
+    echo "Usage: start_cluster.sh | reset | stop | start | restart | remote_reset | remote_upgrade"
     ;;
 esac