diff --git a/.github/workflows/mithril-latest.yml b/.github/workflows/mithril-latest.yml index 3fd578b8c..2997de9cb 100644 --- a/.github/workflows/mithril-latest.yml +++ b/.github/workflows/mithril-latest.yml @@ -7,7 +7,7 @@ jobs: get-version: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: token: ${{ secrets.REPO_SCOPED_TOKEN }} fetch-depth: 0 @@ -18,13 +18,19 @@ jobs: - name: Assigns release version run: | VERSION=$(cat ./files/docker/node/release-versions/mithril-latest.txt) + - name: Source mithril.library and check upgrade safety + run: | + set -e + . scripts/cnode-helper-scripts/mithril.library workflow + check_mithril_upgrade_safe + echo "MITHRIL_UPGRADE_SAFE=$MITHRIL_UPGRADE_SAFE" >> $GITHUB_ENV - name: Check for modified files id: git-check run: echo ::set-output name=modified::$([ -z "`git status --porcelain`" ] && echo "false" || echo "true") - name: Commit latest release version - if: steps.git-check.outputs.modified == 'true' + if: steps.git-check.outputs.modified == 'true' && env.MITHRIL_UPGRADE_SAFE == 'Y' run: | git config --global user.name ${{ secrets.REPO_SCOPED_USER }} git config --global user.email ${{ secrets.REPO_SCOPED_EMAIL }} git commit -am "New mithril release version ${VERSION}" - git push + git push \ No newline at end of file diff --git a/scripts/cnode-helper-scripts/env b/scripts/cnode-helper-scripts/env index 71bd41ba6..0054212de 100644 --- a/scripts/cnode-helper-scripts/env +++ b/scripts/cnode-helper-scripts/env @@ -110,6 +110,7 @@ #CIP0094_POLL_URL="https://raw.githubusercontent.com/cardano-foundation/CIP-0094-polls/main/networks/polls.json" # URL for polls to vote against #MITHRIL_DOWNLOAD="N" # (Y|N) Download latest Mithril snapshot +#MITHRIL_HOME="${CNODE_HOME}/mithril" # Override default Mithril path #MITHRIL_SIGNER_ENABLED="N" # (Y|N) Enable the gLiveView Mithril Signer Section #STRICT_VERSION_CHECK="Y" # (Y|N) Restrict operation to supported major.minor version leaving patch version open. If disabled, any version will be accepted (unsupported) @@ -950,6 +951,7 @@ set_default_vars() { [[ -z ${ASSET_POLICY_ID_FILENAME} ]] && ASSET_POLICY_ID_FILENAME="policy.id" [[ -z ${CIP0094_POLL_URL} ]] && CIP0094_POLL_URL="https://raw.githubusercontent.com/cardano-foundation/CIP-0094-polls/main/networks/polls.json" [[ -z ${MITHRIL_DOWNLOAD} ]] && MITHRIL_DOWNLOAD="N" + [[ -z ${MITHRIL_HOME} ]] && MITHRIL_HOME="${CNODE_HOME}/mithril" [[ -z ${MITHRIL_SIGNER_ENABLED} ]] && MITHRIL_SIGNER_ENABLED="N" [[ -z ${STRICT_VERSION_CHECK} ]] && STRICT_VERSION_CHECK="Y" FG_BLACK='\e[30m' diff --git a/scripts/cnode-helper-scripts/gLiveView.sh b/scripts/cnode-helper-scripts/gLiveView.sh index 1be56c7a7..4dd080332 100755 --- a/scripts/cnode-helper-scripts/gLiveView.sh +++ b/scripts/cnode-helper-scripts/gLiveView.sh @@ -778,8 +778,8 @@ unset cpu_now cpu_last mithrilSignerVars() { # mithril.env sourcing needed to have values in ${METRICS_SERVER_IP} and ${METRICS_SERVER_PORT} - . ${CNODE_HOME}/mithril/mithril.env - signerMetricsEnabled=$(grep -q "ENABLE_METRICS_SERVER=true" ${CNODE_HOME}/mithril/mithril.env && echo "true" || echo "false") + . ${MITHRIL_HOME}/mithril.env + signerMetricsEnabled=$(grep -q "ENABLE_METRICS_SERVER=true" ${MITHRIL_HOME}/mithril.env && echo "true" || echo "false") if [[ "${signerMetricsEnabled}" == "true" ]] ; then mithrilSignerMetrics=$(curl -s "http://${METRICS_SERVER_IP}:${METRICS_SERVER_PORT}/metrics" 2>/dev/null | grep -v -E "HELP|TYPE" | sed 's/mithril_signer_//g') SIGNER_METRICS_HTTP_RESPONSE=$(curl --write-out "%{http_code}" --silent --output /dev/null --connect-timeout 2 http://${METRICS_SERVER_IP}:${METRICS_SERVER_PORT}/metrics) diff --git a/scripts/cnode-helper-scripts/guild-deploy.sh b/scripts/cnode-helper-scripts/guild-deploy.sh index f1a19839f..38ad843f3 100755 --- a/scripts/cnode-helper-scripts/guild-deploy.sh +++ b/scripts/cnode-helper-scripts/guild-deploy.sh @@ -112,6 +112,7 @@ set_defaults() { [[ -z "${CARDANO_NODE_VERSION}" ]] && CARDANO_NODE_VERSION="$(curl -sfk "https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/${BRANCH}/files/docker/node/release-versions/cardano-node-latest.txt")" CNODE_HOME="${CNODE_PATH}/${CNODE_NAME}" CNODE_VNAME=$(echo "$CNODE_NAME" | awk '{print toupper($0)}') + [[ -z ${MITHRIL_HOME} ]] && MITHRIL_HOME="${CNODE_HOME}/mithril" REPO="https://github.com/${G_ACCOUNT}/guild-operators" REPO_RAW="https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators" URL_RAW="${REPO_RAW}/${BRANCH}" @@ -588,7 +589,7 @@ setup_folder() { echo -e "\nexport ${CNODE_VNAME}_HOME=${CNODE_HOME}" >> "${HOME}"/.bashrc fi - $sudo mkdir -p "${CNODE_HOME}"/files "${CNODE_HOME}"/db "${CNODE_HOME}"/guild-db "${CNODE_HOME}"/logs "${CNODE_HOME}"/scripts "${CNODE_HOME}"/scripts/archive "${CNODE_HOME}"/sockets "${CNODE_HOME}"/priv "${CNODE_HOME}"/mithril/data-stores + $sudo mkdir -p "${CNODE_HOME}"/files "${CNODE_HOME}"/db "${CNODE_HOME}"/guild-db "${CNODE_HOME}"/logs "${CNODE_HOME}"/scripts "${CNODE_HOME}"/scripts/archive "${CNODE_HOME}"/sockets "${CNODE_HOME}"/priv "${MITHRIL_HOME}"/data-stores $sudo chown -R "$U_ID":"$G_ID" "${CNODE_HOME}" 2>/dev/null } diff --git a/scripts/cnode-helper-scripts/mithril-client.sh b/scripts/cnode-helper-scripts/mithril-client.sh index e2f5ff8cf..48c100938 100755 --- a/scripts/cnode-helper-scripts/mithril-client.sh +++ b/scripts/cnode-helper-scripts/mithril-client.sh @@ -48,216 +48,100 @@ EOF SKIP_UPDATE=N [[ $1 = "-u" ]] && export SKIP_UPDATE=Y && shift -## mithril environment subcommands - -environment_override() { - local var_to_override="$1" - local new_value="$2" - local env_file="${CNODE_HOME}/mithril/mithril.env" - - # Check if the variable exists in the environment file - if ! grep -q "^${var_to_override}=" "$env_file"; then - echo "Error: Variable $var_to_override does not exist in $env_file" >&2 - return 1 - fi - - # Use sed to replace the variable's value in the environment file - sed -i "s|^${var_to_override}=.*|${var_to_override}=${new_value}|" "$env_file" -} - -mithril_init() { - [[ ! -f "${CNODE_HOME}"/mithril/mithril.env ]] && generate_environment_file - . "${CNODE_HOME}"/mithril/mithril.env -} - - -check_db_dir() { - # If the DB directory does not exist then set DOWNLOAD_SNAPSHOT to Y - if [[ ! -d "${DB_DIRECTORY}" ]]; then - echo "INFO: The db directory does not exist.." - DOWNLOAD_SNAPSHOT="Y" - # If the DB directory is empty then set DOWNLOAD_SNAPSHOT to Y - elif [[ -d "${DB_DIRECTORY}" ]] && [[ -z "$(ls -A "${DB_DIRECTORY}")" ]] && [[ $(du -cs "${DB_DIRECTORY}"/* 2>/dev/null | awk '/total$/ {print $1}') -eq 0 ]]; then - echo "INFO: The db directory is empty.." - DOWNLOAD_SNAPSHOT="Y" - else - echo "INFO: The db directory is not empty, skipping Cardano DB download.." - fi -} - -cleanup_db_directory() { - echo "WARNING: Download failure, cleaning up DB directory.." - # Safety check to prevent accidental deletion of system files - if [[ -z "${DB_DIRECTORY}" ]]; then - echo "ERROR: DB_DIRECTORY is unset or null." - elif [[ -n "${DB_DIRECTORY}" && "${DB_DIRECTORY}" != "/" && "${DB_DIRECTORY}" != "${CNODE_HOME}" ]]; then - # :? Safety check to prevent accidental deletion of system files, even though initial if block should already prevent this - rm -rf "${DB_DIRECTORY:?}/"* - else - echo "INFO: Skipping cleanup of DB directory: ${DB_DIRECTORY}." - fi -} - -## mithril snapshot subcommands - -download_snapshot() { - if [[ "${DOWNLOAD_SNAPSHOT}" == "Y" ]]; then - echo "INFO: Downloading latest mithril snapshot.." - trap 'cleanup_db_directory' INT - if ! "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} cardano-db download --download-dir $(dirname ${DB_DIRECTORY}) --genesis-verification-key ${GENESIS_VERIFICATION_KEY} ${SNAPSHOT_DIGEST} ; then - cleanup_db_directory - exit 1 - fi - else - echo "INFO: Skipping Cardano DB download.." - fi -} - -list_snapshots() { - local json_flag="" - - for arg in "$@"; do - if [[ $arg == "json" ]]; then - json_flag="--json" - fi - done - - "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} cardano-db snapshot list $json_flag -} - -show_snapshot() { - local digest="" - local json_flag="" - - for arg in "$@"; do - if [[ $arg == "json" ]]; then - json_flag="--json" - else - digest="$arg" - fi - done - - if [[ -z $digest ]]; then - echo "ERROR: Snapshot digest is required for the 'show' subcommand" >&2 - exit 1 - fi - - "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} cardano-db snapshot show $digest $json_flag -} - -## mithril-stake-distribution subcommands - -download_stake_distribution() { - if [[ "${DOWNLOAD_STAKE_DISTRIBUTION}" == "Y" ]]; then - echo "INFO: Downloading latest mithril stake distribution.." - "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} mithril-stake-distribution download --download-dir "${CNODE_HOME}/mithril/" ${STAKE_DISTRIBUTION_DIGEST} - else - echo "INFO: Skipping stake distribution download.." - fi -} - -list_stake_distributions() { - local json_flag="" - - for arg in "$@"; do - if [[ $arg == "json" ]]; then - json_flag="--json" - fi - done - - "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} mithril-stake-distribution list $json_flag - -} ##################### # Execution/Main # ##################### -. "$(dirname $0)"/mithril.library - -update_check "$@" - -set_defaults +function main() { + . "$(dirname $0)"/mithril.library + + update_check "$@" + + set_defaults + + # Parse command line options + case $1 in + environment) + case $2 in + setup) + generate_environment_file + ;; + override) + environment_override $3 $4 + ;; + update) + export UPDATE_ENVIRONMENT="Y" + generate_environment_file + ;; + *) + echo "Invalid environment subcommand: $2" >&2 + usage + exit 1 + ;; + esac + ;; + cardano-db) + mithril_init client + case $2 in + download) + check_db_dir + download_snapshot + ;; + snapshot) + case $3 in + list) + case $4 in + json) + list_snapshots json + ;; + *) + list_snapshots + ;; + esac + ;; + show) + show_snapshot $4 $5 + ;; + *) + echo "Invalid snapshot subcommand: $3" >&2 + usage + exit 1 + ;; + esac + esac + ;; + stake-distribution) + mithril_init client + case $2 in + download) + download_stake_distribution + ;; + list) + case $3 in + json) + list_stake_distributions json + ;; + *) + list_stake_distributions + ;; + esac + ;; + *) + echo "Invalid mithril-stake-distribution subcommand: $2" >&2 + usage + exit 1 + ;; + esac + ;; + *) + echo "Invalid $(basename "$0") command: $1" >&2 + usage + exit 1 + ;; + esac -# Parse command line options -case $1 in - environment) - case $2 in - setup) - generate_environment_file - ;; - override) - environment_override $3 $4 - ;; - update) - export UPDATE_ENVIRONMENT="Y" - generate_environment_file - ;; - *) - echo "Invalid environment subcommand: $2" >&2 - usage - exit 1 - ;; - esac - ;; - cardano-db) - mithril_init - case $2 in - download) - check_db_dir - download_snapshot - ;; - snapshot) - case $3 in - list) - case $4 in - json) - list_snapshots json - ;; - *) - list_snapshots - ;; - esac - ;; - show) - show_snapshot $4 $5 - ;; - *) - echo "Invalid snapshot subcommand: $3" >&2 - usage - exit 1 - ;; - esac - esac - ;; - stake-distribution) - mithril_init - case $2 in - download) - download_stake_distribution - ;; - list) - case $3 in - json) - list_stake_distributions json - ;; - *) - list_stake_distributions - ;; - esac - ;; - *) - echo "Invalid mithril-stake-distribution subcommand: $2" >&2 - usage - exit 1 - ;; - esac - ;; - *) - echo "Invalid $(basename "$0") command: $1" >&2 - usage - exit 1 - ;; -esac + exit 0 +} -exit 0 +main "$@" diff --git a/scripts/cnode-helper-scripts/mithril-relay.sh b/scripts/cnode-helper-scripts/mithril-relay.sh index f64c998d1..06650200d 100755 --- a/scripts/cnode-helper-scripts/mithril-relay.sh +++ b/scripts/cnode-helper-scripts/mithril-relay.sh @@ -17,6 +17,7 @@ RELAY_LISTENING_PORT=3132 # Constants # ##################### +ADDITIONAL_ALLOWED_IP=() BLOCK_PRODUCER_IP=() RELAY_LISTENING_IP=() @@ -40,228 +41,66 @@ usage() { } -generate_nginx_conf() { - sudo bash -c "cat > /etc/nginx/nginx.conf <<'EOF' -worker_processes 1; - -events { - worker_connections 1024; -} - -http { - upstream mithril_relays { - $(for ip in "${RELAY_LISTENING_IP[@]}"; do - echo -e " server ${ip}:${RELAY_LISTENING_PORT};" - done) - } - - server { - listen ${SIDECAR_LISTENING_IP}:${RELAY_LISTENING_PORT}; - location / { - proxy_pass http://mithril_relays; - proxy_set_header Host \$host; - proxy_set_header X-Real-IP \$remote_addr; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - } - } -} -EOF" -} - -generate_squid_conf() { - # Write the squid config file - sudo bash -c "cat <<-'EOF' > /etc/squid/squid.conf - # Listening port (port 3132 is recommended) - http_port ${RELAY_LISTENING_PORT} - - EOF" - - # Write the ACLs for each relay IP address - sudo bash -c 'echo "# ACLs for IP of the block producers" >> /etc/squid/squid.conf' - for ip in "${BLOCK_PRODUCER_IP[@]}"; do - sudo bash -c "echo \"acl block_producer_ip src ${ip}\" >> /etc/squid/squid.conf" - done - - # Write the rest of the squid config file - sudo bash -c "cat <<-'EOF' >> /etc/squid/squid.conf - # ACL for aggregator endpoint - acl aggregator_domain dstdomain .mithril.network - - # ACL for SSL port only - acl SSL_port port 443 - - # Allowed traffic - http_access allow block_producer_ip aggregator_domain SSL_port - - # Do not disclose relay internal IP - forwarded_for delete - - # Turn off via header - via off - - # Deny request for original source of a request - follow_x_forwarded_for deny all - - # Anonymize request headers - request_header_access Authorization allow all - request_header_access Proxy-Authorization allow all - request_header_access Cache-Control allow all - request_header_access Content-Length allow all - request_header_access Content-Type allow all - request_header_access Date allow all - request_header_access Host allow all - request_header_access If-Modified-Since allow all - request_header_access Pragma allow all - request_header_access Accept allow all - request_header_access Accept-Charset allow all - request_header_access Accept-Encoding allow all - request_header_access Accept-Language allow all - request_header_access Connection allow all - request_header_access All deny all - - # Disable cache - cache deny all - - # Deny everything else - http_access deny all - EOF" -} - -deploy_nginx_load_balancer() { - # Install nginx and configure load balancing - echo -e "\nInstalling nginx load balancer" - sudo apt-get update - sudo apt-get install -y nginx - - # Read the listening IP addresses from user input - while true; do - read -r -p "Enter the IP address of a relay: " ip - RELAY_LISTENING_IP+=("${ip}") - read -r -p "Are there more relays? (y/n) " yn - case ${yn} in - [Nn]*) break ;; - *) continue ;; - esac - done - - # Read the listening IP for the load balancer - read -r -p "Enter the IP address of the load balancer (press Enter to use default 127.0.0.1): " SIDECAR_LISTENING_IP - SIDECAR_LISTENING_IP=${SIDECAR_LISTENING_IP:-127.0.0.1} - echo "Using IP address ${SIDECAR_LISTENING_IP} for the load balancer configuration." - - # Read the listening port from user input - read -r -p "Enter the relay's listening port (press Enter to use default 3132): " RELAY_LISTENING_PORT - RELAY_LISTENING_PORT=${RELAY_LISTENING_PORT:-3132} - echo "Using port ${RELAY_LISTENING_PORT} for relay's listening port." - - # Generate the nginx configuration file - generate_nginx_conf - # Restart nginx and check status - echo -e "\nStarting Mithril relay sidecar (nginx load balancer)" - sudo systemctl restart nginx - sudo systemctl status nginx - -} - -deploy_squid_proxy() { - # Install squid and make a backup of the config file - echo -e "\nInstalling squid proxy" - sudo apt-get update - sudo apt-get install -y squid - sudo cp /etc/squid/squid.conf /etc/squid/squid.conf.bak - - # Read the listening IP addresses from user input - while true; do - read -r -p "Enter the IP address of your Block Producer: " ip - BLOCK_PRODUCER_IP+=("${ip}") - read -r -p "Are there more block producers? (y/n) " yn - case ${yn} in - [Nn]*) break ;; - *) continue ;; - esac - done - - # Read the listening port from user input - read -r -p "Enter the relay's listening port (press Enter to use default 3132): " RELAY_LISTENING_PORT - RELAY_LISTENING_PORT=${RELAY_LISTENING_PORT:-3132} - echo "Using port ${RELAY_LISTENING_PORT} for relay's listening port." - generate_squid_conf - - # Restart squid and check status - echo -e "\nStarting Mithril relay (squid proxy)" - sudo systemctl restart squid - sudo systemctl status squid - - # Inform the user to create the appropriate firewall rule - for ip in "${RELAY_LISTENING_IP[@]}"; do - echo "Create the appropriate firewall rule: sudo ufw allow from ${ip} to any port ${RELAY_LISTENING_PORT} proto tcp" - done -} - -stop_relays() { - echo " Stopping squid proxy and nginx load balancers.." - sudo systemctl stop squid 2>/dev/null - sudo systemctl stop nginx 2>/dev/null - sleep 5 - exit 0 -} - ##################### # Execution/Main # ##################### -# Parse command line arguments -while getopts :dlsuh opt; do - case ${opt} in - d) - INSTALL_SQUID_PROXY=Y - ;; - l) - INSTALL_NGINX_LOAD_BALANCER=Y - ;; - u) - export SKIP_UPDATE='Y' - ;; - s) - STOP_RELAYS=Y - ;; - h) - usage - exit 0 - ;; - \?) - echo "Invalid option: -${OPTARG}" >&2 - usage - exit 1 - ;; - :) - echo "Option -${OPTARG} requires an argument." >&2 - usage - exit 1 - ;; - *) - usage - exit 1 - ;; - esac -done +function main() { + # Parse command line arguments + while getopts :dlsuh opt; do + case ${opt} in + d) + INSTALL_SQUID_PROXY=Y + ;; + l) + INSTALL_NGINX_LOAD_BALANCER=Y + ;; + u) + export SKIP_UPDATE='Y' + ;; + s) + STOP_RELAYS=Y + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -${OPTARG}" >&2 + usage + exit 1 + ;; + :) + echo "Option -${OPTARG} requires an argument." >&2 + usage + exit 1 + ;; + *) + usage + exit 1 + ;; + esac + done + + # Display usage menu if no flags are provided + if [[ ${OPTIND} -eq 1 ]]; then + usage + exit 1 + fi -# Display usage menu if no flags are provided -if [[ ${OPTIND} -eq 1 ]]; then - usage - exit 1 -fi + . "$(dirname $0)"/mithril.library -. "$(dirname $0)"/mithril.library + [[ "${STOP_RELAYS}" == "Y" ]] && stop_relays -[[ "${STOP_RELAYS}" == "Y" ]] && stop_relays + update_check "$@" -update_check "$@" + if [[ ${INSTALL_SQUID_PROXY} = Y ]]; then + deploy_squid_proxy + fi -if [[ ${INSTALL_SQUID_PROXY} = Y ]]; then - deploy_squid_proxy -fi + if [[ ${INSTALL_NGINX_LOAD_BALANCER} = Y ]]; then + deploy_nginx_load_balancer + fi +} -if [[ ${INSTALL_NGINX_LOAD_BALANCER} = Y ]]; then - deploy_nginx_load_balancer -fi +main "$@" \ No newline at end of file diff --git a/scripts/cnode-helper-scripts/mithril-signer.sh b/scripts/cnode-helper-scripts/mithril-signer.sh index 2c0de636b..ef1631bf0 100755 --- a/scripts/cnode-helper-scripts/mithril-signer.sh +++ b/scripts/cnode-helper-scripts/mithril-signer.sh @@ -36,206 +36,108 @@ usage() { EOF } -mithril_init() { - [[ ! -f "${CNODE_HOME}"/mithril/mithril.env ]] && generate_environment_file - for line in $(cat "${CNODE_HOME}"/mithril/mithril.env); do - export "${line}" - done - # Move logs to archive - [[ -d "${LOG_DIR}"/archive ]] || mkdir -p "${LOG_DIR}"/archive - [[ -f "${LOG_DIR}"/$(basename "${0::-3}").log ]] && mv "${LOG_DIR}/$(basename "${0::-3}")".log "${LOG_DIR}"/archive/ ; touch "${LOG_DIR}/$(basename "${0::-3}")".log -} - -deploy_systemd() { - echo "Creating ${CNODE_VNAME}-$(basename "${0::-3}") systemd service environment file.." - - echo "Deploying ${CNODE_VNAME}-$(basename "${0::-3}") as systemd service.." - sudo bash -c "cat <<-'EOF' > /etc/systemd/system/${CNODE_VNAME}-$(basename "${0::-3}").service - [Unit] - Description=Cardano Mithril signer service - StartLimitIntervalSec=0 - Wants=network-online.target - After=network-online.target - BindsTo=${CNODE_VNAME}.service - After=${CNODE_VNAME}.service - - [Service] - Type=simple - Restart=always - RestartSec=60 - User=${USER} - EnvironmentFile=${CNODE_HOME}/mithril/mithril.env - ExecStart=/bin/bash -l -c \"exec ${HOME}/.local/bin/$(basename "${0::-3}") -vv\" - KillSignal=SIGINT - SuccessExitStatus=143 - SyslogIdentifier=${CNODE_VNAME}-$(basename "${0::-3}") - TimeoutStopSec=5 - KillMode=mixed - - [Install] - WantedBy=multi-user.target - EOF" && echo "${CNODE_VNAME}-$(basename "${0::-3}").service deployed successfully!!" && sudo systemctl daemon-reload && sudo systemctl enable ${CNODE_VNAME}-"$(basename "${0::-3}")".service -} - -stop_signer() { - CNODE_PID=$(pgrep -fn "$(basename ${CNODEBIN}).*.--port ${CNODE_PORT}" 2>/dev/null) # env was only called in offline mode - kill -2 ${CNODE_PID} 2>/dev/null - # touch clean "${DB_DIRECTORY}"/clean # Disabled as it's a bit hacky, but only runs when SIGINT is passed to node process. Should not be needed if node does it's job - echo " Sending SIGINT to $(basename "${0::-3}") process.." - sleep 5 - exit 0 -} - - -user_interrupt_received() { - echo " SIGINT received, stopping $(basename "${0::-3}").." |tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 - stop_signer - -} - -verify_signer_registration() { - set -e - - if [ -z "$AGGREGATOR_ENDPOINT" ] || [ -z "$PARTY_ID" ]; then - echo ">> ERROR: Required environment variables AGGREGATOR_ENDPOINT and/or PARTY_ID are not set." - exit 1 - fi - - CURRENT_EPOCH=$(curl -s "$AGGREGATOR_ENDPOINT/epoch-settings" -H 'accept: application/json' | jq -r '.epoch') - SIGNERS_REGISTERED_RESPONSE=$(curl -s "$AGGREGATOR_ENDPOINT/signers/registered/$CURRENT_EPOCH" -H 'accept: application/json') - - if echo "$SIGNERS_REGISTERED_RESPONSE" | grep -q "$PARTY_ID"; then - echo ">> Congrats, your signer node is registered!" - else - echo ">> Oops, your signer node is not registered. Party ID not found among the signers registered at epoch ${CURRENT_EPOCH}." - fi - -} - -verify_signer_signature() { - set -e - - if [ -z "$AGGREGATOR_ENDPOINT" ] || [ -z "$PARTY_ID" ]; then - echo ">> ERROR: Required environment variables AGGREGATOR_ENDPOINT and/or PARTY_ID are not set." - exit 1 - fi - - CERTIFICATES_RESPONSE=$(curl -s "$AGGREGATOR_ENDPOINT/certificates" -H 'accept: application/json') - CERTIFICATES_COUNT=$(echo "$CERTIFICATES_RESPONSE" | jq '. | length') - - echo "$CERTIFICATES_RESPONSE" | jq -r '.[] | .hash' | while read -r HASH; do - RESPONSE=$(curl -s "$AGGREGATOR_ENDPOINT/certificate/$HASH" -H 'accept: application/json') - SIGNER_COUNT=$(echo "$RESPONSE" | jq '.metadata.signers | length') - for (( i=0; i < SIGNER_COUNT; i++ )); do - PARTY_ID_RESPONSE=$(echo "$RESPONSE" | jq -r ".metadata.signers[$i].party_id") - if [[ "$PARTY_ID_RESPONSE" == "$PARTY_ID" ]]; then - echo ">> Congrats, you have signed this certificate: $AGGREGATOR_ENDPOINT/certificate/$HASH !" - exit 1 - fi - done - done - - echo ">> Oops, your party id was not found in the last ${CERTIFICATES_COUNT} certificates. Please try again later." - -} - ##################### # Execution / Main # ##################### -# Parse command line options -while getopts :dDekrsuh opt; do - case ${opt} in - d ) - DEPLOY_SYSTEMD="Y" ;; - D ) - SIGNER_DAEMON="Y" - ;; - e ) - export UPDATE_ENVIRONMENT="Y" - ;; - k ) - STOP_SIGNER="Y" - ;; - r ) - VERIFY_REGISTRATION="Y" - ;; - s ) - VERIFY_SIGNATURE="Y" - ;; - u ) - export SKIP_UPDATE="Y" - ;; - h) - usage - exit 0 - ;; - \?) - echo "Invalid $(basename "$0") option: -${OPTARG}" >&2 - usage - exit 1 - ;; - :) - echo "Option -${OPTARG} requires an argument." >&2 - usage - exit 1 - ;; - esac -done +function main() { + # Parse command line options + while getopts :dDekrsuh opt; do + case ${opt} in + d ) + DEPLOY_SYSTEMD="Y" ;; + D ) + SIGNER_DAEMON="Y" + ;; + e ) + export UPDATE_ENVIRONMENT="Y" + ;; + k ) + STOP_SIGNER="Y" + ;; + r ) + VERIFY_REGISTRATION="Y" + ;; + s ) + VERIFY_SIGNATURE="Y" + ;; + u ) + export SKIP_UPDATE="Y" + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid $(basename "$0") option: -${OPTARG}" >&2 + usage + exit 1 + ;; + :) + echo "Option -${OPTARG} requires an argument." >&2 + usage + exit 1 + ;; + esac + done -. "$(dirname $0)"/mithril.library + . "$(dirname $0)"/mithril.library -[[ "${STOP_SIGNER}" == "Y" ]] && stop_signer + [[ "${STOP_SIGNER}" == "Y" ]] && stop_signer -# Check for updates -update_check "$@" + # Check for updates + update_check "$@" -# Set defaults and do basic sanity checks -set_defaults + # Set defaults and do basic sanity checks + set_defaults -#Deploy systemd if -d argument was specified -if [[ "${UPDATE_ENVIRONMENT}" == "Y" ]]; then - generate_environment_file - exit 0 -else - mithril_init - if [[ "${DEPLOY_SYSTEMD}" == "Y" ]]; then - if deploy_systemd ; then - echo "Mithril signer Systemd service successfully deployed" - exit 0 - else - echo "Failed to deploy Mithril signer Systemd service" - exit 2 - fi - elif [[ "${VERIFY_REGISTRATION}" == "Y" ]]; then - # Verify signer registration - echo "Verifying Mithril Signer registration.." - verify_signer_registration - exit 0 - elif [[ "${VERIFY_SIGNATURE}" == "Y" ]]; then - # Verify signer signature - echo "Verifying Mithril Signer signature.." - verify_signer_signature + #Deploy systemd if -d argument was specified + if [[ "${UPDATE_ENVIRONMENT}" == "Y" ]]; then + generate_environment_file exit 0 - elif [[ "${SIGNER_DAEMON}" == "Y" ]]; then - # Run Mithril Signer Server - echo "Starting Mithril Signer Server.." - trap 'user_interrupt_received' INT - - if grep -q "ENABLE_METRICS_SERVER=true" ${CNODE_HOME}/mithril/mithril.env; then - METRICS_SERVER_PARAMS="--enable-metrics-server --metrics-server-ip ${METRICS_SERVER_IP} --metrics-server-port ${METRICS_SERVER_PORT}" - # If ENABLE_METRICS_SERVER is true, then an environment update will enable gLiveView automatically. - sudo sed -i 's/#MITHRIL_SIGNER_ENABLED="[YN]"/MITHRIL_SIGNER_ENABLED="Y"/' ${CNODE_HOME}/scripts/env - if ! "${MITHRILBIN}" ${METRICS_SERVER_PARAMS} -vv | tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 ; then - echo "Failed to start Mithril Signer Server with metrics enabled" | tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 - exit 1 + else + mithril_init + if [[ "${DEPLOY_SYSTEMD}" == "Y" ]]; then + if deploy_systemd ; then + echo "Mithril signer Systemd service successfully deployed" + exit 0 + else + echo "Failed to deploy Mithril signer Systemd service" + exit 2 fi - else - if ! "${MITHRILBIN}" -vv | tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 ; then - echo "Failed to start Mithril Signer Server" | tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 - exit 1 + elif [[ "${VERIFY_REGISTRATION}" == "Y" ]]; then + # Verify signer registration + echo "Verifying Mithril Signer registration.." + verify_signer_registration + exit 0 + elif [[ "${VERIFY_SIGNATURE}" == "Y" ]]; then + # Verify signer signature + echo "Verifying Mithril Signer signature.." + verify_signer_signature + exit 0 + elif [[ "${SIGNER_DAEMON}" == "Y" ]]; then + # Run Mithril Signer Server + echo "Starting Mithril Signer Server.." + trap 'user_interrupt_received' INT + + if grep -q "ENABLE_METRICS_SERVER=true" ${MITHRIL_HOME}/mithril.env; then + METRICS_SERVER_PARAMS="--enable-metrics-server --metrics-server-ip ${METRICS_SERVER_IP} --metrics-server-port ${METRICS_SERVER_PORT}" + # If ENABLE_METRICS_SERVER is true, then an environment update will enable gLiveView automatically. + # shellcheck disable=SC2154 + sudo sed -i 's/#MITHRIL_SIGNER_ENABLED="[YN]"/MITHRIL_SIGNER_ENABLED="Y"/' ${CNODE_HOME}/scripts/env + if ! "${MITHRILBIN}" ${METRICS_SERVER_PARAMS} -vv | tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 ; then + echo "Failed to start Mithril Signer Server with metrics enabled" | tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 + exit 1 + fi + else + if ! "${MITHRILBIN}" -vv | tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 ; then + echo "Failed to start Mithril Signer Server" | tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 + exit 1 + fi fi fi fi -fi +} + +main "$@" diff --git a/scripts/cnode-helper-scripts/mithril.library b/scripts/cnode-helper-scripts/mithril.library index b64444060..fa8bca425 100644 --- a/scripts/cnode-helper-scripts/mithril.library +++ b/scripts/cnode-helper-scripts/mithril.library @@ -1,17 +1,50 @@ #!/usr/bin/env bash # shellcheck disable=SC2034,SC2086,SC2230,SC2206,SC2140,SC2059,SC2154 -#shellcheck source=/dev/null +# shellcheck source=/dev/null ###################################### # Do NOT modify code below # ###################################### -. "$(dirname $0)"/env offline +CI_MODE='N' +[[ $1 = "workflow" ]] && CI_MODE='Y' + +# Sourcing mithril.library in CI mode is for GitHub Actions workflow to safely +# update the files/docker/node/release-versions/mithril-latest.txt file without +# setting it to an incompatible version with the currently suported cardano-node +# version. There is no need to source the environment file in CI mode. +if [[ ${CI_MODE} == 'N' ]] ; then + . "$(dirname $0)"/env +fi + +############################# +# Mithril General functions # +############################# U_ID=$(id -u) G_ID=$(id -g) MITHRILBIN="${HOME}"/.local/bin/$(basename "${0::-3}") +# Make setup inside containers easier where SUDO may not exist +[[ -z ${SUDO} ]] && SUDO='Y' +[[ "${SUDO}" = 'Y' ]] && sudo="sudo" || sudo="" +[[ "${SUDO}" = 'Y' && $(id -u) -eq 0 ]] && err_exit "Please run as non-root user." + +check_mithril_environment_file_exists() { + local env_file="${MITHRIL_HOME}/mithril.env" + + if [[ -f "$env_file" ]]; then + if [[ "$UPDATE_ENVIRONMENT" != "Y" ]]; then + echo "Error: $env_file already exists. To update it, set UPDATE_ENVIRONMENT to 'Y'." >&2 + return 1 + else + echo "Updating $env_file..." + fi + else + echo "Creating $env_file..." + fi +} + compare_versions() { local min_version=$1 local test_version=$2 @@ -22,14 +55,181 @@ compare_versions() { fi } +component_environment_setup() { + check_mithril_environment_file_exists + + if [[ -n "${POOL_NAME}" ]] && [[ "${POOL_NAME}" != "CHANGE_ME" ]] && [[ "$(basename "$0")" == "mithril-signer.sh" ]]; then + update_mithril_environment_for_signer + else + update_mithril_environment_for_client + fi +} + +create_data_stores_directory() { + if [[ ! -d "${MITHRIL_HOME}/data-stores" ]]; then + ${sudo} mkdir -p "${MITHRIL_HOME}"/data-stores + ${sudo} chown -R "$U_ID":"$G_ID" "${MITHRIL_HOME}" 2>/dev/null + fi +} + +generate_environment_file() { + create_data_stores_directory + component_environment_setup + set_env_file_ownership +} + +# Function to initialize the mithril environment +mithril_init() { + if [[ "$1" == "client" ]]; then + mithril_client_init + elif [[ "$1" == "signer" ]]; then + mithril_signer_init + else + echo "Invalid argument. Please use 'client' or 'signer'." + return 1 + fi +} + +# Function to read IP addresses into an array with a customizable prompt and confirmation message +read_ips_from_input() { + local -n array_ref=$1 # Use nameref to reference the array passed by name + local prompt_message=$2 # Prompt message for IP input + local confirm_message=$3 # Confirmation message to ask if there are more IP addresses + + while true; do + read -r -p "$prompt_message" ip + array_ref+=("${ip}") + read -r -p "$confirm_message" yn + case ${yn} in + [Nn]*) break ;; + *) continue ;; + esac + done +} + +# Function to read optional IP addresses into an array with customizable messages +read_optional_ips_from_input() { + # shellcheck disable=SC2178 + local -n array_ref=$1 # Use nameref to reference the array passed by name + local confirm_message=$2 # Confirmation message to ask if there are IP addresses to add + local prompt_message=$3 # Prompt message for IP input if the user wants to add more IP addresses + + while true; do + read -r -p "$confirm_message" yn + case ${yn} in + [Nn]*) break ;; + *) read -r -p "$prompt_message" ip + array_ref+=("${ip}") + ;; + esac + done +} + +semantic_version_compare () { + if [[ "${1}" == "${2}" ]] + then + echo 0 + return + fi + local IFS=. + local i semantic_version1=($1) semantic_version2=($2) + # fill empty fields in semantic_version1 with zeros + for ((i=${#semantic_version1[@]}; i<${#semantic_version2[@]}; i++)) + do + semantic_version1[i]=0 + done + for ((i=0; i<${#semantic_version1[@]}; i++)) + do + if ((10#${semantic_version1[i]:=0} > 10#${semantic_version2[i]:=0})) + then + echo 1 + return + fi + if ((10#${semantic_version1[i]} < 10#${semantic_version2[i]})) + then + echo 2 + return + fi + done + echo 0 + return +} + +set_defaults() { + [[ -z "${MITHRILBIN}" ]] && MITHRILBIN="${HOME}"/.local/bin/"$(basename "${0::-3}")" + if [[ $(basename "${0::-3}") == "mithril-signer" ]] && { [[ -z "${POOL_NAME}" ]] || [[ "${POOL_NAME}" == "CHANGE_ME" ]]; }; then + echo "ERROR: The POOL_NAME must be set before deploying mithril-signer as a systemd service!!" + exit 1 + else + case "${NETWORK_NAME,,}" in + mainnet|preprod|guild) + RELEASE="release" + ;; + preview) + RELEASE="pre-release" + ;; + sanchonet) + RELEASE="testing" + ;; + *) + echo "ERROR: The NETWORK_NAME must be set to mainnet, preprod, preview, or sanchonet before $(basename "${0::-3}") can be deployed!!" + exit 1 + esac + fi + AGGREGATOR_ENDPOINT=https://aggregator.${RELEASE}-${NETWORK_NAME,,}.api.mithril.network/aggregator + GENESIS_VERIFICATION_KEY=$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/${RELEASE}-${NETWORK_NAME,,}/genesis.vkey) +} + +set_env_file_ownership() { + ${sudo} chown $USER:$USER "${MITHRIL_HOME}"/mithril.env +} + +check_mithril_upgrade_safe() { + MITHRIL_LATEST_VERSION=$(curl -s https://raw.githubusercontent.com/cardano-community/guild-operators/alpha/files/docker/node/release-versions/mithril-latest.txt) + if [[ "${CI_MODE}" == "Y" ]]; then + # When run in CI mode, the node version is obtained from the repository + NODE_CURRENT_VERSION=$(cat files/docker/node/release-versions/cardano-node-latest.txt) + else + # When run in non workflow mode, the node version is obtained from the cardano-node binary + NODE_CURRENT_VERSION=$(cardano-node --version | awk 'NR==1{print $2}') + fi + set_node_minimum_version + if [[ -n "${MITHRIL_MINIMUM_NODE_VERSION}" ]]; then + + RC=$(semantic_version_compare "${NODE_CURRENT_VERSION}" "${MITHRIL_MINIMUM_NODE_VERSION}") + if [[ ${RC} -lt 2 ]]; then + # Node version is greater than or equal to the minimum required version for latest mithril release + # Set MITHRIL_UPGRADE_SAFE to Y to allow mithril upgrade by scripts + MITHRIL_UPGRADE_SAFE="Y" + echo "INFO: A mithril upgrade is safe." + echo "INFO: The latest mithril release version: ${MITHRIL_LATEST_VERSION}." + echo "INFO: The current Node version: ${NODE_CURRENT_VERSION}." + echo "INFO: The Mithril minimum required node version: ${MITHRIL_MINIMUM_NODE_VERSION}." + else + # Node version is less than the minimum required version for latest mithril release + echo "WARNING: A mithril upgrade is not safe." + echo "WARNING: The latest mithril release version: ${MITHRIL_LATEST_VERSION}." + echo "WARNING: The current Node version: ${NODE_CURRENT_VERSION}." + echo "WARNING: The Mithril minimum required node version: ${MITHRIL_MINIMUM_NODE_VERSION}." + echo "WARNING: The latest mithril release does not support the installed node version. Please upgrade the node version first." + MITHRIL_UPGRADE_SAFE="N" + fi + else + echo "ERROR: Failed to set the minimum required node version for the latest mithril release. Setting MITHRIL_UPGRADE_SAFE to N." + MITHRIL_UPGRADE_SAFE="N" + fi +} + set_node_minimum_version() { response_file=$(mktemp) status_code=$(curl -s -o "$response_file" -w "%{http_code}" https://raw.githubusercontent.com/input-output-hk/mithril/${MITHRIL_LATEST_VERSION}/networks.json) - if [[ "$status_code" -ge 400 ]]; then - NODE_MINIMUM_VERSION="" - else - NODE_MINIMUM_VERSION=$(jq -r ".${NETWORK_NAME,,}.\"cardano-minimum-version\".\"mithril-signer\"" "$response_file") + if [[ "${status_code}" -gt 200 ]]; then + echo "ERROR: Failed to download the networks.json file from the mithril repository! curl status code: ${status_code}." + elif [[ "${status_code}" -eq 200 ]]; then + NETWORK=${NETWORK_NAME,,} + NETWORK=${NETWORK:-mainnet} + MITHRIL_MINIMUM_NODE_VERSION=$(jq -r ".${NETWORK}.\"cardano-minimum-version\".\"mithril-signer\"" "$response_file") fi rm -f "$response_file" } @@ -69,67 +269,380 @@ update_check() { } -set_defaults() { - MITHRIL_LATEST_VERSION=$(curl -s https://raw.githubusercontent.com/cardano-community/guild-operators/alpha/files/docker/node/release-versions/mithril-latest.txt) - set_node_minimum_version - NODE_CURRENT_VERSION=$(cardano-node --version | awk 'NR==1{print $2}') +update_mithril_environment_for_client() { + echo "Info: Setting minimal environment variables supporting only the Mithril client use case." + ${sudo} bash -c "cat <<-'EOF' > ${MITHRIL_HOME}/mithril.env + NETWORK=${NETWORK_NAME,,} + RELEASE=${RELEASE} + AGGREGATOR_ENDPOINT=https://aggregator.${RELEASE}-${NETWORK_NAME,,}.api.mithril.network/aggregator + DB_DIRECTORY=${DB_DIR} + GENESIS_VERIFICATION_KEY=$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/${RELEASE}-${NETWORK_NAME,,}/genesis.vkey) + SNAPSHOT_DIGEST=latest + EOF" +} - [[ -z "${MITHRILBIN}" ]] && MITHRILBIN="${HOME}"/.local/bin/"$(basename "${0::-3}")" - if [[ $(basename "${0::-3}") == "mithril-signer" ]] && { [[ -z "${POOL_NAME}" ]] || [[ "${POOL_NAME}" == "CHANGE_ME" ]]; }; then - echo "ERROR: The POOL_NAME must be set before deploying mithril-signer as a systemd service!!" - exit 1 +############################ +# Mithril Client functions # +############################ + + +check_db_dir() { + # If the DB directory does not exist then set DOWNLOAD_SNAPSHOT to Y + if [[ ! -d "${DB_DIRECTORY}" ]]; then + echo "INFO: The db directory does not exist.." + DOWNLOAD_SNAPSHOT="Y" + # If the DB directory is empty then set DOWNLOAD_SNAPSHOT to Y + elif [[ -d "${DB_DIRECTORY}" ]] && [[ -z "$(ls -A "${DB_DIRECTORY}")" ]] && [[ $(du -cs "${DB_DIRECTORY}"/* 2>/dev/null | awk '/total$/ {print $1}') -eq 0 ]]; then + echo "INFO: The db directory is empty.." + DOWNLOAD_SNAPSHOT="Y" else - case "${NETWORK_NAME,,}" in - mainnet|preprod|guild) - RELEASE="release" - ;; - preview) - RELEASE="pre-release" - ;; - sanchonet) - RELEASE="testing" - ;; - *) - echo "ERROR: The NETWORK_NAME must be set to mainnet, preprod, preview, or sanchonet before $(basename "${0::-3}") can be deployed!!" + echo "INFO: The db directory is not empty, skipping Cardano DB download.." + fi +} + +cleanup_db_directory() { + echo "WARNING: Download failure, cleaning up DB directory.." + # Safety check to prevent accidental deletion of system files + if [[ -z "${DB_DIRECTORY}" ]]; then + echo "ERROR: DB_DIRECTORY is unset or null." + elif [[ -n "${DB_DIRECTORY}" && "${DB_DIRECTORY}" != "/" && "${DB_DIRECTORY}" != "${MITHRIL_HOME}" ]]; then + # :? Safety check to prevent accidental deletion of system files, even though initial if block should already prevent this + rm -rf "${DB_DIRECTORY:?}/"* + else + echo "INFO: Skipping cleanup of DB directory: ${DB_DIRECTORY}." + fi +} + +# mithril client snapshot subcommand +download_snapshot() { + if [[ "${DOWNLOAD_SNAPSHOT}" == "Y" ]]; then + echo "INFO: Downloading latest mithril snapshot.." + trap 'cleanup_db_directory' INT + if ! "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} cardano-db download --download-dir "$(dirname ${DB_DIRECTORY})" --genesis-verification-key ${GENESIS_VERIFICATION_KEY} ${SNAPSHOT_DIGEST} ; then + cleanup_db_directory exit 1 - esac + fi + else + echo "INFO: Skipping Cardano DB download.." fi - AGGREGATOR_ENDPOINT=https://aggregator.${RELEASE}-${NETWORK_NAME,,}.api.mithril.network/aggregator - GENESIS_VERIFICATION_KEY=$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/${RELEASE}-${NETWORK_NAME,,}/genesis.vkey) } -create_data_stores_directory() { - if [[ ! -d "${CNODE_HOME}/mithril/data-stores" ]]; then - sudo mkdir -p "${CNODE_HOME}"/mithril/data-stores - sudo chown -R "$U_ID":"$G_ID" "${CNODE_HOME}"/mithril 2>/dev/null - fi +# mithril client mithril-stake-distribution subcommand +download_stake_distribution() { + if [[ "${DOWNLOAD_STAKE_DISTRIBUTION}" == "Y" ]]; then + echo "INFO: Downloading latest mithril stake distribution.." + "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} mithril-stake-distribution download --download-dir "${MITHRIL_HOME}/" ${STAKE_DISTRIBUTION_DIGEST} + else + echo "INFO: Skipping stake distribution download.." + fi } -set_env_file_ownership() { - chown $USER:$USER "${CNODE_HOME}"/mithril/mithril.env +# mithril client environment subcommand +environment_override() { + local var_to_override="$1" + local new_value="$2" + local env_file="${MITHRIL_HOME}/mithril.env" + + # Check if the variable exists in the environment file + if ! grep -q "^${var_to_override}=" "$env_file"; then + echo "Error: Variable $var_to_override does not exist in $env_file" >&2 + return 1 + fi + + # Use sed to replace the variable's value in the environment file + sed -i "s|^${var_to_override}=.*|${var_to_override}=${new_value}|" "$env_file" } +mithril_client_init() { + [[ ! -f "${MITHRIL_HOME}"/mithril.env ]] && generate_environment_file + . "${MITHRIL_HOME}"/mithril.env +} -check_mithril_environment_file_exists() { - local env_file="${CNODE_HOME}/mithril/mithril.env" +# mithril client snapshot subcommand +list_snapshots() { + local json_flag="" - if [[ -f "$env_file" ]]; then - if [[ "$UPDATE_ENVIRONMENT" != "Y" ]]; then - echo "Error: $env_file already exists. To update it, set UPDATE_ENVIRONMENT to 'Y'." >&2 - return 1 + for arg in "$@"; do + if [[ $arg == "json" ]]; then + json_flag="--json" + fi + done + + "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} cardano-db snapshot list $json_flag +} + +# mithril client mithril-stake-distribution subcommand +list_stake_distributions() { + local json_flag="" + + for arg in "$@"; do + if [[ $arg == "json" ]]; then + json_flag="--json" + fi + done + + "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} mithril-stake-distribution list $json_flag + +} + +# mithril client snapshot subcommand +show_snapshot() { + local digest="" + local json_flag="" + + for arg in "$@"; do + if [[ $arg == "json" ]]; then + json_flag="--json" else - echo "Updating $env_file..." + digest="$arg" fi - else - echo "Creating $env_file..." + done + + if [[ -z $digest ]]; then + echo "ERROR: Snapshot digest is required for the 'show' subcommand" >&2 + exit 1 fi + + "${MITHRILBIN}" -v --aggregator-endpoint ${AGGREGATOR_ENDPOINT} cardano-db snapshot show $digest $json_flag } -get_relay_endpoint() { - read -r -p "Enter the IP address of the relay endpoint: " RELAY_ENDPOINT_IP - read -r -p "Enter the port of the relay endpoint (press Enter to use default 3132): " RELAY_PORT - RELAY_PORT=${RELAY_PORT:-3132} - echo "Using RELAY_ENDPOINT=http://${RELAY_ENDPOINT_IP}:${RELAY_PORT} for the Mithril signer relay endpoint." +########################### +# Mithril Relay functions # +########################### + +deploy_nginx_load_balancer() { + # Install nginx and configure load balancing + echo -e "\nInstalling nginx load balancer" + ${sudo} apt-get update + ${sudo} apt-get install -y nginx + + # Read the listening IP addresses from user input + read_ips_from_input RELAY_LISTENING_IP \ + "Enter the IP address of a relay: " \ + "Are there more relays? (y/n) " + + # Read the listening IP for the load balancer + read -r -p "Enter the IP address of the load balancer (press Enter to use default 127.0.0.1): " SIDECAR_LISTENING_IP + SIDECAR_LISTENING_IP=${SIDECAR_LISTENING_IP:-127.0.0.1} + echo "Using IP address ${SIDECAR_LISTENING_IP} for the load balancer configuration." + + # Read the listening port from user input + read -r -p "Enter the relay's listening port (press Enter to use default 3132): " RELAY_LISTENING_PORT + RELAY_LISTENING_PORT=${RELAY_LISTENING_PORT:-3132} + echo "Using port ${RELAY_LISTENING_PORT} for relay's listening port." + + # Generate the nginx configuration file + generate_nginx_conf + # Restart nginx and check status + echo -e "\nStarting Mithril relay sidecar (nginx load balancer)" + ${sudo} systemctl restart nginx + ${sudo} systemctl status nginx + +} + +deploy_squid_proxy() { + # Install squid and make a backup of the config file + echo -e "\nInstalling squid proxy" + ${sudo} apt-get update + ${sudo} apt-get install -y squid + ${sudo} cp /etc/squid/squid.conf /etc/squid/squid.conf.bak + + # Read the block producer IP addresses from user input + read_ips_from_input BLOCK_PRODUCER_IP \ + "Enter the IP address of your Block Producer: " \ + "Are there more block producers? (y/n) " + + # Read any additional IP addresses from user input + read_optional_ips_from_input ADDITIONAL_ALLOWED_IP \ + "Are there more IP addresses you would like to allow like the local relay IP (to be used for testing, etc.)? (y/n) " \ + "Enter the IP address you would like to allow: " + + # Read the listening port from user input + while true; do + read -r -p "Enter the relay's listening port (press Enter to use default 3132): " RELAY_LISTENING_PORT + RELAY_LISTENING_PORT=${RELAY_LISTENING_PORT:-3132} + + # Check if the input is a valid integer + if ! [[ "$RELAY_LISTENING_PORT" =~ ^[0-9]+$ ]]; then + echo "Invalid input. Please enter a numeric port number." + continue + fi + + if [[ "${RELAY_LISTENING_PORT}" -lt 1024 || "${RELAY_LISTENING_PORT}" -gt 65535 ]]; then + echo "Invalid port number. Please enter a port number between 1024 and 65535." + else + break + fi + done + echo "Using port ${RELAY_LISTENING_PORT} for relay's listening port." + generate_squid_conf + + # Restart squid and check status + echo -e "\nStarting Mithril relay (squid proxy)" + ${sudo} systemctl restart squid + ${sudo} systemctl status squid + + # Inform the user to create the appropriate firewall rule + for ip in "${RELAY_LISTENING_IP[@]}"; do + echo "Create the appropriate firewall rule: sudo ufw allow from ${ip} to any port ${RELAY_LISTENING_PORT} proto tcp" + done +} + +generate_nginx_conf() { + ${sudo} bash -c "cat > /etc/nginx/nginx.conf <<'EOF' +worker_processes 1; + +events { + worker_connections 1024; +} + +stream { + upstream mithril_relays { + $(for ip in "${RELAY_LISTENING_IP[@]}"; do + echo -e " server ${ip}:${RELAY_LISTENING_PORT} max_fails=1 fail_timeout=${#RELAY_LISTENING_IP[@]}0;" + done) + } + + server { + listen ${SIDECAR_LISTENING_IP}:${RELAY_LISTENING_PORT}; + proxy_connect_timeout 10; + proxy_pass mithril_relays; + } +} +EOF" +} + +generate_squid_conf() { + # Write the squid config file + ${sudo} bash -c "cat <<-'EOF' > /etc/squid/squid.conf + # Listening port (port 3132 is recommended) + http_port ${RELAY_LISTENING_PORT} + + # ACL for aggregator endpoint + acl aggregator_domain dstdomain .mithril.network + + # ACL for SSL port only + acl SSL_port port 443 + + EOF" + + # Write the ACLs for block producer IP addresses + ${sudo} bash -c 'echo "# ACL alias for IP of the block producers" >> /etc/squid/squid.conf' + int=0 + for ip in "${BLOCK_PRODUCER_IP[@]}"; do + ((int++)) + ${sudo} bash -c "echo \"acl block_producer_ip${int} src ${ip}\" >> /etc/squid/squid.conf" + done + ${sudo} bash -c 'echo "" >> /etc/squid/squid.conf' + unset int + + # Write the ACLs for any additional allowed IP addresses + if [ ${#ADDITIONAL_ALLOWED_IP[@]} -gt 0 ]; then + ${sudo} bash -c 'echo "# ACL alias for any additional IPs" >> /etc/squid/squid.conf' + int=0 + for ip in "${ADDITIONAL_ALLOWED_IP[@]}"; do + ((int++)) + ${sudo} bash -c "echo \"acl additional_allowed_ip${int} src ${ip}\" >> /etc/squid/squid.conf" + done + ${sudo} bash -c 'echo "" >> /etc/squid/squid.conf' + unset int + fi + + # Write the allow rules + ${sudo} bash -c 'echo "# Allowed traffic" >> /etc/squid/squid.conf' + int=0 + for ip in "${BLOCK_PRODUCER_IP[@]}"; do + ((int++)) + ${sudo} bash -c "echo \"http_access allow block_producer_ip${int} aggregator_domain SSL_port\" >> /etc/squid/squid.conf" + done + int=0 + for ip in "${ADDITIONAL_ALLOWED_IP[@]}"; do + ((int++)) + ${sudo} bash -c "echo \"http_access allow additional_allowed_ip${int} aggregator_domain SSL_port\" >> /etc/squid/squid.conf" + done + unset int + + # Write the fix chunk of the squid config file + ${sudo} bash -c "cat <<-'EOF' >> /etc/squid/squid.conf + + # Do not disclose relay internal IP + forwarded_for delete + + # Turn off via header + via off + + # Deny request for original source of a request + follow_x_forwarded_for deny all + + # Anonymize request headers + request_header_access Authorization allow all + request_header_access Proxy-Authorization allow all + request_header_access Cache-Control allow all + request_header_access Content-Length allow all + request_header_access Content-Type allow all + request_header_access Date allow all + request_header_access Host allow all + request_header_access If-Modified-Since allow all + request_header_access Pragma allow all + request_header_access Accept allow all + request_header_access Accept-Charset allow all + request_header_access Accept-Encoding allow all + request_header_access Accept-Language allow all + request_header_access Connection allow all + request_header_access All deny all + + # Disable cache + cache deny all + + # Deny everything else + http_access deny all + EOF" +} + +stop_relays() { + echo " Stopping squid proxy and nginx load balancers.." + ${sudo} systemctl stop squid 2>/dev/null + ${sudo} systemctl stop nginx 2>/dev/null + sleep 5 + exit 0 +} + + +############################ +# Mithril Signer functions # +############################ + +deploy_systemd() { + echo "Creating ${CNODE_VNAME}-$(basename "${0::-3}") systemd service environment file.." + + echo "Deploying ${CNODE_VNAME}-$(basename "${0::-3}") as systemd service.." + ${sudo} bash -c "cat <<-'EOF' > /etc/systemd/system/${CNODE_VNAME}-$(basename "${0::-3}").service + [Unit] + Description=Cardano Mithril signer service + StartLimitIntervalSec=0 + Wants=network-online.target + After=network-online.target + BindsTo=${CNODE_VNAME}.service + After=${CNODE_VNAME}.service + + [Service] + Type=simple + Restart=always + RestartSec=60 + User=${USER} + EnvironmentFile=${MITHRIL_HOME}/mithril.env + ExecStart=/bin/bash -l -c \"exec ${HOME}/.local/bin/$(basename "${0::-3}") -vv\" + KillSignal=SIGINT + SuccessExitStatus=143 + StandardOutput=syslog + StandardError=syslog + SyslogIdentifier=${CNODE_VNAME}-$(basename "${0::-3}") + TimeoutStopSec=5 + KillMode=mixed + + [Install] + WantedBy=multi-user.target + EOF" && echo "${CNODE_VNAME}-$(basename "${0::-3}").service deployed successfully!!" && ${sudo} systemctl daemon-reload && ${sudo} systemctl enable ${CNODE_VNAME}-"$(basename "${0::-3}")".service } get_metrics_endpoint() { @@ -140,6 +653,31 @@ get_metrics_endpoint() { echo "Using ${METRICS_SERVER_IP}:${METRICS_SERVER_PORT} for the Mithril signer metrics endpoint." } +get_relay_endpoint() { + read -r -p "Enter the IP address of the relay endpoint: " RELAY_ENDPOINT_IP + read -r -p "Enter the port of the relay endpoint (press Enter to use default 3132): " RELAY_PORT + RELAY_PORT=${RELAY_PORT:-3132} + echo "Using RELAY_ENDPOINT=http://${RELAY_ENDPOINT_IP}:${RELAY_PORT} for the Mithril signer relay endpoint." +} + +mithril_signer_init() { + [[ ! -f "${MITHRIL_HOME}"/mithril.env ]] && generate_environment_file + for line in $(cat "${MITHRIL_HOME}"/mithril.env) ; do + export "${line?}" + done + # Move logs to archive + [[ -d "${LOG_DIR}"/archive ]] || mkdir -p "${LOG_DIR}"/archive + [[ -f "${LOG_DIR}"/$(basename "${0::-3}").log ]] && mv "${LOG_DIR}/$(basename "${0::-3}")".log "${LOG_DIR}"/archive/ ; touch "${LOG_DIR}/$(basename "${0::-3}")".log +} + +stop_signer() { + SIGNER_PID=$(pgrep -fn "$(basename "${0::-3}").*" 2>/dev/null) # env was only called in offline mode + kill -2 ${SIGNER_PID} 2>/dev/null + echo " Sending SIGINT to $(basename "${0::-3}") process.." + sleep 5 + exit 0 +} + update_mithril_environment_for_signer() { echo "Info: Setting all environment variables, supporting the Mithril signer use case." # Inquire about the relay endpoint @@ -165,7 +703,7 @@ update_mithril_environment_for_signer() { # Generate the full set of environment variables required by Mithril signer use case export ERA_READER_ADDRESS=https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/${RELEASE}-${NETWORK_NAME,,}/era.addr export ERA_READER_VKEY=https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/${RELEASE}-${NETWORK_NAME,,}/era.vkey - sudo bash -c "cat <<-'EOF' > ${CNODE_HOME}/mithril/mithril.env + ${sudo} bash -c "cat <<-'EOF' > ${MITHRIL_HOME}/mithril.env KES_SECRET_KEY_PATH=${POOL_DIR}/${POOL_HOTKEY_SK_FILENAME} OPERATIONAL_CERTIFICATE_PATH=${POOL_DIR}/${POOL_OPCERT_FILENAME} NETWORK=${NETWORK_NAME,,} @@ -175,7 +713,7 @@ update_mithril_environment_for_signer() { DB_DIRECTORY=${DB_DIR} CARDANO_NODE_SOCKET_PATH=${CARDANO_NODE_SOCKET_PATH} CARDANO_CLI_PATH=${HOME}/.local/bin/cardano-cli - DATA_STORES_DIRECTORY=${CNODE_HOME}/mithril/data-stores + DATA_STORES_DIRECTORY=${MITHRIL_HOME}/data-stores STORE_RETENTION_LIMITS=5 ERA_READER_ADAPTER_TYPE=cardano-chain ERA_READER_ADAPTER_PARAMS=$(jq -nc --arg address "$(wget -q -O - "${ERA_READER_ADDRESS}")" --arg verification_key "$(wget -q -O - "${ERA_READER_VKEY}")" '{"address": $address, "verification_key": $verification_key}') @@ -185,42 +723,82 @@ update_mithril_environment_for_signer() { EOF" if [[ "${ENABLE_RELAY_ENDPOINT}" == "y" ]]; then - sudo bash -c "echo RELAY_ENDPOINT=http://${RELAY_ENDPOINT_IP}:${RELAY_PORT} >> ${CNODE_HOME}/mithril/mithril.env" + ${sudo} bash -c "echo RELAY_ENDPOINT=http://${RELAY_ENDPOINT_IP}:${RELAY_PORT} >> ${MITHRIL_HOME}/mithril.env" fi if [[ "${ENABLE_MITHRIL_METRICS}" == "y" ]]; then - sudo bash -c "echo ENABLE_METRICS_SERVER=true >> ${CNODE_HOME}/mithril/mithril.env" - sudo bash -c "echo METRICS_SERVER_IP=${METRICS_SERVER_IP} >> ${CNODE_HOME}/mithril/mithril.env" - sudo bash -c "echo METRICS_SERVER_PORT=${METRICS_SERVER_PORT} >> ${CNODE_HOME}/mithril/mithril.env" + ${sudo} bash -c "echo ENABLE_METRICS_SERVER=true >> ${MITHRIL_HOME}/mithril.env" + ${sudo} bash -c "echo METRICS_SERVER_IP=${METRICS_SERVER_IP} >> ${MITHRIL_HOME}/mithril.env" + ${sudo} bash -c "echo METRICS_SERVER_PORT=${METRICS_SERVER_PORT} >> ${MITHRIL_HOME}/mithril.env" fi } -update_mithril_environment_for_client() { - echo "Info: Setting minimal environment variables supporting only the Mithril client use case." - bash -c "cat <<-'EOF' > ${CNODE_HOME}/mithril/mithril.env - NETWORK=${NETWORK_NAME,,} - RELEASE=${RELEASE} - AGGREGATOR_ENDPOINT=https://aggregator.${RELEASE}-${NETWORK_NAME,,}.api.mithril.network/aggregator - DB_DIRECTORY=${DB_DIR} - GENESIS_VERIFICATION_KEY=$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/${RELEASE}-${NETWORK_NAME,,}/genesis.vkey) - SNAPSHOT_DIGEST=latest - EOF" + +user_interrupt_received() { + echo " SIGINT received, stopping $(basename "${0::-3}").." |tee -a "${LOG_DIR}/$(basename "${0::-3}")".log 2>&1 + stop_signer + } -component_environment_setup() { - check_mithril_environment_file_exists - - if [[ -n "${POOL_NAME}" ]] && [[ "${POOL_NAME}" != "CHANGE_ME" ]] && [[ "$(basename "$0")" == "mithril-signer.sh" ]]; then - update_mithril_environment_for_signer +verify_signer_registration() { + set -e + + if [ -z "${AGGREGATOR_ENDPOINT}" ] || [ -z "${PARTY_ID}" ]; then + echo ">> ERROR: Required environment variables AGGREGATOR_ENDPOINT and/or PARTY_ID are not set." + exit 1 + fi + + check_registration() { + local EPOCH=$1 + SIGNERS_REGISTERED_RESPONSE=$(curl -s "${AGGREGATOR_ENDPOINT}/signers/registered/$EPOCH" -H 'accept: application/json') + if echo "${SIGNERS_REGISTERED_RESPONSE}" | grep -q "${PARTY_ID}"; then + return 0 + else + return 1 + fi + } + + CURRENT_EPOCH=$(curl -s "${AGGREGATOR_ENDPOINT}/epoch-settings" -H 'accept: application/json' | jq -r '.epoch') + SIGNING_EPOCH=$((CURRENT_EPOCH + 2)) + TWO_PRIOR_EPOCH=$((CURRENT_EPOCH - 2)) + + if check_registration "${CURRENT_EPOCH}" ; then + echo ">> Your signer node is registered in the current epoch, it will be able to sign for epoch ${SIGNING_EPOCH}!" + if check_registration "${TWO_PRIOR_EPOCH}" ; then + echo ">> Your signer node was registered in epoch ${TWO_PRIOR_EPOCH} and can sign for the current epoch ${CURRENT_EPOCH}!" + else + echo ">> Your signer node is not eligible to sign for the current epoch. Party ID not found among the registered signers for epoch: ${TWO_PRIOR_EPOCHS} (two epochs ago)." + fi else - update_mithril_environment_for_client + echo ">> Oops, your signer node is not registered. Party ID not found among the signers registered at epoch ${CURRENT_EPOCH}. Please try again later." fi -} -generate_environment_file() { - create_data_stores_directory - component_environment_setup - set_env_file_ownership } +verify_signer_signature() { + set -e + + if [ -z "$AGGREGATOR_ENDPOINT" ] || [ -z "$PARTY_ID" ]; then + echo ">> ERROR: Required environment variables AGGREGATOR_ENDPOINT and/or PARTY_ID are not set." + exit 1 + fi + + CERTIFICATES_RESPONSE=$(curl -s "$AGGREGATOR_ENDPOINT/certificates" -H 'accept: application/json') + CERTIFICATES_COUNT=$(echo "$CERTIFICATES_RESPONSE" | jq '. | length') + + echo "$CERTIFICATES_RESPONSE" | jq -r '.[] | .hash' | while read -r HASH; do + RESPONSE=$(curl -s "$AGGREGATOR_ENDPOINT/certificate/$HASH" -H 'accept: application/json') + SIGNER_COUNT=$(echo "$RESPONSE" | jq '.metadata.signers | length') + for (( i=0; i < SIGNER_COUNT; i++ )); do + PARTY_ID_RESPONSE=$(echo "$RESPONSE" | jq -r ".metadata.signers[$i].party_id") + if [[ "$PARTY_ID_RESPONSE" == "$PARTY_ID" ]]; then + echo ">> Congrats, you have signed this certificate: $AGGREGATOR_ENDPOINT/certificate/$HASH !" + exit 1 + fi + done + done + + echo ">> Oops, your party id was not found in the last ${CERTIFICATES_COUNT} certificates. Please try again later." + +}