From b6eb9de089af826099c37c99deea2fd16ed8debc Mon Sep 17 00:00:00 2001 From: illuminatus Date: Wed, 4 Sep 2024 05:55:01 -0700 Subject: [PATCH] Support for koios API registered user authentication (#1804) ## Description Implements environment variables and arrays: * variable: **KOIOS_API_TOKEN** * array: **KOIOS_API_HEADERS** 1. When variable **KOIOS_API_TOKEN** is a zero length or undefined string the array **KOIOS_API_HEADERS** gets instantiated but is empty. 2. When variable **KOIOS_API_TOKEN** is defined and not zero length the array **KOIOS_API_HEADERS** gets instantiated with `-H "'Authorization: Bearer ${KOIOS_API_TOKEN}'"`. Throughout all scripts which leverage KOIOS_API the headers: * When no other headers are used: * Passes along the Bearer token header for authentication via curl and any `echo`, `printl` or other debug/error statements. * When other headers are already used: * Array **HEADERS** is created from the array **KOIOS_API_HEADERS** and the existing headers are appended to the array. * This is to prevent altering the original KOIOS_API_HEADERS array for any subsequent calls to KOIOS during the same script execution * Passes along all headers for authentication via curl and any `echo`, `printl` or other debug/error statements. ## Motivation and context Nodes, and/or relays, which run Preview, Preprod, Guild, Sanchonet or Mainnet networks may end up using a single outbound IP address, for a number of reasons: * A single system running multiple (hopefully testnet) nodes/relays * Multiple servers behind a single NAT'd IP * Kubernetes environments where multiple pods have started on the same node * Also Edge deployments running containers on the same host. Even with multiple IP's and dedicated to a host some architectures may end up querying Koios using the same IP address which could exceed the daily limits. ## Which issue it fixes? None ## How has this been tested? Mostly testing via executing curl commands manually, have not generated a testing container in ghcr.io yet and loaded onto testnet stakepools and relays to confirm consistent behavior. While the PR is ready for review I would appreciate testing from the community to confirm everything appears to be working as expected. No rush to merge PR if the preferred reviewers and testers are unavailable for the time being. I'd rather have extensive testing and extended review periods than rush the merge. Thanks --------- Co-authored-by: RdLrT <3169068+rdlrt@users.noreply.github.com> --- scripts/cnode-helper-scripts/cncli.sh | 4 +- scripts/cnode-helper-scripts/cntools.library | 39 ++++++++++++-------- scripts/cnode-helper-scripts/env | 14 +++++-- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/scripts/cnode-helper-scripts/cncli.sh b/scripts/cnode-helper-scripts/cncli.sh index bdec8094c..25e6fb1b3 100755 --- a/scripts/cnode-helper-scripts/cncli.sh +++ b/scripts/cnode-helper-scripts/cncli.sh @@ -140,8 +140,8 @@ getConsensus() { } getKoiosData() { - if ! stake_snapshot=$(curl -sSL -f -d _pool_bech32=${POOL_ID_BECH32} "${KOIOS_API}/pool_stake_snapshot" 2>&1); then - echo "ERROR: Koios pool_stake_snapshot query failed: curl -sSL -f -d _pool_bech32=${POOL_ID_BECH32} ${KOIOS_API}/pool_stake_snapshot" + if ! stake_snapshot=$(curl -sSL -f "${KOIOS_API_HEADERS[@]}" -d _pool_bech32=${POOL_ID_BECH32} "${KOIOS_API}/pool_stake_snapshot" 2>&1); then + echo "ERROR: Koios pool_stake_snapshot query failed: curl -sSL -f ${KOIOS_API_HEADERS[*]} -d _pool_bech32=${POOL_ID_BECH32} ${KOIOS_API}/pool_stake_snapshot" return 1 fi read -ra stake_mark <<<"$(jq -r '.[] | select(.snapshot=="Mark") | [.pool_stake, .active_stake, .nonce] | @tsv' <<< ${stake_snapshot})" diff --git a/scripts/cnode-helper-scripts/cntools.library b/scripts/cnode-helper-scripts/cntools.library index 1fc35bb4b..6b70f987b 100644 --- a/scripts/cnode-helper-scripts/cntools.library +++ b/scripts/cnode-helper-scripts/cntools.library @@ -765,8 +765,9 @@ isPoolRegistered() { [[ -f "${POOL_FOLDER}/${1}/${POOL_REGCERT_FILENAME}" ]] && return 2 || return 1 else getPoolID "$1" - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -d '{\"_pool_bech32_ids\":[\"${pool_id_bech32}\"]}' ${KOIOS_API}/pool_info" - ! pool_info=$(curl -sSL -f -X POST -H "Content-Type: application/json" -d '{"_pool_bech32_ids":["'${pool_id_bech32}'"]}' "${KOIOS_API}/pool_info" 2>&1) && error_msg=${pool_info} && return 0 + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_pool_bech32_ids\":[\"${pool_id_bech32}\"]}' ${KOIOS_API}/pool_info" + ! pool_info=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_pool_bech32_ids":["'${pool_id_bech32}'"]}' "${KOIOS_API}/pool_info" 2>&1) && error_msg=${pool_info} && return 0 if [[ ${pool_info} = '[]' ]]; then return 1 fi @@ -833,8 +834,8 @@ getAssetInfo() { if [[ ${CNTOOLS_MODE} != "LIGHT" || $# -lt 1 ]]; then return 2 else - println ACTION "curl -sSL -f -d _asset_policy=$1 -d _asset_name=$2 ${KOIOS_API}/asset_info" - ! asset_info=$(curl -sSL -f -d _asset_policy=$1 -d _asset_name=$2 "${KOIOS_API}/asset_info" 2>&1) && error_msg="${asset_info}" && return 1 + println ACTION "curl -sSL -f ${KOIOS_API_HEADERS[*]} -d _asset_policy=$1 -d _asset_name=$2 ${KOIOS_API}/asset_info" + ! asset_info=$(curl -sSL -f "${KOIOS_API_HEADERS[@]}" -d _asset_policy=$1 -d _asset_name=$2 "${KOIOS_API}/asset_info" 2>&1) && error_msg="${asset_info}" && return 1 if [[ ${asset_info} = '[]' ]]; then return 2 fi @@ -999,8 +1000,9 @@ verifyTx() { return 1 fi if [[ -n ${KOIOS_API} ]]; then - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -H \"accept: text/csv\" -d '{\"_tx_hashes\":[\"${tx_id}\"]' ${KOIOS_API}/tx_status?select=num_confirmations" - ! num_confirmations=$(curl -sSL -f -X POST -H "Content-Type: application/json" -H "accept: text/csv" -d '{"_tx_hashes":["'${tx_id}'"]}' "${KOIOS_API}/tx_status?select=num_confirmations" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${num_confirmations}\n" && return 1 # print error and return + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'" -H "'accept: text/csv'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_tx_hashes\":[\"${tx_id}\"]' ${KOIOS_API}/tx_status?select=num_confirmations" + ! num_confirmations=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_tx_hashes":["'${tx_id}'"]}' "${KOIOS_API}/tx_status?select=num_confirmations" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${num_confirmations}\n" && return 1 # print error and return result=$(tail -n +2 <<< ${num_confirmations}) else println ACTION "${CCLI} ${NETWORK_ERA} query utxo --tx-in ${tx_id}#0 ${NETWORK_IDENTIFIER}| tail -n +3" @@ -1503,8 +1505,9 @@ getBalanceKoios() { if [[ -n ${KOIOS_API} && -n ${addr_list+x} ]]; then printf -v addr_list_joined '\"%s\",' "${addr_list[@]}" [[ $1 != false ]] && extended=true || extended=false - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -H \"accept: text/csv\" -d '{\"_addresses\":[${addr_list_joined%,}],\"_extended\":${extended}}' ${KOIOS_API}/address_utxos?select=address,tx_hash,tx_index,value,asset_list" - ! address_utxo_list=$(curl -sSL -f -X POST -H "Content-Type: application/json" -H "accept: text/csv" -d '{"_addresses":['${addr_list_joined%,}'],"_extended":'${extended}'}' "${KOIOS_API}/address_utxos?select=address,tx_hash,tx_index,value,asset_list" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${address_utxo_list}\n" && return 1 # print error and return + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'" -H "'accept: text/csv'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_addresses\":[${addr_list_joined%,}],\"_extended\":${extended}}' ${KOIOS_API}/address_utxos?select=address,tx_hash,tx_index,value,asset_list" + ! address_utxo_list=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_addresses":['${addr_list_joined%,}'],"_extended":'${extended}'}' "${KOIOS_API}/address_utxos?select=address,tx_hash,tx_index,value,asset_list" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${address_utxo_list}\n" && return 1 # print error and return [[ -z ${address_utxo_list} ]] && return while IFS=',' read -r _address _tx_hash _tx_index _value _asset_list; do [[ ${_address} = address ]] && continue # header line @@ -1684,8 +1687,9 @@ getRewardInfoKoios() { if [[ -n ${KOIOS_API} && -n ${reward_addr_list+x} ]]; then printf -v addr_list_joined '\"%s\",' "${reward_addr_list[@]}" - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -H \"accept: text/csv\" -d '{\"_stake_addresses\":[${addr_list_joined%,}]}' ${KOIOS_API}/account_info?select=stake_address,status,delegated_pool,rewards_available,deposit" - ! account_info_list=$(curl -sSL -f -X POST -H "Content-Type: application/json" -H "accept: text/csv" -d '{"_stake_addresses":['${addr_list_joined%,}']}' "${KOIOS_API}/account_info?select=stake_address,status,delegated_pool,rewards_available,deposit" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${account_info_list}\n" && return 1 # print error and return + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'" -H "'accept: text/csv'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_stake_addresses\":[${addr_list_joined%,}]}' ${KOIOS_API}/account_info?select=stake_address,status,delegated_pool,rewards_available,deposit" + ! account_info_list=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_stake_addresses":['${addr_list_joined%,}']}' "${KOIOS_API}/account_info?select=stake_address,status,delegated_pool,rewards_available,deposit" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${account_info_list}\n" && return 1 # print error and return [[ -z ${account_info_list} ]] && return while IFS=',' read -r stake_address status delegated_pool rewards_available deposit; do [[ ${stake_address} = stake_address ]] && continue # header line @@ -3343,8 +3347,9 @@ rotatePoolKeys() { fi elif [[ -n ${KOIOS_API} ]]; then ! getPoolID "${pool_name}" && println "ERROR" "\n${FG_RED}ERROR${NC}: failed to get pool ID!\n" && return 1 - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -d '{\"_pool_bech32_ids\":[\"${pool_id_bech32}\"]}' ${KOIOS_API}/pool_info" - ! pool_info=$(curl -sSL -f -X POST -H "Content-Type: application/json" -d '{"_pool_bech32_ids":["'${pool_id_bech32}'"]}' "${KOIOS_API}/pool_info" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${pool_info}\n" && p_opcert="" # print error but ignore + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_pool_bech32_ids\":[\"${pool_id_bech32}\"]}' ${KOIOS_API}/pool_info" + ! pool_info=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_pool_bech32_ids":["'${pool_id_bech32}'"]}' "${KOIOS_API}/pool_info" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${pool_info}\n" && p_opcert="" # print error but ignore if old_counter_nbr=$(jq -er '.[0].op_cert_counter' <<< "${pool_info}" 2>/dev/null); then new_counter_nbr=$(( old_counter_nbr + 1 )) else @@ -4470,8 +4475,9 @@ submitTxKoiosSubmitAPI() { cborHex=$(jq -er '.cborHex' "$1" 2>/dev/null) || { println ERROR "\n${FG_RED}ERROR${NC}: Invalid tx file format, 'cborHex' missing in: $1"; return 1; } txdata="$(mktemp "${TMP_DIR}/tx.signed_XXXXXXXXXX")" xxd -p -r <<< ${cborHex} > ${txdata} - println ACTION "curl -sfSL -X POST -H \"Content-Type: application/cbor\" --data-binary @${txdata} \"${KOIOS_API}/submittx\"" - if ! stdout=$(curl -sfSL -X POST -H "Content-Type: application/cbor" --data-binary @${txdata} "${KOIOS_API}/submittx" 2>&1); then + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/cbor'") + println ACTION "curl -sfSL -X POST ${HEADERS[*]} --data-binary @${txdata} \"${KOIOS_API}/submittx\"" + if ! stdout=$(curl -sfSL -X POST "${HEADERS[@]}" --data-binary @${txdata} "${KOIOS_API}/submittx" 2>&1); then println ERROR "\n${FG_RED}ERROR${NC}: Transaction submit failed !!\n${stdout}"; return 1 fi println LOG "Submit result: ${stdout}" @@ -4485,8 +4491,9 @@ submitTxKoiosOgmios() { cborHex=$(jq -er '.cborHex' "$1" 2>/dev/null) || { println ERROR "\n${FG_RED}ERROR${NC}: Invalid tx file format, 'cborHex' missing in: $1"; return 1; } jsonrpc=$(jq -n -c --arg cbor "${cborHex}" '{jsonrpc: "2.0", method: "submitTransaction", params: {transaction: {cbor: $cbor}}}') unset ogmios_error - println ACTION "curl -sSL -X POST -H \"accept: application/json\" -H \"Content-Type: application/json\" -d \"${jsonrpc}\" \"${KOIOS_API}/ogmios/\"" - stdout=$(curl -sSL -X POST -H "accept: application/json" -H "Content-Type: application/json" -d "${jsonrpc}" "${KOIOS_API}/ogmios/" 2>&1) + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'accept: application/json'" -H "'Content-Type: application/json'") + println ACTION "curl -sSL -X POST ${HEADERS[*]} -d \"${jsonrpc}\" \"${KOIOS_API}/ogmios/\"" + stdout=$(curl -sSL -X POST "${HEADERS[@]}" -d "${jsonrpc}" "${KOIOS_API}/ogmios/" 2>&1) if [[ -z ${stdout} ]] || ogmios_error=$(jq -er '.error //empty' <<< "${stdout}") || ! jq -er '.result //empty' <<< "${stdout}" &>/dev/null; then println ERROR "\n${FG_RED}ERROR${NC}: Transaction submit failed !!" if [[ -n ${ogmios_error} ]]; then diff --git a/scripts/cnode-helper-scripts/env b/scripts/cnode-helper-scripts/env index 079a2d42c..033e335ae 100644 --- a/scripts/cnode-helper-scripts/env +++ b/scripts/cnode-helper-scripts/env @@ -37,6 +37,7 @@ #KOIOS_API="https://api.koios.rest/api/v1" # Koios API for blockchain queries instead of local cli lookup. # Leave commented for automatic network detection between MainNet, Preview, Preprod or Guild network. # https://www.koios.rest/ +#KOIOS_API_TOKEN="" # The Koios API token to use for authenticated requests. If left empty, unauthenticated requests will be made. #DBSYNC_QUERY_FOLDER="${CNODE_HOME}/files/dbsync/queries" # [advanced feature] Folder containing DB-Sync chain analysis queries #G_ACCOUNT="cardano-community" # Override default github repository if you forked project @@ -819,8 +820,8 @@ slotInterval() { # 3 : Koios - general error getProtocolParams() { if [[ -n ${KOIOS_API} ]]; then - [[ $(type -t println) = function ]] && println ACTION "curl -sSL -f -X GET -H \"accept: application/json\" ${KOIOS_API}/cli_protocol_params" - if ! PROT_PARAMS=$(curl -sSL -f -X GET -H "accept: application/json" "${KOIOS_API}/cli_protocol_params" 2>&1); then + [[ $(type -t println) = function ]] && println ACTION "curl -sSL -f -X GET ${KOIOS_API_HEADERS[*]} ${KOIOS_API}/cli_protocol_params" + if ! PROT_PARAMS=$(curl -sSL -f -X GET "${KOIOS_API_HEADERS[@]}" "${KOIOS_API}/cli_protocol_params" 2>&1); then return 3 fi else @@ -955,6 +956,13 @@ set_default_vars() { [[ -z ${MITHRIL_HOME} ]] && MITHRIL_HOME="${CNODE_HOME}/mithril" [[ -z ${MITHRIL_SIGNER_ENABLED} ]] && MITHRIL_SIGNER_ENABLED="N" [[ -z ${STRICT_VERSION_CHECK} ]] && STRICT_VERSION_CHECK="Y" + if [[ -z "${KOIOS_API_HEADERS[*]}" ]] ; then + if [[ -n "${KOIOS_API_TOKEN}" ]] ; then + KOIOS_API_HEADERS=(-H "'Authorization: Bearer ${KOIOS_API_TOKEN}'") + else + KOIOS_API_HEADERS=() + fi + fi FG_BLACK='\e[30m' FG_RED='\e[31m' FG_GREEN='\e[32m' @@ -992,7 +1000,7 @@ read_genesis() { test_koios() { # make sure KOIOS_API is reachable, else fall back to cli - [[ ${ENABLE_KOIOS} = 'Y' && -n ${KOIOS_API} && $(curl -sfk -o /dev/null -w "%{http_code}" -m 5 ${KOIOS_API}/tip | awk '{print $1}') = "200" ]] || unset KOIOS_API + [[ ${ENABLE_KOIOS} = 'Y' && -n ${KOIOS_API} && $(curl -sfk -o /dev/null -w "%{http_code}" -m 5 "${KOIOS_API_HEADERS[@]}" ${KOIOS_API}/tip | awk '{print $1}') = "200" ]] || unset KOIOS_API } [[ ${0} != '-bash' ]] && PARENT=$(dirname $0) || PARENT="$(pwd)" # If sourcing at terminal, $0 would be "-bash" , which is invalid. Thus, fallback to present working directory