diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..d9e400c27 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,29 @@ +name: SH Linter + +on: +# push: +# branches: [ '*' ] +# branches: [ master,alpha ] + pull_request: + branches: [ '*' ] + +jobs: + shlint: + name: Shellcheck + runs-on: ubuntu-latest + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v4 + - name: SH Lint Check + uses: ludeeus/action-shellcheck@master + with: + scandir: "./scripts" + severity: "error" + additional_files: "./scripts/cnode-helper-scripts/env, ./scripts/cnode-helper-scripts/cntools.library, ./scripts/cnode-helper-scripts/mithril.library" + jsonlint: + name: JSON and YAML Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: JSON/YAML Validate + uses: GrantBirki/json-yaml-validate@v3.2.1 \ No newline at end of file diff --git a/.github/workflows/shlint.yml b/.github/workflows/shlint.yml deleted file mode 100644 index 19fd98648..000000000 --- a/.github/workflows/shlint.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: SH Linter - -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch -on: - push: - branches: [ master,alpha ] - pull_request: - branches: [ master,alpha ] - -jobs: - shlint: - runs-on: ubuntu-latest - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v1 - - name: SH Lint Check - uses: azohra/shell-linter@latest - with: - path: "scripts/cnode-helper-scripts,scripts/grest-helper-scripts" - severity: "error" diff --git a/docs/Build/dbsync.md b/docs/Build/dbsync.md index 254ae3b03..c795ca868 100644 --- a/docs/Build/dbsync.md +++ b/docs/Build/dbsync.md @@ -60,7 +60,7 @@ ln -s ~/git/cardano-db-sync/schema $CNODE_HOME/guild-db/schema #### Restore using Snapshot -If you're running a mainnet/preview/preprod instance of dbsync, you might want to consider use of dbsync snapshots as documented [here](https://github.com/intersectmbo/cardano-db-sync/blob/master/doc/state-snapshot.md). The snapshot files as of recent epoch are available via links in [release notes](https://github.com/intersectmbo/cardano-db-sync/releases). +If you're running a mainnet/preview/preprod instance of dbsync, you might want to consider use of dbsync snapshots as documented [here](https://github.com/intersectmbo/cardano-db-sync/blob/master/doc/state-snapshot.md). The snapshot files from IO for their default configs as of recent epoch are available via links in [release notes](https://github.com/intersectmbo/cardano-db-sync/releases). Note that the snapshots should only be used pertaining to their specific configs, if using configs from Koios - you'd want to look at snapshots [here](https://share.koios.rest/share/xFdZDfM4/dbsync/) instead. At high-level, this would involve steps as below (read and update paths as per your environment): diff --git a/docs/Build/grest-changelog.md b/docs/Build/grest-changelog.md index 5c2d71468..732b3be14 100644 --- a/docs/Build/grest-changelog.md +++ b/docs/Build/grest-changelog.md @@ -1,5 +1,88 @@ # Koios gRest Changelog +## [1.2.0] - For all networks. + +This is a finalised release that builds on `1.2.0a` to provide support for CIP-129 and add a summary of votes for given proposal. The changes accordingly are primarily only targetting Governance endpoints. This will be the version used for mainnet upgrade as well. Please go through the changelogs below + +### New endpoints added: +- `/proposal_voting_summary` - Get a summary of votes cast on specified governance action [#300] + +### Data Input/Output Changes: +- Input - `/commitee_votes` - Will require `_cc_hot_id` which will accept committee member hot key formatted in bech32 as per CIP-0005/129 [#300] +- Input - `/voter_proposal_list` - Will require `_voter_id` which will accept DRep/SPO/Committee member formatted in bech32 as per CIP-0005/129 [#300] +- Input - `/proposal_votes` - Will require `_proposal_id` which will accept government proposal ID formatted in bech32 as per CIP-129 [#300] +- Output - `/drep_metadata` , `/drep_updates`, - added column `has_script` which shows if given credential is a script hash [#300] +- Output - `/drep_votes` , `/proposal_list` , `/committee_info` - added column `proposal_id` to show proposal action ID in accordance with CIP-129 [#300] +- Output - `/proposal_votes` , - `voter` is renamed to `voter_id` and shows DRep/Pool/Committee member formatted in bech32 as per CIP-129 [#300] +- Output - Any references to drep in output columns is now assumed to be in CIP-129 format [#300] + +### Deprecations: +- None + +### Retirements: +- None + +### Chores: +- Change indexing for dreps from view to hex [#300] +- Extend utility functions for CIP-129 conversions from hex [#300] + +[#300]: https://github.com/cardano-community/koios-artifacts/pull/300 + +## [1.2.0a] - For non-mainnet networks. + +This release starts providing Conway support providing 14 new endpoints - primarily focusing on new governance data. Also, based on community requests/feedbacks - it introduces a few breaking changes for `tx_info` and `block_tx_info` endpoints. Please go through the changelogs below + +### New endpoints added: +- `/tx_cbor` - Raw transaction CBOR against a transaction [#298] +- `/drep_epoch_summary` - Summary of voting power and DRep count for each epoch [#298] +- `/drep_list` - List of all active delegated representatives (DReps) [#298] +- `/drep_info` - Get detailed information about requested delegated representatives (DReps) [#298] +- `/drep_metadata` - List metadata for requested delegated representatives (DReps) [#298] +- `/drep_updates` - List of updates for requested (or all) delegated representatives (DReps) [#298] +- `/drep_votes` - List of all votes casted by requested delegated representative (DRep) [#298] +- `/drep_delegators` - List of all delegators to requested delegated representative (DRep) [#298] +- `/committee_info` - Information about active committee and its members [#298] +- `/committee_votes` - List of all votes casted by given committee member or collective [#298] +- `/proposal_list` - List of all governance proposals [#298] +- `/voter_proposal_list` - List of all governance proposals for specified DRep, SPO or Committee credential [#298] +- `/proposal_votes` - List of all votes cast on specified governance action [#298] +- `/pool_votes` - List of all votes casted by a pool [#298] + +### Data Input/Output Changes: + +- Input - `/block_tx_info`, `/tx_info` - `collateral_tx_out` -> `asset_list` - Outputs for collateral tx out are never created on-chain and thus, cannot be queried from `ma_tx_out`. Instead a rough description of assets involved are saved , which is now returned as info from ledger. This is returned as-is from dbsync and does not adhere to `asset_list` schema we use in other endpoints. [#298] +- Input - `/tx_info` , `/block_tx_info` - These endpoints now require you to specify what all information you'd like to retrieve from a transaction, providing flags `_inputs` , `_metadata`, `_assets` , `_withdrawals`, `_certs`, `_scripts`, `_bytecode`, `_governance` [#298] +- Output - `/policy_asset_mints` , `/policy_asset_info`, `/asset_info` - Will return latest mint transaction that has metadata (instead of latest mint transaction) details (excluding burn transactions) [#298] +- Output - `/account_info` , `/pool_info` , `/pool_list` - Add `deposit` field to output for deposit associated with registration [#298] +- Output - `/account_info` - Add `delegated_drep` field to the output [#298] +- Output - `/block_tx_info` , `/tx_info` - Add `treasury_deposit`, `voting_procedures` and `proposal_procedures` to the output [#298] +- Output - `/epoch_params` - Add various fields to `epoch_params` as per Conway protocol parameters [#298] +- Output - `/pool_metadata`, `/pool_relays` - Remove `pool_status` field from output (it's already listed in pool_info and list) [#298] +- Output - `/pool_updates` - owners is now a JSONB field instead of JSONB array [#298] + +### Deprecations: +- None + +### Retirements: +- None + +### Chores: + +- Remove unused info from `asset_info_cache` - `first_mint_tx_id` , `first_mint_keys` , `last_mint_keys` are not used/required [#286] +- Add `last_mint_meta_tx_id` field to `asset_info_cache` - to return latest asset that does have metadata [#286] +- Reduce redundant cache information for pool stake as we now only retain 3 epochs in pool_active_stake_cache as the rest is already in `pool_history_cache` [#289] +- Retire v0 SQL files (endpoints were already removed) from repository [#286] +- Overwrite next epoch once on every execution (this is to avoid nonce mismatch if calculated too early from node) [#286] +- Reduce reliance on pool_info_cache where possible to query live metadata [#298] +- Make use of `pool_stat` instead of `epoch_stake` for `pool_history_cache` [#294] +- `instant_reward` table in dbsync moved to `reward_rest` +- `ada_pots` : `deposit` now split into three different types of deposits + +[#298]: https://github.com/cardano-community/koios-artifacts/pull/298 +[#289]: https://github.com/cardano-community/koios-artifacts/pull/289 +[#286]: https://github.com/cardano-community/koios-artifacts/pull/286 +[#294]: https://github.com/cardano-community/koios-artifacts/pull/294 + ## [1.1.2] - For all networks. This release is minor bugfix for data consistency changes behind the scenes. It has no impact to any of the API endpoints. diff --git a/files/configs/guild/config.json b/files/configs/guild/config.json index 137ea8744..1adfc299b 100644 --- a/files/configs/guild/config.json +++ b/files/configs/guild/config.json @@ -12,6 +12,8 @@ "RequiresNetworkMagic": "RequiresMagic", "PBftSignatureThreshold": 1, "ShelleyGenesisFile": "/opt/cardano/cnode/files/shelley-genesis.json", + "SnapshotInterval": 1800, + "MempoolCapacityBytesOverride": 8388608, "EnableLogging": true, "EnableLogMetrics": false, "TargetNumberOfActivePeers": 10, diff --git a/files/configs/guild/db-sync-config.json b/files/configs/guild/db-sync-config.json index 742b14985..4ca1b32ae 100644 --- a/files/configs/guild/db-sync-config.json +++ b/files/configs/guild/db-sync-config.json @@ -36,6 +36,7 @@ "json_type": "text", "offchain_pool_data": "enable", "pool_stat": "enable", + "pool_stats": "enable", "tx_cbor": "enable" }, "minSeverity": "Info", diff --git a/files/configs/mainnet/config.json b/files/configs/mainnet/config.json index d2b1d2c1a..6518a24d9 100644 --- a/files/configs/mainnet/config.json +++ b/files/configs/mainnet/config.json @@ -12,6 +12,8 @@ "Protocol": "Cardano", "RequiresNetworkMagic": "RequiresNoMagic", "ShelleyGenesisFile": "/opt/cardano/cnode/files/shelley-genesis.json", + "SnapshotInterval": 86400, + "MempoolCapacityBytesOverride": 67108864, "EnableLogging": true, "EnableLogMetrics": false, "TargetNumberOfActivePeers": 20, diff --git a/files/configs/mainnet/db-sync-config.json b/files/configs/mainnet/db-sync-config.json index e160033a6..9193d6bc6 100644 --- a/files/configs/mainnet/db-sync-config.json +++ b/files/configs/mainnet/db-sync-config.json @@ -36,6 +36,7 @@ "json_type": "text", "offchain_pool_data": "enable", "pool_stat": "enable", + "pool_stats": "enable", "tx_cbor": "enable" }, "minSeverity": "Info", diff --git a/files/configs/preprod/config.json b/files/configs/preprod/config.json index 1a5a27be2..cecb74bc8 100644 --- a/files/configs/preprod/config.json +++ b/files/configs/preprod/config.json @@ -11,6 +11,8 @@ "Protocol": "Cardano", "RequiresNetworkMagic": "RequiresMagic", "ShelleyGenesisFile": "/opt/cardano/cnode/files/shelley-genesis.json", + "SnapshotInterval": 86400, + "MempoolCapacityBytesOverride": 8388608, "EnableLogging": true, "EnableLogMetrics": false, "TargetNumberOfActivePeers": 20, diff --git a/files/configs/preprod/db-sync-config.json b/files/configs/preprod/db-sync-config.json index 7ebc4aedb..e018289a4 100644 --- a/files/configs/preprod/db-sync-config.json +++ b/files/configs/preprod/db-sync-config.json @@ -36,6 +36,7 @@ "json_type": "text", "offchain_pool_data": "enable", "pool_stat": "enable", + "pool_stats": "enable", "tx_cbor": "enable" }, "minSeverity": "Info", diff --git a/files/configs/preview/config.json b/files/configs/preview/config.json index 70e9fb338..e5b3950eb 100644 --- a/files/configs/preview/config.json +++ b/files/configs/preview/config.json @@ -13,6 +13,8 @@ "Protocol": "Cardano", "RequiresNetworkMagic": "RequiresMagic", "ShelleyGenesisFile": "/opt/cardano/cnode/files/shelley-genesis.json", + "SnapshotInterval": 43200, + "MempoolCapacityBytesOverride": 8388608, "EnableLogging": true, "EnableLogMetrics": false, "TargetNumberOfActivePeers": 20, diff --git a/files/configs/preview/db-sync-config.json b/files/configs/preview/db-sync-config.json index b87c788f5..51d0ab3ff 100644 --- a/files/configs/preview/db-sync-config.json +++ b/files/configs/preview/db-sync-config.json @@ -36,6 +36,7 @@ "json_type": "text", "offchain_pool_data": "enable", "pool_stat": "enable", + "pool_stats": "enable", "tx_cbor": "enable" }, "minSeverity": "Info", diff --git a/files/configs/sanchonet/config.json b/files/configs/sanchonet/config.json index 4d6643164..b0b14483f 100644 --- a/files/configs/sanchonet/config.json +++ b/files/configs/sanchonet/config.json @@ -13,6 +13,8 @@ "Protocol": "Cardano", "RequiresNetworkMagic": "RequiresMagic", "ShelleyGenesisFile": "/opt/cardano/cnode/files/shelley-genesis.json", + "SnapshotInterval": 43200, + "MempoolCapacityBytesOverride": 8388608, "EnableLogging": true, "EnableLogMetrics": false, "TargetNumberOfActivePeers": 20, diff --git a/files/configs/sanchonet/db-sync-config.json b/files/configs/sanchonet/db-sync-config.json index f9e1eca07..ad5d76ea5 100644 --- a/files/configs/sanchonet/db-sync-config.json +++ b/files/configs/sanchonet/db-sync-config.json @@ -36,6 +36,7 @@ "json_type": "text", "offchain_pool_data": "enable", "pool_stat": "enable", + "pool_stats": "enable", "tx_cbor": "enable" }, "minSeverity": "Info", diff --git a/files/docker/node/release-versions/cardano-node-latest.txt b/files/docker/node/release-versions/cardano-node-latest.txt index e977f5eae..8ce0f0f36 100644 --- a/files/docker/node/release-versions/cardano-node-latest.txt +++ b/files/docker/node/release-versions/cardano-node-latest.txt @@ -1 +1 @@ -9.1.0 \ No newline at end of file +9.1.1 \ No newline at end of file diff --git a/files/node-deps.json b/files/node-deps.json index 0f22af8b0..a929d9c62 100644 --- a/files/node-deps.json +++ b/files/node-deps.json @@ -6,6 +6,12 @@ "sodium": "dbb48cce5429cb6585c9034f002568964f1ce567" }, "8.9.4": + { + "blst": "3dd0f804b1819e5d03fb22ca2e6fac105932043a", + "secp256k1": "v0.3.2", + "sodium": "dbb48cce5429cb6585c9034f002568964f1ce567" + }, + "9.1.1": { "blst": "3dd0f804b1819e5d03fb22ca2e6fac105932043a", "secp256k1": "v0.3.2", diff --git a/guild.svg b/guild.svg new file mode 100644 index 000000000..335cd9b48 --- /dev/null +++ b/guild.svg @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scripts/cnode-helper-scripts/cncli.sh b/scripts/cnode-helper-scripts/cncli.sh index d87860cce..6e9fe0af9 100755 --- a/scripts/cnode-helper-scripts/cncli.sh +++ b/scripts/cnode-helper-scripts/cncli.sh @@ -25,7 +25,6 @@ #CONFIRM_SLOT_CNT=600 # CNCLI validate: require at least these many slots to have passed before validating #CONFIRM_BLOCK_CNT=15 # CNCLI validate: require at least these many blocks on top of minted before validating #BATCH_AUTO_UPDATE=N # Set to Y to automatically update the script if a new version is available without user interaction -#USE_KOIOS_API=Y # Use Koios API in cncli leaderlog instead of local stake-snapshot query to reduce system resources. (default true) #CNCLI_PROM_PORT=12799 # Set Prometheus port for cncli block metrics available through metrics operation (default: 12799) ###################################### @@ -148,9 +147,8 @@ getConsensus() { } getKoiosData() { - [[ -z ${KOIOS_API} ]] && return 1 - if ! stake_snapshot=$(curl -sSL -f -d _pool_bech32=${POOL_ID_BECH32} "${KOIOS_API}/pool_stake_snapshot" 2>&1); then - echo "ERROR: Koios pool_stake_snapshot query failed: curl -sSL -f -d _pool_bech32=${POOL_ID_BECH32} ${KOIOS_API}/pool_stake_snapshot" + if ! stake_snapshot=$(curl -sSL -f "${KOIOS_API_HEADERS[@]}" -d _pool_bech32=${POOL_ID_BECH32} "${KOIOS_API}/pool_stake_snapshot" 2>&1); then + echo "ERROR: Koios pool_stake_snapshot query failed: curl -sSL -f ${KOIOS_API_HEADERS[*]} -d _pool_bech32=${POOL_ID_BECH32} ${KOIOS_API}/pool_stake_snapshot" return 1 fi read -ra stake_mark <<<"$(jq -r '.[] | select(.snapshot=="Mark") | [.pool_stake, .active_stake, .nonce] | @tsv' <<< ${stake_snapshot})" @@ -217,7 +215,9 @@ cncliInit() { echo "sleeping for 10s and testing again..." sleep 10 done - + + test_koios + TMP_DIR="${TMP_DIR}/cncli" if ! mkdir -p "${TMP_DIR}" 2>/dev/null; then echo "ERROR: Failed to create directory for temporary files: ${TMP_DIR}"; exit 1; fi @@ -228,7 +228,6 @@ cncliInit() { [[ -z "${CNCLI_DIR}" ]] && CNCLI_DIR="${CNODE_HOME}/guild-db/cncli" if ! mkdir -p "${CNCLI_DIR}" 2>/dev/null; then echo "ERROR: Failed to create CNCLI DB directory: ${CNCLI_DIR}"; exit 1; fi CNCLI_DB="${CNCLI_DIR}/cncli.db" - [[ -z "${USE_KOIOS_API}" ]] && USE_KOIOS_API=Y [[ -z "${CNODE_HOST}" ]] && CNODE_HOST="127.0.0.1" [[ -z "${SLEEP_RATE}" ]] && SLEEP_RATE=60 [[ -z "${CONFIRM_SLOT_CNT}" ]] && CONFIRM_SLOT_CNT=600 @@ -296,7 +295,7 @@ cncliLeaderlog() { echo "Leaderlogs already calculated for epoch ${curr_epoch}, skipping!" else echo "Running leaderlogs for epoch ${curr_epoch}" - if [[ ${USE_KOIOS_API} = Y ]]; then + if [[ -n ${KOIOS_API} ]]; then getKoiosData || exit 1 else getLedgerData || exit 1 @@ -368,7 +367,7 @@ cncliLeaderlog() { else continue; fi fi echo "Running leaderlogs for next epoch[${next_epoch}]" - if [[ ${USE_KOIOS_API} = Y ]]; then + if [[ -n ${KOIOS_API} ]]; then if ! getKoiosData; then sleep 60; continue; fi # Sleep for 1 min before retrying to query koios again in case of error else if ! getLedgerData; then sleep 300; continue; fi # Sleep for 5 min before retrying to query stake snapshot in case of error diff --git a/scripts/cnode-helper-scripts/cnode.sh b/scripts/cnode-helper-scripts/cnode.sh index 19f179a1a..f554f5d3a 100755 --- a/scripts/cnode-helper-scripts/cnode.sh +++ b/scripts/cnode-helper-scripts/cnode.sh @@ -8,7 +8,6 @@ ###################################### #CPU_CORES=4 # Number of CPU cores cardano-node process has access to (please don't set higher than physical core count, recommended to set atleast to 4) -#MEMPOOL_BYTES=8388608 # Override mempool in bytes (Default: Do not override) #CNODE_LISTEN_IP4=0.0.0.0 # IP to use for listening (only applicable to Node Connection Port) for IPv4 #CNODE_LISTEN_IP6=:: # IP to use for listening (only applicable to Node Connection Port) for IPv6 @@ -42,7 +41,6 @@ set_defaults() { host_addr=() [[ ${IP_VERSION} = "4" || ${IP_VERSION} = "mix" ]] && host_addr+=("--host-addr" "${CNODE_LISTEN_IP4}") [[ ${IP_VERSION} = "6" || ${IP_VERSION} = "mix" ]] && host_addr+=("--host-ipv6-addr" "${CNODE_LISTEN_IP6}") - [[ -z ${MEMPOOL_BYTES} ]] && MEMPOOL_OVERRIDE="" || MEMPOOL_OVERRIDE="--mempool-capacity-override ${MEMPOOL_BYTES}" } pre_startup_sanity() { @@ -65,7 +63,7 @@ mithril_snapshot_download() { if [[ ! -f "${MITHRIL_CLIENT}" ]] || [[ ! -e "${MITHRIL_CLIENT}" ]]; then echo "ERROR: Could not locate mithril-client.sh script or script is not executable. Skipping mithril cardano-db snapshot download!!" else - "${MITHRIL_CLIENT}" cardano-db download + "${MITHRIL_CLIENT}" -u cardano-db download fi } diff --git a/scripts/cnode-helper-scripts/cntools.library b/scripts/cnode-helper-scripts/cntools.library index 1fc35bb4b..6b70f987b 100644 --- a/scripts/cnode-helper-scripts/cntools.library +++ b/scripts/cnode-helper-scripts/cntools.library @@ -765,8 +765,9 @@ isPoolRegistered() { [[ -f "${POOL_FOLDER}/${1}/${POOL_REGCERT_FILENAME}" ]] && return 2 || return 1 else getPoolID "$1" - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -d '{\"_pool_bech32_ids\":[\"${pool_id_bech32}\"]}' ${KOIOS_API}/pool_info" - ! pool_info=$(curl -sSL -f -X POST -H "Content-Type: application/json" -d '{"_pool_bech32_ids":["'${pool_id_bech32}'"]}' "${KOIOS_API}/pool_info" 2>&1) && error_msg=${pool_info} && return 0 + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_pool_bech32_ids\":[\"${pool_id_bech32}\"]}' ${KOIOS_API}/pool_info" + ! pool_info=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_pool_bech32_ids":["'${pool_id_bech32}'"]}' "${KOIOS_API}/pool_info" 2>&1) && error_msg=${pool_info} && return 0 if [[ ${pool_info} = '[]' ]]; then return 1 fi @@ -833,8 +834,8 @@ getAssetInfo() { if [[ ${CNTOOLS_MODE} != "LIGHT" || $# -lt 1 ]]; then return 2 else - println ACTION "curl -sSL -f -d _asset_policy=$1 -d _asset_name=$2 ${KOIOS_API}/asset_info" - ! asset_info=$(curl -sSL -f -d _asset_policy=$1 -d _asset_name=$2 "${KOIOS_API}/asset_info" 2>&1) && error_msg="${asset_info}" && return 1 + println ACTION "curl -sSL -f ${KOIOS_API_HEADERS[*]} -d _asset_policy=$1 -d _asset_name=$2 ${KOIOS_API}/asset_info" + ! asset_info=$(curl -sSL -f "${KOIOS_API_HEADERS[@]}" -d _asset_policy=$1 -d _asset_name=$2 "${KOIOS_API}/asset_info" 2>&1) && error_msg="${asset_info}" && return 1 if [[ ${asset_info} = '[]' ]]; then return 2 fi @@ -999,8 +1000,9 @@ verifyTx() { return 1 fi if [[ -n ${KOIOS_API} ]]; then - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -H \"accept: text/csv\" -d '{\"_tx_hashes\":[\"${tx_id}\"]' ${KOIOS_API}/tx_status?select=num_confirmations" - ! num_confirmations=$(curl -sSL -f -X POST -H "Content-Type: application/json" -H "accept: text/csv" -d '{"_tx_hashes":["'${tx_id}'"]}' "${KOIOS_API}/tx_status?select=num_confirmations" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${num_confirmations}\n" && return 1 # print error and return + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'" -H "'accept: text/csv'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_tx_hashes\":[\"${tx_id}\"]' ${KOIOS_API}/tx_status?select=num_confirmations" + ! num_confirmations=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_tx_hashes":["'${tx_id}'"]}' "${KOIOS_API}/tx_status?select=num_confirmations" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${num_confirmations}\n" && return 1 # print error and return result=$(tail -n +2 <<< ${num_confirmations}) else println ACTION "${CCLI} ${NETWORK_ERA} query utxo --tx-in ${tx_id}#0 ${NETWORK_IDENTIFIER}| tail -n +3" @@ -1503,8 +1505,9 @@ getBalanceKoios() { if [[ -n ${KOIOS_API} && -n ${addr_list+x} ]]; then printf -v addr_list_joined '\"%s\",' "${addr_list[@]}" [[ $1 != false ]] && extended=true || extended=false - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -H \"accept: text/csv\" -d '{\"_addresses\":[${addr_list_joined%,}],\"_extended\":${extended}}' ${KOIOS_API}/address_utxos?select=address,tx_hash,tx_index,value,asset_list" - ! address_utxo_list=$(curl -sSL -f -X POST -H "Content-Type: application/json" -H "accept: text/csv" -d '{"_addresses":['${addr_list_joined%,}'],"_extended":'${extended}'}' "${KOIOS_API}/address_utxos?select=address,tx_hash,tx_index,value,asset_list" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${address_utxo_list}\n" && return 1 # print error and return + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'" -H "'accept: text/csv'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_addresses\":[${addr_list_joined%,}],\"_extended\":${extended}}' ${KOIOS_API}/address_utxos?select=address,tx_hash,tx_index,value,asset_list" + ! address_utxo_list=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_addresses":['${addr_list_joined%,}'],"_extended":'${extended}'}' "${KOIOS_API}/address_utxos?select=address,tx_hash,tx_index,value,asset_list" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${address_utxo_list}\n" && return 1 # print error and return [[ -z ${address_utxo_list} ]] && return while IFS=',' read -r _address _tx_hash _tx_index _value _asset_list; do [[ ${_address} = address ]] && continue # header line @@ -1684,8 +1687,9 @@ getRewardInfoKoios() { if [[ -n ${KOIOS_API} && -n ${reward_addr_list+x} ]]; then printf -v addr_list_joined '\"%s\",' "${reward_addr_list[@]}" - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -H \"accept: text/csv\" -d '{\"_stake_addresses\":[${addr_list_joined%,}]}' ${KOIOS_API}/account_info?select=stake_address,status,delegated_pool,rewards_available,deposit" - ! account_info_list=$(curl -sSL -f -X POST -H "Content-Type: application/json" -H "accept: text/csv" -d '{"_stake_addresses":['${addr_list_joined%,}']}' "${KOIOS_API}/account_info?select=stake_address,status,delegated_pool,rewards_available,deposit" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${account_info_list}\n" && return 1 # print error and return + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'" -H "'accept: text/csv'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_stake_addresses\":[${addr_list_joined%,}]}' ${KOIOS_API}/account_info?select=stake_address,status,delegated_pool,rewards_available,deposit" + ! account_info_list=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_stake_addresses":['${addr_list_joined%,}']}' "${KOIOS_API}/account_info?select=stake_address,status,delegated_pool,rewards_available,deposit" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${account_info_list}\n" && return 1 # print error and return [[ -z ${account_info_list} ]] && return while IFS=',' read -r stake_address status delegated_pool rewards_available deposit; do [[ ${stake_address} = stake_address ]] && continue # header line @@ -3343,8 +3347,9 @@ rotatePoolKeys() { fi elif [[ -n ${KOIOS_API} ]]; then ! getPoolID "${pool_name}" && println "ERROR" "\n${FG_RED}ERROR${NC}: failed to get pool ID!\n" && return 1 - println ACTION "curl -sSL -f -X POST -H \"Content-Type: application/json\" -d '{\"_pool_bech32_ids\":[\"${pool_id_bech32}\"]}' ${KOIOS_API}/pool_info" - ! pool_info=$(curl -sSL -f -X POST -H "Content-Type: application/json" -d '{"_pool_bech32_ids":["'${pool_id_bech32}'"]}' "${KOIOS_API}/pool_info" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${pool_info}\n" && p_opcert="" # print error but ignore + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/json'") + println ACTION "curl -sSL -f -X POST ${HEADERS[*]} -d '{\"_pool_bech32_ids\":[\"${pool_id_bech32}\"]}' ${KOIOS_API}/pool_info" + ! pool_info=$(curl -sSL -f -X POST "${HEADERS[@]}" -d '{"_pool_bech32_ids":["'${pool_id_bech32}'"]}' "${KOIOS_API}/pool_info" 2>&1) && println "ERROR" "\n${FG_RED}KOIOS_API ERROR${NC}: ${pool_info}\n" && p_opcert="" # print error but ignore if old_counter_nbr=$(jq -er '.[0].op_cert_counter' <<< "${pool_info}" 2>/dev/null); then new_counter_nbr=$(( old_counter_nbr + 1 )) else @@ -4470,8 +4475,9 @@ submitTxKoiosSubmitAPI() { cborHex=$(jq -er '.cborHex' "$1" 2>/dev/null) || { println ERROR "\n${FG_RED}ERROR${NC}: Invalid tx file format, 'cborHex' missing in: $1"; return 1; } txdata="$(mktemp "${TMP_DIR}/tx.signed_XXXXXXXXXX")" xxd -p -r <<< ${cborHex} > ${txdata} - println ACTION "curl -sfSL -X POST -H \"Content-Type: application/cbor\" --data-binary @${txdata} \"${KOIOS_API}/submittx\"" - if ! stdout=$(curl -sfSL -X POST -H "Content-Type: application/cbor" --data-binary @${txdata} "${KOIOS_API}/submittx" 2>&1); then + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'Content-Type: application/cbor'") + println ACTION "curl -sfSL -X POST ${HEADERS[*]} --data-binary @${txdata} \"${KOIOS_API}/submittx\"" + if ! stdout=$(curl -sfSL -X POST "${HEADERS[@]}" --data-binary @${txdata} "${KOIOS_API}/submittx" 2>&1); then println ERROR "\n${FG_RED}ERROR${NC}: Transaction submit failed !!\n${stdout}"; return 1 fi println LOG "Submit result: ${stdout}" @@ -4485,8 +4491,9 @@ submitTxKoiosOgmios() { cborHex=$(jq -er '.cborHex' "$1" 2>/dev/null) || { println ERROR "\n${FG_RED}ERROR${NC}: Invalid tx file format, 'cborHex' missing in: $1"; return 1; } jsonrpc=$(jq -n -c --arg cbor "${cborHex}" '{jsonrpc: "2.0", method: "submitTransaction", params: {transaction: {cbor: $cbor}}}') unset ogmios_error - println ACTION "curl -sSL -X POST -H \"accept: application/json\" -H \"Content-Type: application/json\" -d \"${jsonrpc}\" \"${KOIOS_API}/ogmios/\"" - stdout=$(curl -sSL -X POST -H "accept: application/json" -H "Content-Type: application/json" -d "${jsonrpc}" "${KOIOS_API}/ogmios/" 2>&1) + HEADERS=("${KOIOS_API_HEADERS[@]}" -H "'accept: application/json'" -H "'Content-Type: application/json'") + println ACTION "curl -sSL -X POST ${HEADERS[*]} -d \"${jsonrpc}\" \"${KOIOS_API}/ogmios/\"" + stdout=$(curl -sSL -X POST "${HEADERS[@]}" -d "${jsonrpc}" "${KOIOS_API}/ogmios/" 2>&1) if [[ -z ${stdout} ]] || ogmios_error=$(jq -er '.error //empty' <<< "${stdout}") || ! jq -er '.result //empty' <<< "${stdout}" &>/dev/null; then println ERROR "\n${FG_RED}ERROR${NC}: Transaction submit failed !!" if [[ -n ${ogmios_error} ]]; then diff --git a/scripts/cnode-helper-scripts/env b/scripts/cnode-helper-scripts/env index cdc0443fe..a0bfba390 100644 --- a/scripts/cnode-helper-scripts/env +++ b/scripts/cnode-helper-scripts/env @@ -37,6 +37,7 @@ #KOIOS_API="https://api.koios.rest/api/v1" # Koios API for blockchain queries instead of local cli lookup. # Leave commented for automatic network detection between MainNet, Preview, Preprod or Guild network. # https://www.koios.rest/ +#KOIOS_API_TOKEN="" # The Koios API token to use for authenticated requests. If left empty, unauthenticated requests will be made. #DBSYNC_QUERY_FOLDER="${CNODE_HOME}/files/dbsync/queries" # [advanced feature] Folder containing DB-Sync chain analysis queries #G_ACCOUNT="cardano-community" # Override default github repository if you forked project @@ -384,10 +385,11 @@ getDecimalPlaces() { # : $2 = normal | pretty (default) # : $3 = full | trim (default, removes trailing zeros from fraction) formatLovelace() { + local frac if isNumber $1; then [[ $1 -eq 0 ]] && echo 0 && return - [[ $1 -le 999999 ]] && printf '%s' "$(local frac=$(printf '%06d' "$1"); [[ -z $3 || $3 = 'trim' ]] && ([[ ${frac} =~ (^[0-9]*[1-9])?0*$ ]]; printf '0.%s' "${BASH_REMATCH[1]}") || printf '0.%s' "${frac}")" && return - [[ -z $3 || $3 = 'trim' ]] && fraction="$(local frac=${1: -6}; [[ ${frac} =~ (^[0-9]*[1-9])?0*$ ]]; printf '.%s' "${BASH_REMATCH[1]}")" || fraction=".${1: -6}" + [[ $1 -le 999999 ]] && printf '%s' "$(frac=$(printf '%06d' "$1"); [[ -z $3 || $3 = 'trim' ]] && ([[ ${frac} =~ (^[0-9]*[1-9])?0*$ ]]; printf '0.%s' "${BASH_REMATCH[1]}") || printf '0.%s' "${frac}")" && return + [[ -z $3 || $3 = 'trim' ]] && fraction="$(frac=${1: -6}; [[ ${frac} =~ (^[0-9]*[1-9])?0*$ ]]; printf '.%s' "${BASH_REMATCH[1]}")" || fraction=".${1: -6}" [[ ${fraction} = '.' ]] && fraction="" [[ -z $2 || $2 = 'pretty' ]] && printf '%s%s' "$(sed ':a;s/\B[0-9]\{3\}\>/,&/;ta' <<< ${1::-6})" "${fraction}" || printf '%s%s' "${1::-6}" "${fraction}" else @@ -818,8 +820,8 @@ slotInterval() { # 3 : Koios - general error getProtocolParams() { if [[ -n ${KOIOS_API} ]]; then - [[ $(type -t println) = function ]] && println ACTION "curl -sSL -f -X GET -H \"accept: application/json\" ${KOIOS_API}/cli_protocol_params" - if ! PROT_PARAMS=$(curl -sSL -f -X GET -H "accept: application/json" "${KOIOS_API}/cli_protocol_params" 2>&1); then + [[ $(type -t println) = function ]] && println ACTION "curl -sSL -f -X GET ${KOIOS_API_HEADERS[*]} ${KOIOS_API}/cli_protocol_params" + if ! PROT_PARAMS=$(curl -sSL -f -X GET "${KOIOS_API_HEADERS[@]}" "${KOIOS_API}/cli_protocol_params" 2>&1); then return 3 fi else @@ -972,6 +974,13 @@ set_default_vars() { [[ -z ${MITHRIL_HOME} ]] && MITHRIL_HOME="${CNODE_HOME}/mithril" [[ -z ${MITHRIL_SIGNER_ENABLED} ]] && MITHRIL_SIGNER_ENABLED="N" [[ -z ${STRICT_VERSION_CHECK} ]] && STRICT_VERSION_CHECK="Y" + if [[ -z "${KOIOS_API_HEADERS[*]}" ]] ; then + if [[ -n "${KOIOS_API_TOKEN}" ]] ; then + KOIOS_API_HEADERS=(-H "'Authorization: Bearer ${KOIOS_API_TOKEN}'") + else + KOIOS_API_HEADERS=() + fi + fi FG_BLACK='\e[30m' FG_RED='\e[31m' FG_GREEN='\e[32m' @@ -986,6 +995,9 @@ set_default_vars() { STANDOUT='\e[7m' BOLD='\e[1m' NC='\e[0m' + # Due to bug introduced in bash upstream at 5.2.21, need to temporarily enable POSIX mode for newer bash versions, until fix is live + # https://github.com/bminor/bash/commit/e327891b52513bef0b34aac625c44f8fa6811f53 + versionCheck 5.2.20 ${BASH_VERSION//(*/} && set -o posix } read_genesis() { @@ -1006,7 +1018,7 @@ read_genesis() { test_koios() { # make sure KOIOS_API is reachable, else fall back to cli - [[ ${ENABLE_KOIOS} = 'Y' && -n ${KOIOS_API} && $(curl -sfk -o /dev/null -w "%{http_code}" -m 5 ${KOIOS_API}/tip | awk '{print $1}') = "200" ]] || unset KOIOS_API + [[ ${ENABLE_KOIOS} = 'Y' && -n ${KOIOS_API} && $(curl -sfk -o /dev/null -w "%{http_code}" -m 5 "${KOIOS_API_HEADERS[@]}" ${KOIOS_API}/tip | awk '{print $1}') = "200" ]] || unset KOIOS_API } [[ ${0} != '-bash' ]] && PARENT=$(dirname $0) || PARENT="$(pwd)" # If sourcing at terminal, $0 would be "-bash" , which is invalid. Thus, fallback to present working directory @@ -1050,7 +1062,7 @@ fi node_version="$(${CNODEBIN} version | head -1 | cut -d' ' -f2)" cli_version="$(${CCLI} version | head -1 | cut -d' ' -f2)" -if ! versionCheckNode "9.1.0" "${node_version}" || ! versionCheckNode "9.2.1.0" "${cli_version}"; then +if ! versionCheckNode "9.1.1" "${node_version}" || ! versionCheckNode "9.2.1.0" "${cli_version}"; then echo -e "\nKoios scripts have now been upgraded to support cardano-node 9.1.x ('${node_version}' found) / cardano-cli 9.2.x.x ('${cli_version}' found).\nPlease update cardano-node binaries (ensure to read release notes and update various configs using guild-deploy (use appropriate options to download/install/overwrite parts you need) or use tagged branches for older node version (eg: ./