Koios v1.1.0rc

.github/workflows/docker_bin.yml

@@ -32,10 +32,13 @@ jobs:
         sudo apt-get update -y 
         sudo apt-get remove -y '^dotnet-.*'
         sudo apt-get remove -y 'php.*'
-        sudo apt-get remove -y azure-cli google-cloud-sdk google-chrome-stable firefox powershell mono-devel
+        sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel
         sudo apt-get autoremove -y
         sudo apt-get clean
-        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf "/usr/share/dotnet"
+        sudo rm -rf "/usr/local/lib/android"
+        sudo rm -rf "/opt/ghc"
+        sudo rm -rf "/opt/hostedtoolcache/CodeQL"
         sudo rm -rf "/usr/local/share/boost"
         sudo rm -rf "$AGENT_TOOLSDIRECTORY"
     - name: docker push latest

.github/workflows/premerge.yml

@@ -30,12 +30,13 @@ jobs:
         sudo apt-get update -y
         sudo apt-get remove -y '^dotnet-.*'
         sudo apt-get remove -y 'php.*'
-        sudo apt-get remove -y azure-cli google-cloud-sdk google-chrome-stable firefox powershell mono-devel
+        sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel
         sudo apt-get autoremove -y
         sudo apt-get clean
         sudo rm -rf "/usr/share/dotnet"
         sudo rm -rf "/usr/local/lib/android"
         sudo rm -rf "/opt/ghc"
+        sudo rm -rf "/opt/hostedtoolcache/CodeQL"
         sudo rm -rf "/usr/local/share/boost"
         sudo rm -rf "$AGENT_TOOLSDIRECTORY"
     - name: Login to GitHub Container Registry
@@ -48,7 +49,7 @@ jobs:
     - name: Define BRANCH, COMMIT and G_ACCOUNT in environment
       run: |
         echo "G_ACCOUNT=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV
-        if [[ -z ${{ env.BRANCH }} ]]; then
+        if [[ -z "${{ env.BRANCH }}" ]]; then
           echo "BRANCH=${GITHUB_HEAD_REF}" >> $GITHUB_ENV
           echo "COMMIT=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_ENV
         else

docs/Build/grest-changelog.md

@@ -1,5 +1,47 @@
 # Koios gRest Changelog
 
+## [1.1.0rc] - For all networks.
+
+This is a breaking release for Koios, and accordingly - the base URL for queries will be shifting to `v1` instead of `v0`. With updates across the board for underlying node, dbsync and ogmios infrastructure, 12 new endpoints, quite a few value additions [new columns or flattended output] to the output of existing endpoints, as well as performance improvements especially for large volume transaction consumptions.
+
+### New endpoints added:
+- `/pool_registrations` - List of all pool registrations initiated in the requested epoch [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/pool_retirements` - List of all pool retirements initiated in the requested epoch [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/treasury_withdrawals` - List of withdrawals made from treasury [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/reserve_withdrawals` - List of withdrawals made from reserves (MIRs) [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/account_txs` - Transactions associated with a given stake address [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/address_utxos` - Get UTxO details for requested addresses [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/asset_utxos` - Get UTxO details for requested assets [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/script_utxos` - Get UTxO details for requested script hashes [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/utxo_info` - Details for requested UTxO arrays [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/script_info` - Information about a given script FROM script hashes [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `/ogmios/?EvaluateTransaction` - Expose [Ogmios EvaluateTransaction](https://ogmios.dev/api/#operation-publish-/?EvaluateTransaction) as an endpoint
+- `/ogmios/?SubmitTransaction` - Expose [Ogmios SubmitTransaction](https://ogmios.dev/api/#operation-publish-/?SubmitTransaction) as an endpoint
+
+### Data Input/Output Changes:
+- Input - `/account_utxos` , `/credential_utxos` - Accept `extended` as an additional flag - which enables `asset_list`, `reference_script` and `inline_datum` to the output [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/block_txs` - Flatten output with transaction details (`tx_hash`, `epoch_no`, `block_height`, `block_time`) instead of `tx_hashes` array [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/epoch_params` - Update `cost_models` to JSON (upstream change in node) [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/account_assets` , `/address_assets` - Flatten the output result (instead of `asset_list` array) making it easier to apply horizontal filtering based on any of the fields
+- Output - Align output fields for `/account_utxos` , `/address_utxos`, `/asset_utxos` , `/script_utxos` and `/utxo_info` to return same schema giving complete details about UTxOs involved, with few fields toggled based on `extended` input flag [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/pool_list` - Add various details to the endpoint for each pool (`pool_id_hex`,`active_epoch_no`,`margin`,`fixed_cost`,`pledge`,`reward_addr`,`owners`,`relays`,`ticker`,`meta_url`,`meta_hash`,`pool_status`,`retiring_epoch`) - this should mean *some* of the requests to `pool_info` should no longer be required [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/pool_updates` - In v0, `pool_updates` only provided pool registration updates, while `pool_status` corresponded to current status of pool. With v1, we will have registration as well as deregistration transactions, and each transaction will have `update_type` (enum of either `registration` or `deregistration`) instead of `pool_status`. As a side-effect, since a registration transaction only has `retiring_epoch` as metadata, all the other fields will show up as `null` for such a transaction [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/pool_metadata` , `/pool_relays` - Add `pool_status` field to denote whether pool is retired [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/datum_info` - Rename `hash` to `datum_hash` and add `creation_tx_hash` [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/native_script_list` - Remove `script` column (as it has pretty large output better queried against `script_info`), add `size` and change `type` to text [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/plutus_script_list` - Add `type` and `size` to output [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Output - `/asset_info` - Add `cip68_metadata` JSONB field [#239](https://github.com/cardano-community/koios-artifacts/pull/227)
+
+### Deprecations:
+- `/tx_utxos` - No longer required as replaced by `/utxo_info` [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+
+### Chores:
+- Update base version to `v1` from `v0` [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Allow Bearer Authentication against endpoints [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Cron job to apply corrections to epoch info [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- `epoch_info_cache` Remove protocol parameters, as they can be queried from live table. Accordingly update dependent queries [#227](https://github.com/cardano-community/koios-artifacts/pull/227)
+- Make use of new `consumed_by_tx_in_id` column in `tx_out` from dbsync 13.1.1.3 across endpoints.
+
 ## [1.0.10] - For all networks.
 
 The release is effectively same as `1.0.10rc` except with one minor modification below.

scripts/cnode-helper-scripts/env

@@ -34,7 +34,7 @@
 #TIMEOUT_LEDGER_STATE=300                               # Timeout in seconds for querying and dumping ledger-state
 #IP_VERSION=4                                           # The IP version to use for push and fetch, valid options: 4 | 6 | mix (Default: 4)
 #ENABLE_KOIOS=Y                                         # (Y|N) Enable KOIOS API. If disabled, local node queries will be used instead with increased system resource requirements (default Y)
-#KOIOS_API="https://api.koios.rest/api/v0"              # Koios API for blockchain queries instead of local cli lookup.
+#KOIOS_API="https://api.koios.rest/api/v1"              # Koios API for blockchain queries instead of local cli lookup.
                                                         # Leave commented for automatic network detection between MainNet, Preview, Preprod or Guild network.
                                                         # https://www.koios.rest/
 #DBSYNC_QUERY_FOLDER="${CNODE_HOME}/files/dbsync/queries" # [advanced feature] Folder containing DB-Sync chain analysis queries
@@ -1010,19 +1010,19 @@ case ${NWMAGIC} in
   764824073) 
     NETWORK_NAME="Mainnet"
     SHELLEY_TRANS_EPOCH=208
-    [[ -z ${KOIOS_API} ]] && KOIOS_API="https://api.koios.rest/api/v0" ;;
+    [[ -z ${KOIOS_API} ]] && KOIOS_API="https://api.koios.rest/api/v1" ;;
   1)
     NETWORK_NAME="PreProd"
     SHELLEY_TRANS_EPOCH=4
-    [[ -z ${KOIOS_API} ]] && KOIOS_API="https://preprod.koios.rest/api/v0";;
+    [[ -z ${KOIOS_API} ]] && KOIOS_API="https://preprod.koios.rest/api/v1";;
   141) 
     NETWORK_NAME="Guild"
     SHELLEY_TRANS_EPOCH=2
-    [[ -z ${KOIOS_API} ]] && KOIOS_API="https://guild.koios.rest/api/v0" ;;
+    [[ -z ${KOIOS_API} ]] && KOIOS_API="https://guild.koios.rest/api/v1";;
   2)
     NETWORK_NAME="Preview"
     SHELLEY_TRANS_EPOCH=0
-    [[ -z ${KOIOS_API} ]] && KOIOS_API="https://preview.koios.rest/api/v0" ;;
+    [[ -z ${KOIOS_API} ]] && KOIOS_API="https://preview.koios.rest/api/v1";;
   *) 
     NETWORK_NAME="Custom"
     [[ -z ${SHELLEY_TRANS_EPOCH} ]] && SHELLEY_TRANS_EPOCH=0

scripts/cnode-helper-scripts/guild-deploy.sh

@@ -429,7 +429,7 @@ download_ogmios() {
   if command -v ogmios >/dev/null; then ogmios_version="$(ogmios --version)"; else ogmios_version="v0.0.0"; fi
   rm -rf /tmp/ogmios && mkdir /tmp/ogmios
   pushd /tmp/ogmios >/dev/null || err_exit
-  ogmios_asset_url="$(curl -s https://api.github.com/repos/CardanoSolutions/ogmios/releases/latest | jq -r '.assets[].browser_download_url')"
+  ogmios_asset_url="$(curl -s https://api.github.com/repos/CardanoSolutions/ogmios/releases | jq -r '.[0].assets[].browser_download_url')"
   if curl -sL -f -m ${CURL_TIMEOUT} -o ogmios.zip ${ogmios_asset_url}; then
     unzip ogmios.zip &>/dev/null
     rm -f ogmios.zip

scripts/grest-helper-scripts/grest-poll.sh

@@ -119,7 +119,7 @@ function chk_tip() {
 }
 
 function chk_rpc_struct() {
-  srvr_spec="$(curl -skL "${1}" | jq 'leaf_paths as $p | [$p[] | tostring] |join(".")' 2>/dev/null)"
+  srvr_spec="$(curl -skL "${1}" | jq '[leaf_paths as $p | { "key": $p | map(tostring) | join("_"), "value": getpath($p) }] | from_entries' | awk '{print $1 " " $2}' | grep -e ^\"paths -e ^\"parameters -e ^\"definitions 2>/dev/null)"
   api_endpts="$(grep ^\ \ / "${LOCAL_SPEC}" | sed -e 's#  /#/#g' -e 's#:##' | sort)"
   for endpt in ${api_endpts}
   do
@@ -155,7 +155,7 @@ function chk_cache_status() {
   else
       [[ -z "${GENESIS_JSON}" ]] && GENESIS_JSON="${PARENT}"/../files/shelley-genesis.json
       epoch_length=$(jq -r .epochLength "${GENESIS_JSON}" 2>/dev/null)
-      if [[ ${epoch_slot} -ge $(( epoch_length / 10 )) ]]; then
+      if [[ ${epoch_slot} -ge $(( epoch_length / 8 )) ]]; then
         if [[ "${last_actvstake_epoch}" != "${epoch}" ]]; then
           log_err "Active Stake cache for epoch ${epoch} still not populated as of ${epoch_slot} slot, maximum tolerance was $(( epoch_length / 10 )) !!"
           optexit
@@ -194,7 +194,7 @@ function chk_endpt_post() {
 }
 
 function chk_asset_registry() {
-  ct=$(curl -sfkL -H 'Prefer: count=exact' "${GURL}/asset_registry_cache?select=name&limit=1" -I 2>/dev/null | grep -i "content-range" | cut -d/ -f2 | tr -d '[:space:]')
+  ct=$(curl -sfkL -H 'Prefer: count=exact' "${GURL}/asset_token_registry?select=asset_name&limit=1" -I 2>/dev/null | grep -i "content-range" | cut -d/ -f2 | tr -d '[:space:]')
   if [[ "${ct}" == "" ]] || [[ $ct -lt 150 ]]; then
     log_err "Asset registry cache seems incomplete (<150) assets, try deleting key: asset_registry_commit in control_table and wait for next cron run"
     optexit

scripts/grest-helper-scripts/setup-grest.sh

@@ -16,7 +16,7 @@
 # Do NOT modify code below           #
 ######################################
 
-SGVERSION=v1.0.11rc
+SGVERSION=v1.1.0rc
 
 ######## Functions ########
   usage() {
@@ -275,7 +275,8 @@ SGVERSION=v1.0.11rc
   common_update() {
     # Create skeleton whitelist URL file if one does not already exist using most common option
     curl -sfkL "https://${KOIOS_SRV}/koiosapi.yaml" -o "${CNODE_HOME}"/files/koiosapi.yaml 2>/dev/null
-    grep " #RPC" "${CNODE_HOME}"/files/koiosapi.yaml | sed -e 's#^  /#/#' | cut -d: -f1 | sort > "${CNODE_HOME}"/files/grestrpcs 2>/dev/null
+    grep "^  /" "${CNODE_HOME}"/files/koiosapi.yaml | grep -v -e submittx -e "#RPC" | sed -e 's#^  /#/#' | cut -d: -f1 | sort > "${CNODE_HOME}"/files/grestrpcs 2>/dev/null
+    echo "/control_table" >> "${CNODE_HOME}"/files/grestrpcs 2>/dev/null
     checkUpdate grest-poll.sh Y N N grest-helper-scripts >/dev/null
     sed -i "s# API_STRUCT_DEFINITION=\"https://api.koios.rest/koiosapi.yaml\"# API_STRUCT_DEFINITION=\"https://${KOIOS_SRV}/koiosapi.yaml\"#g" grest-poll.sh
     checkUpdate checkstatus.sh Y N N grest-helper-scripts >/dev/null
@@ -341,9 +342,9 @@ SGVERSION=v1.0.11rc
   }
 
   deploy_haproxy() {
-    printf "[Re]Installing HAProxy.."
+    printf "\n[Re]Installing HAProxy.."
     pushd ~/tmp >/dev/null || err_exit
-    haproxy_url="http://www.haproxy.org/download/2.6/src/haproxy-2.6.5.tar.gz"
+    haproxy_url="http://www.haproxy.org/download/2.8/src/haproxy-2.8.3.tar.gz"
     if curl -sL -f -m ${CURL_TIMEOUT} -o haproxy.tar.gz "${haproxy_url}"; then
       tar xf haproxy.tar.gz &>/dev/null && rm -f haproxy.tar.gz
       if command -v apt-get >/dev/null; then
@@ -352,7 +353,7 @@ SGVERSION=v1.0.11rc
       if command -v yum >/dev/null; then
         sudo yum -y install pcre-devel >/dev/null || err_exit "'sudo yum -y install prce-devel' failed!"
       fi
-      cd haproxy-2.6.5 || return
+      cd haproxy-2.8.3 || return
       make clean >/dev/null
       make -j $(nproc) TARGET=linux-glibc USE_ZLIB=1 USE_LIBCRYPT=1 USE_OPENSSL=1 USE_PCRE=1 USE_SYSTEMD=1 USE_PROMEX=1 >/dev/null
       sudo make install-bin >/dev/null
@@ -440,23 +441,27 @@ SGVERSION=v1.0.11rc
 			  bind 0.0.0.0:8053
 			  ## If using SSL, comment line above and uncomment line below
 			  #bind :8453 ssl crt /etc/ssl/server.pem no-sslv3
+			  compression direction response
+			  compression algo-res gzip
+			  compression type-res application/json
+			  option http-buffer-request
 			  http-request set-log-level silent
 			  acl srv_down nbsrv(grest_postgrest) eq 0
 			  acl is_wss hdr(Upgrade) -i websocket
 			  http-request use-service prometheus-exporter if { path /metrics }
 			  http-request track-sc0 src table flood_lmt_rate
-			  http-request deny deny_status 429 if { sc_http_req_rate(0) gt 250 }
-			  use_backend ogmios if { path_beg /api/v0/ogmios } || { path_beg /dashboard.js } || { path_beg /assets } || { path_beg /health } || is_wss
-			  use_backend submitapi if { path_beg /api/v0/submittx }
+			  http-request deny deny_status 429 if { sc_http_req_rate(0) gt 500 }
+			  use_backend ogmios if { path_beg /api/ogmios } || { path_beg /dashboard.js } || { path_beg /assets } || { path_beg /health } || is_wss
+			  use_backend submitapi if { path_beg /api/v1/submittx }
 			  use_backend grest_failover if srv_down
 			  default_backend grest_postgrest
 
 			backend grest_postgrest
 			  balance first
 			  #option external-check
-			  acl grestrpcs path_beg -f \"\\\$GRESTTOP\"/files/grestrpcs
-			  http-request set-path \"%[path,regsub(^/api/v0/,/)]\"
-			  http-request set-path \"%[path,regsub(^/,/rpc/)]\" if grestrpcs
+			  acl grestviews path_beg -f \"\\\$GRESTTOP\"/files/grestrpcs
+			  http-request set-path \"%[path,regsub(^/api/v1/,/)]\"
+			  http-request set-path \"%[path,regsub(^/,/rpc/)]\" if !grestviews !{ path_beg /rpc } !{ path -i / }
 			  #external-check path \"/usr/bin:/bin:/tmp:/sbin:/usr/sbin\"
 			  #external-check command \"\\\$GRESTTOP\"/scripts/grest-poll.sh
 			  server local 127.0.0.1:8050 check inter 20000 fall 1 rise 2
@@ -468,7 +473,7 @@ SGVERSION=v1.0.11rc
 
 			backend ogmios
 			  balance first
-			  http-request set-path \"%[path,regsub(^/api/v0/ogmios/,/)]\"
+			  http-request set-path \"%[path,regsub(^/api/ogmios/,/)]\"
 			  option httpchk GET /health
 			  http-check expect status 200
 			  default-server inter 20s fall 1 rise 2
@@ -477,7 +482,7 @@ SGVERSION=v1.0.11rc
 			backend submitapi
 			  balance first
 			  option httpchk POST /api/submit/tx
-			  http-request set-path \"%[path,regsub(^/api/v0/submittx,/api/submit/tx)]\"
+			  http-request set-path \"%[path,regsub(^/api/v1/submittx,/api/submit/tx)]\"
 			  http-check expect status 415
 			  default-server inter 20s fall 1 rise 2
 			  server local 127.0.0.1:8090 check