diff --git a/.github/workflows/docker_bin.yml b/.github/workflows/docker_bin.yml index a56fe0bf6..f6c164e1e 100644 --- a/.github/workflows/docker_bin.yml +++ b/.github/workflows/docker_bin.yml @@ -15,6 +15,7 @@ on: push: paths: - 'files/docker/node/release-versions/cardano-node-latest.txt' + - 'files/docker/node/release-versions/mithril-latest.txt' jobs: build: @@ -30,13 +31,14 @@ jobs: run: | echo "G_ACCOUNT=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV echo "CNVERSION=$(cat files/docker/node/release-versions/cardano-node-latest.txt)" >> $GITHUB_ENV + echo "PUSH_TO_GA=false" >> $GITHUB_ENV - name: Compiling new node software suite run: | DOCKER_BUILDKIT=1 docker build . \ --file files/docker/node/dockerfile_bin \ --compress \ --build-arg G_ACCOUNT=${{ env.G_ACCOUNT }} \ - --build-arg GUILD_DEPLOY_BRANCH=${{ env.guild_deploy_branch }} \ + --build-arg GUILD_DEPLOY_BRANCH=${{ github.event.inputs.guild_deploy_branch }} \ --tag ${{ env.REGISTRY }}/${{ secrets.DOCKER_USER }}/cardano-node:latest # Workaround to provide additional free space for builds. # https://github.com/actions/virtual-environments/issues/2840 @@ -53,9 +55,10 @@ jobs: sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" - name: docker push latest - if: env.testing == 'false' && env.guild_deploy_branch == 'master' + if: github.event.inputs.testing == 'false' && github.event.inputs.guild_deploy_branch == 'master' run: | CNVERSION=`cat files/docker/node/release-versions/cardano-node-latest.txt` + echo "PUSH_TO_GA=true" >> $GITHUB_ENV docker push ${{ env.REGISTRY }}/${{ secrets.DOCKER_USER }}/cardano-node:latest docker tag ${{ env.REGISTRY }}/${{ secrets.DOCKER_USER }}/cardano-node:latest ${{ secrets.DOCKER_USER }}/cardano-node:${{ env.CNVERSION }} docker push ${{ env.REGISTRY }}/${{ secrets.DOCKER_USER }}/cardano-node:${{ env.CNVERSION }} @@ -65,4 +68,6 @@ jobs: echo "## Summary Details" >> $GITHUB_STEP_SUMMARY echo "* Docker Image: ${{ env.REGISTRY }}/${{ secrets.DOCKER_USER }}/cardano-node:${{ env.CNVERSION }}" >> $GITHUB_STEP_SUMMARY echo "* G_ACCOUNT: ${GITHUB_REPOSITORY_OWNER}" >> $GITHUB_STEP_SUMMARY + echo "* GUILD_DEPLOY_BRANCH: ${{ github.event.inputs.guild_deploy_branch }}" >> $GITHUB_STEP_SUMMARY + echo "* Push to GA Registry: ${{ env.PUSH_TO_GA }}" >> $GITHUB_STEP_SUMMARY echo "* CNVERSION: ${{ env.CNVERSION }}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/mithril-latest.yml b/.github/workflows/mithril-latest.yml new file mode 100644 index 000000000..3fd578b8c --- /dev/null +++ b/.github/workflows/mithril-latest.yml @@ -0,0 +1,30 @@ +name: Get latest Mithril release version +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' +jobs: + get-version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + token: ${{ secrets.REPO_SCOPED_TOKEN }} + fetch-depth: 0 + ref: alpha + - name: Fetch Mithril release version + run: | + curl -sL https://api.github.com/repos/input-output-hk/mithril/releases/latest | jq -r '.tag_name' > files/docker/node/release-versions/mithril-latest.txt + - name: Assigns release version + run: | + VERSION=$(cat ./files/docker/node/release-versions/mithril-latest.txt) + - name: Check for modified files + id: git-check + run: echo ::set-output name=modified::$([ -z "`git status --porcelain`" ] && echo "false" || echo "true") + - name: Commit latest release version + if: steps.git-check.outputs.modified == 'true' + run: | + git config --global user.name ${{ secrets.REPO_SCOPED_USER }} + git config --global user.email ${{ secrets.REPO_SCOPED_EMAIL }} + git commit -am "New mithril release version ${VERSION}" + git push diff --git a/docs/Build/grest-changelog.md b/docs/Build/grest-changelog.md index 95848b0c6..d831bdcc9 100644 --- a/docs/Build/grest-changelog.md +++ b/docs/Build/grest-changelog.md @@ -1,5 +1,23 @@ # Koios gRest Changelog +## [1.1.0] - For all networks. + +This will be first major [breaking] release for Koios consumers in a while, and will be rolled out under new base prefix (`/api/v1`). +The major work with this release was to start making use of newer flags in dbsync which help performance of queries under new endpoints. Please ensure to check out the release notes for `1.1.0rc` below. The list for this section is only a small addendum to `1.1.0rc`: + +### Chores: +- Make use of asset-txo-cache for top assets on mainnet, and use this cache for `asset_addresses` and `policy_asset_addresses` [#250](https://github.com/cardano-community/koios-artifacts/pull/250) +- Add v0 RPC redirectors to keep serve v0 endpoints from v1 [#250](https://github.com/cardano-community/koios-artifacts/pull/250) +- Convert few simple RPC functions from PLPGSQL to SQL language to help with inline filtering [#250](https://github.com/cardano-community/koios-artifacts/pull/250) +- Address linting results from SQLFluff [#250](https://github.com/cardano-community/koios-artifacts/pull/250) +- Move db-scripts from guild-operators repository to koios-artifacts repository [#250](https://github.com/cardano-community/koios-artifacts/pull/250) +- Drop stale db-scripts/genesis_table.sql file [#250](https://github.com/cardano-community/koios-artifacts/pull/250) +- Add 3 additional indexes for collateral and reference inputs based on query times [#250](https://github.com/cardano-community/koios-artifacts/pull/250) +- Add top 3 assets for preview/preprod to asset-txo-cache [#250](https://github.com/cardano-community/koios-artifacts/pull/250) +- Bump schema version for koios-1.1.0 [#250](https://github.com/cardano-community/koios-artifacts/pull/250) +- Minor patch for output data type (`pool_registrations` and `pool_retirements`) [#249](https://github.com/cardano-community/koios-artifacts/pull/249) + + ## [1.1.0rc] - For all networks. This will be first major [breaking] release for Koios consumers in a while, and will be rolled out under new base prefix (`/api/v1`). diff --git a/docs/Scripts/gliveview.md b/docs/Scripts/gliveview.md index 56eb8ead2..9ac26cad7 100644 --- a/docs/Scripts/gliveview.md +++ b/docs/Scripts/gliveview.md @@ -9,6 +9,9 @@ For most setups, it's enough to set `CNODE_PORT` in the `env` file. The rest of The tool can be run in legacy mode with only standard ASCII characters for terminals with trouble displaying the box-drawing characters. Run `./gLiveView.sh -h` to show available command-line parameters or permanently set it directly in script. +!!! info "Note !!" + Keeping gLiveView to it's intent of being a dashboard and not a full-fledged monitoring tool, we intend to keep most relevant information for a node operator in a minimalistic dashboard, accordingly - gLiveView runs by default in compact mode. One can enable verbose mode by pressing 'v' to unhide additional fields. + A sample output from both core and relay together with peer analysis: === "Core" @@ -23,7 +26,6 @@ A sample output from both core and relay together with peer analysis: ![Peer-Analysis](https://raw.githubusercontent.com/cardano-community/guild-operators/images/glv-peers.png ':size=35%') - ###### Upper main section Displays live metrics from cardano-node gathered through the nodes EKG/Prometheus(env setting) endpoint. diff --git a/scripts/cnode-helper-scripts/cntools.library b/scripts/cnode-helper-scripts/cntools.library index 32d638aad..d75b0034c 100644 --- a/scripts/cnode-helper-scripts/cntools.library +++ b/scripts/cnode-helper-scripts/cntools.library @@ -149,6 +149,7 @@ protectionPreRequisites() { fi rm -f "${TMP_DIR}"/test fi + return 0 } # Command : safeDel [path] diff --git a/scripts/cnode-helper-scripts/env b/scripts/cnode-helper-scripts/env index 6fa113ba6..1f018c9ba 100644 --- a/scripts/cnode-helper-scripts/env +++ b/scripts/cnode-helper-scripts/env @@ -933,19 +933,22 @@ if ! command -v "jq" &>/dev/null; then return 1 fi -read -ra CONFIG_CONTENTS <<<"$(jq -r '[ .AlonzoGenesisFile //"null", .ByronGenesisFile //"null", .ShelleyGenesisFile //"null", .Protocol //"Cardano", .TraceChainDb //"null", .EnableP2P //"false"]| @tsv' "${CONFIG}" 2>/dev/null)" +read -ra CONFIG_CONTENTS <<<"$(jq -r '[ .AlonzoGenesisFile //"null", .ByronGenesisFile //"null", .ShelleyGenesisFile //"null", .Protocol //"Cardano", .TraceChainDb //"null", .EnableP2P //"false", .ConwayGenesisFile //"null"]| @tsv' "${CONFIG}" 2>/dev/null)" if [[ ${CONFIG_CONTENTS[4]} != "true" ]]; then echo "Could not find TraceChainDb when attempting to parse ${CONFIG} file in JSON format, please double-check the syntax of your config, or simply download it from guild-operators repository!" return 1 else + CONWAY_GENESIS_JSON="${CONFIG_CONTENTS[6]}" ALONZO_GENESIS_JSON="${CONFIG_CONTENTS[0]}" BYRON_GENESIS_JSON="${CONFIG_CONTENTS[1]}" GENESIS_JSON="${CONFIG_CONTENTS[2]}" # if relative path is used, assume same parent dir as config + [[ ! ${CONWAY_GENESIS_JSON} =~ ^/ ]] && CONWAY_GENESIS_JSON="$(dirname "${CONFIG}")/${CONWAY_GENESIS_JSON}" [[ ! ${ALONZO_GENESIS_JSON} =~ ^/ ]] && ALONZO_GENESIS_JSON="$(dirname "${CONFIG}")/${ALONZO_GENESIS_JSON}" [[ ! ${BYRON_GENESIS_JSON} =~ ^/ ]] && BYRON_GENESIS_JSON="$(dirname "${CONFIG}")/${BYRON_GENESIS_JSON}" [[ ! ${GENESIS_JSON} =~ ^/ ]] && GENESIS_JSON="$(dirname "${CONFIG}")/${GENESIS_JSON}" # if genesis files not found, exit with rc 1 + [[ ! -f "${CONWAY_GENESIS_JSON}" ]] && echo "Byron genesis file not found: ${CONWAY_GENESIS_JSON}" && return 1 [[ ! -f "${ALONZO_GENESIS_JSON}" ]] && echo "Byron genesis file not found: ${ALONZO_GENESIS_JSON}" && return 1 [[ ! -f "${BYRON_GENESIS_JSON}" ]] && echo "Byron genesis file not found: ${BYRON_GENESIS_JSON}" && return 1 [[ ! -f "${GENESIS_JSON}" ]] && echo "Shelley genesis file not found: ${GENESIS_JSON}" && return 1 diff --git a/scripts/cnode-helper-scripts/gLiveView.sh b/scripts/cnode-helper-scripts/gLiveView.sh index 4b5927830..bcd343f1b 100755 --- a/scripts/cnode-helper-scripts/gLiveView.sh +++ b/scripts/cnode-helper-scripts/gLiveView.sh @@ -59,7 +59,7 @@ setTheme() { # Do NOT modify code below # ###################################### -GLV_VERSION=v1.28.1 +GLV_VERSION=v1.28.3 PARENT="$(dirname $0)" @@ -515,6 +515,11 @@ getOpCert () { op_cert_disk="?" op_cert_node="?" opcert_file="${POOL_DIR}/${POOL_OPCERT_FILENAME}" + if [[ ! -f ${opcert_file} && -n ${CNODE_PID} ]]; then + if [[ $(ps -p ${CNODE_PID} -o cmd=) =~ --shelley-operational-certificate[[:space:]]([^[:space:]]+) ]]; then + opcert_file="${BASH_REMATCH[1]}" + fi + fi if [[ -f ${opcert_file} ]]; then op_cert_tsv=$(jq -r '[ .qKesNodeStateOperationalCertificateNumber //"?", @@ -823,7 +828,7 @@ while true; do cpu_util="0.0" fi if [[ ${about_to_lead} -gt 0 ]]; then - [[ ${nodemode} != "Core" ]] && clrScreen && nodemode="Core" + [[ ${nodemode} != "Core" ]] && nodemode="Core" && getOpCert && clrScreen else [[ ${nodemode} != "Relay" ]] && clrScreen && nodemode="Relay" fi diff --git a/scripts/cnode-helper-scripts/guild-deploy.sh b/scripts/cnode-helper-scripts/guild-deploy.sh index 7df4081df..0c9041ba0 100755 --- a/scripts/cnode-helper-scripts/guild-deploy.sh +++ b/scripts/cnode-helper-scripts/guild-deploy.sh @@ -493,10 +493,7 @@ download_cardanosigner() { download_mithril() { echo -e "\nDownloading Mithril..." pushd "${HOME}"/tmp >/dev/null || err_exit - # dynamic latest release updated automatically, uncomment and comment out the hardcoded release below if needed - # mithril_release="$(curl -s https://api.github.com/repos/input-output-hk/mithril/releases/latest | jq -r '.tag_name')" - # hardcoded latest release requiring a bump - mithril_release="2342.0" + mithril_release="$(curl -s https://api.github.com/repos/input-output-hk/mithril/releases/latest | jq -r '.tag_name')" echo -e "\n Downloading Mithril Signer/Client ${mithril_release}..." rm -f mithril-signer mithril-client curl -m 200 -sfL https://github.com/input-output-hk/mithril/releases/download/${mithril_release}/mithril-${mithril_release}-linux-x64.tar.gz -o mithril.tar.gz || err_exit " Could not download mithril's latest release archive from IO github!" diff --git a/scripts/grest-helper-scripts/setup-grest.sh b/scripts/grest-helper-scripts/setup-grest.sh index cc3420a33..51484dcb3 100755 --- a/scripts/grest-helper-scripts/setup-grest.sh +++ b/scripts/grest-helper-scripts/setup-grest.sh @@ -197,7 +197,7 @@ SGVERSION=v1.1.0 # Description : Remove all grest-related cron entries. remove_all_grest_cron_jobs() { printf "\nRemoving all installed cron jobs..." - grep -rl ${CRON_SCRIPTS_DIR} ${CRON_DIR} | xargs rm -f + grep -rl ${CRON_SCRIPTS_DIR} ${CRON_DIR} | xargs sudo rm -f rm -f ${CRON_SCRIPTS_DIR}/*.sh psql "${PGDATABASE}" -qt -c "SELECT PG_CANCEL_BACKEND(pid) FROM pg_stat_activity WHERE usename='${USER}' AND application_name = 'psql' AND query NOT LIKE '%pg_stat_activity%';" psql "${PGDATABASE}" -qt -c "SELECT PG_TERMINATE_BACKEND(pid) FROM pg_stat_activity WHERE usename='${USER}' AND application_name = 'psql' AND query NOT LIKE '%pg_stat_activity%';" @@ -302,7 +302,8 @@ SGVERSION=v1.1.0 else pgrest_binary=linux-static-x64.tar.xz fi - pgrest_asset_url="$(curl -s https://api.github.com/repos/PostgREST/postgrest/releases/latest | jq -r '.assets[].browser_download_url' | grep ${pgrest_binary})" + #pgrest_asset_url="$(curl -s https://api.github.com/repos/PostgREST/postgrest/releases/latest | jq -r '.assets[].browser_download_url' | grep ${pgrest_binary})" + pgrest_asset_url="https://github.com/PostgREST/postgrest/releases/download/v11.2.2/postgrest-v11.2.2-linux-static-x64.tar.xz" # Fix PostgREST to v11.2.2 until v12 headers are updated if curl -sL -f -m ${CURL_TIMEOUT} -o postgrest.tar.xz "${pgrest_asset_url}"; then tar xf postgrest.tar.xz &>/dev/null && rm -f postgrest.tar.xz [[ -f postgrest ]] || err_exit "PostgREST archive downloaded but binary not found after attempting to extract package!" @@ -614,7 +615,7 @@ SGVERSION=v1.1.0 printf "\n (Re)Deploying GRest objects to DBSync..." populate_genesis_table for row in $(jq -r '.[] | @base64' <<<${rpc_file_list}); do - if [[ $(jqDecode '.type' "${row}") = 'dir' ]]; then + if [[ $(jqDecode '.type' "${row}") = 'dir' ]] && [[ $(jqDecode '.name' "${row}") != 'db-scripts' ]]; then printf "\n Downloading pSQL executions from subdir $(jqDecode '.name' "${row}")" if ! rpc_file_list_subdir=$(curl -s -m ${CURL_TIMEOUT} "https://api.github.com/repos/${G_ACCOUNT}/koios-artifacts/contents/files/grest/rpc/$(jqDecode '.name' "${row}")?ref=${SGVERSION}"); then printf "\n \e[31mERROR\e[0m: ${rpc_file_list_subdir}" && continue @@ -667,7 +668,7 @@ SGVERSION=v1.1.0 [[ "${INSTALL_MONITORING_AGENTS}" == "Y" ]] && deploy_monitoring_agents [[ "${OVERWRITE_CONFIG}" == "Y" ]] && deploy_configs [[ "${OVERWRITE_SYSTEMD}" == "Y" ]] && deploy_systemd - [[ "${RESET_GREST}" == "Y" ]] && remove_all_grest_cron_jobs && setup_db_basics && reset_grest + [[ "${RESET_GREST}" == "Y" ]] && remove_all_grest_cron_jobs && reset_grest [[ "${DB_QRY_UPDATES}" == "Y" ]] && remove_all_grest_cron_jobs && setup_db_basics && deploy_query_updates && update_grest_version pushd -0 >/dev/null || err_exit dirs -c