Provisioning script updates (#1774)

## Description
<!--- Describe your changes -->

### Guild Deploy changes:
- Only run build dependencies as part of os dependencies when included
as flag
- Adapt libsodium flag to only use os_dependencies (and include minimal
set of dependency there) to make it faster/lighter to deploy libsodium
fork on block producing nodes
- Include os_dependencies prior to running cardano-hw-cli download (to
ensure libusb dependencies are available)
- Allow CARDANO_NODE_VERSION to be set as environment variable for those
who'd like to test guild-deploy against different node release
- Bump CARDANO_NODE_VERSION to 8.9.3

### files/docker:
- Remove docker stage files
- Remove cardano-wallet, as it is no longer relevant in the ecosystem
now
- for dockerfile_bin, remove tcpping (was unused due to comment), and
remove cleanup references for build packages, as they're no longer
deployed with `-s p` flag. Also, remove gnupg git from initial run, as
they're already included as part of `guild-deploy.sh -s -p` call

### setup-grest.sh:
- Dont use `command -v haproxy` (as the shell isnt always refreshed and
/usr/sbin may not be part of $PATH), instead look for /usr/sbin/haproxy
to verify if haproxy was already installed
- Add haproxy installer dependencies (if guild-deploy is only run with
os_dependencies flag, haproxy wouldnt compile in it's current state)

.github/workflows/premerge.yml

@@ -10,7 +10,7 @@ on:
     paths:
       - scripts/cnode-helper-scripts/guild-deploy.sh
       - scripts/cnode-helper-scripts/cabal-build-all.sh
-      - files/tests/pre-merge/debian-guild-deploy.sh-l.containerfile
+      - files/tests/pre-merge/ubuntu-guild-deploy.sh.containerfile
 
 jobs:
   guild-deploy-and-build:
@@ -61,10 +61,10 @@ jobs:
           echo "This means the PR is from the same repository"
           echo "BUILD_G_ACCOUNT=${{ github.repository_owner }}" >> $GITHUB_ENV
         fi
-    - name: Testing guild-deploy.sh (IO fork of libsodium)
+    - name: Testing guild-deploy.sh (with libsodium)
       run: |
         docker build . \
-          --file files/tests/pre-merge/${{ matrix.distro }}-guild-deploy.sh-l.containerfile \
+          --file files/tests/pre-merge/${{ matrix.distro }}-guild-deploy.sh.containerfile \
           --compress \
           --build-arg BRANCH=$BRANCH \
           --build-arg COMMIT=$COMMIT_SHA \
@@ -77,11 +77,11 @@ jobs:
     - name: Push pre-merge-${{ matrix.distro }}:guild-deploy-l_${{ env.COMMIT_SHA }}
       run: |
         docker push ${{ env.REGISTRY }}/${{ env.G_ACCOUNT }}/pre-merge-${{ matrix.distro }}:guild-deploy-l_${{ env.COMMIT_SHA }}
-    - name: Testing cabal-build-all.sh (IO fork of libsodium)
+    - name: Testing cabal-build-all.sh (with libsodium)
       run: |
         echo "Working from PR Branch ${{ env.G_ACCOUNT }}/guild-operators/${{ env.BRANCH }} on COMMIT_SHA ${{ env.COMMIT_SHA }}"
         docker build . \
-        --file files/tests/pre-merge/${{ matrix.distro }}-cabal-l.containerfile \
+        --file files/tests/pre-merge/${{ matrix.distro }}-cabal.containerfile \
         --compress \
         --build-arg BRANCH=${{ env.BRANCH }} \
         --build-arg COMMIT=${{ env.COMMIT_SHA }} \

files/docker/node/dockerfile_bin

@@ -23,7 +23,7 @@ ENV \
     UPDATE_CHECK=N
 
 RUN apt-get update && apt-get install --no-install-recommends -y locales apt-utils sudo \
-    && apt install -y curl wget gnupg git udev jq \
+    && apt install -y curl wget udev jq \
     && apt-get -y purge \
     && apt-get -y clean \
     && apt-get -y autoremove \
@@ -49,7 +49,6 @@ RUN set -x && apt update \
     && chmod +x ./guild-deploy.sh && ./guild-deploy.sh -b ${GUILD_DEPLOY_BRANCH} -s p \
     && ls /opt/ \
     && mkdir -p $CNODE_HOME/priv/files \
-    && apt-get -y remove libpq-dev build-essential pkg-config libffi-dev libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev make g++ \
     && apt-get -y purge \
     && apt-get -y clean \
     && apt-get -y autoremove \
@@ -59,9 +58,6 @@ RUN set -x && export SUDO='N' \
     && export SKIP_DBSYNC_DOWNLOAD='Y' \
     && export G_ACCOUNT=${G_ACCOUNT} \
     && ./guild-deploy.sh -b ${GUILD_DEPLOY_BRANCH} -s dcmowx \
-    && cd /usr/bin \
-    && wget http://www.vdberg.org/~richard/tcpping \
-    && chmod 755 tcpping \
     && chown -R guild:guild $CNODE_HOME \
     && mv /root/.local/bin /home/guild/.local/ \
     && chown -R guild:guild /home/guild/.*
@@ -98,7 +94,6 @@ RUN  curl -sL -H "Accept: application/vnd.github.everest-preview+json" -H "Conte
 
 # ENTRY SCRIPT
 ADD https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/${GUILD_DEPLOY_BRANCH}/files/docker/node/addons/banner.txt \
-    https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/${GUILD_DEPLOY_BRANCH}/files/docker/node/addons/guild-topology.sh \
     https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/${GUILD_DEPLOY_BRANCH}/files/docker/node/addons/block_watcher.sh \
     https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/${GUILD_DEPLOY_BRANCH}/files/docker/node/addons/healthcheck.sh /home/guild/.scripts/
 ADD https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/${GUILD_DEPLOY_BRANCH}/scripts/cnode-helper-scripts/guild-deploy.sh \

files/node-deps.json

@@ -5,7 +5,7 @@
     "secp256k1": "v0.3.2",
     "sodium": "dbb48cce5429cb6585c9034f002568964f1ce567"
   },
-  "8.9.2":
+  "8.9.3":
   {
     "blst": "3dd0f804b1819e5d03fb22ca2e6fac105932043a",
     "secp256k1": "v0.3.2",

scripts/cnode-helper-scripts/guild-deploy.sh

@@ -107,9 +107,9 @@ set_defaults() {
   [[ -z "${BRANCH}" ]] && BRANCH="master"
   [[ "${SUDO}" = 'Y' ]] && sudo="sudo" || sudo=""
   [[ "${SUDO}" = 'Y' && $(id -u) -eq 0 ]] && err_exit "Please run as non-root user."
+  [[ -z "${CARDANO_NODE_VERSION}" ]] && CARDANO_NODE_VERSION="8.9.3"
   CNODE_HOME="${CNODE_PATH}/${CNODE_NAME}"
   CNODE_VNAME=$(echo "$CNODE_NAME" | awk '{print toupper($0)}')
-  CARDANO_NODE_VERSION="8.9.2"
   REPO="https://github.com/${G_ACCOUNT}/guild-operators"
   REPO_RAW="https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators"
   URL_RAW="${REPO_RAW}/${BRANCH}"
@@ -204,13 +204,22 @@ os_dependencies() {
   if [[ "${OS_ID}" =~ ebian ]] || [[ "${OS_ID}" =~ buntu ]] || [[ "${DISTRO}" =~ ebian ]] || [[ "${DISTRO}" =~ buntu ]]; then
     #Debian/Ubuntu
     pkgmgrcmd="env NEEDRESTART_MODE=a env DEBIAN_FRONTEND=noninteractive env DEBIAN_PRIORITY=critical apt-get"
-    libncurses_pkg="libncursesw5"
-    [[ -f /etc/debian_version ]] && grep -q trixie /etc/debian_version && libncurses_pkg="libncursesw6"
-    pkg_list="python3 pkg-config libssl-dev ${libncurses_pkg} libtinfo-dev systemd libsystemd-dev libsodium-dev tmux git jq libtool bc gnupg libtool secure-delete iproute2 tcptraceroute sqlite3 bsdmainutils libusb-1.0-0-dev libudev-dev unzip llvm clang libnuma-dev libpq-dev build-essential libffi-dev libgmp-dev zlib1g-dev make g++ autoconf automake liblmdb-dev procps xxd"
+    pkg_list="python3 pkg-config systemd tmux git jq libtool bc gnupg libtool secure-delete iproute2 tcptraceroute sqlite3 bsdmainutils unzip procps xxd"
+    if [[ "${LIBSODIUM_FORK}" == "Y" ]] || [[ "${WANT_BUILD_DEPS}" == "Y" ]]; then
+      pkg_list="${pkg_list} build-essential make g++ autoconf automake"
+    fi
+    if [[ "${WANT_BUILD_DEPS}" == "Y" ]]; then
+      libncurses_pkg="libncursesw5"
+      [[ -f /etc/debian_version ]] && grep -q trixie /etc/debian_version && libncurses_pkg="libncursesw6"
+      pkg_list="${pkg_list} ${libncurses_pkg} libtinfo-dev libnuma-dev libpq-dev liblmdb-dev libffi-dev libgmp-dev libssl-dev libsystemd-dev libsodium-dev zlib1g-dev llvm clang"
+    fi
+    if [[ "${INSTALL_CWHCLI}" == "Y" ]]; then
+      pkg_list="${pkg_list} libusb-1.0-0-dev libudev-dev"
+    fi
   elif [[ "${OS_ID}" =~ rhel ]] || [[ "${OS_ID}" =~ fedora ]] || [[ "${DISTRO}" =~ Fedora ]]; then
     #CentOS/RHEL/Fedora/RockyLinux
     pkgmgrcmd="dnf"
-    pkg_list="python3 coreutils ncurses-devel ncurses-libs openssl-devel systemd systemd-devel libsodium-devel tmux git jq gnupg2 libtool iproute bc traceroute sqlite util-linux xz wget unzip procps-ng llvm clang numactl-devel libffi-devel gmp-devel zlib-devel make gcc-c++ autoconf udev lmdb-devel vim-common"
+    pkg_list="python3 coreutils systemd tmux git jq gnupg2 libtool iproute bc traceroute sqlite util-linux xz unzip procps-ng udev vim-common"
     if [[ "${VERSION_ID}" == "2" ]] ; then
       #AmazonLinux2
       pkg_list="${pkg_list} libusb ncurses-compat-libs pkgconfig srm"
@@ -228,14 +237,13 @@ os_dependencies() {
       pkg_opts="${pkg_opts} --allowerasing"
       pkg_list="${pkg_list} libusbx ncurses-compat-libs pkgconf-pkg-config srm"
     fi
-    add_epel_repository "${DISTRO}" "${VERSION_ID}" "${pkg_opts}"
-    if [ -f /usr/lib64/libtinfo.so ] && [ -f /usr/lib64/libtinfo.so.5 ]; then
-      echo -e "\n  Symlink updates not required for ncurse libs, skipping.."
-    else
-      echo -e "\n  Updating symlinks for ncurse libs.."
-      $sudo ln -s "$(find /usr/lib64/libtinfo.so* | tail -1)" /usr/lib64/libtinfo.so
-      $sudo ln -s "$(find /usr/lib64/libtinfo.so* | tail -1)" /usr/lib64/libtinfo.so.5
+    if [[ "${LIBSODIUM_FORK}" == "Y" ]] || [[ "${WANT_BUILD_DEPS}" == "Y" ]]; then
+      pkg_list="${pkg_list} make gcc-c++ autoconf automake"
     fi
+    if [[ "${WANT_BUILD_DEPS}" == "Y" ]]; then
+      pkg_list="${pkg_list} ncurses-libs ncurses-devel openssl-devel systemd-devel libsodium-devel llvm clang numactl-devel libffi-devel gmp-devel zlib-devel lmdb-devel"
+    fi
+    add_epel_repository "${DISTRO}" "${VERSION_ID}" "${pkg_opts}"
   else
     echo -e "\nWe have no automated procedures for this ${DISTRO} system"
     err_exit
@@ -253,6 +261,15 @@ os_dependencies() {
     echo -e "\nIt would be best if you could submit an issue at ${REPO} with the details to tackle in future, as some errors may be due to external/already present dependencies"
     err_exit
   fi
+  if [[ "${OS_ID}" =~ rhel ]] || [[ "${OS_ID}" =~ fedora ]] || [[ "${DISTRO}" =~ Fedora ]]; then
+    if [ -f /usr/lib64/libtinfo.so ] && [ -f /usr/lib64/libtinfo.so.5 ]; then
+      echo -e "\n  Symlink updates not required for ncurse libs, skipping.."
+    else
+      echo -e "\n  Updating symlinks for ncurse libs.."
+      $sudo ln -s "$(find /usr/lib64/libtinfo.so* | tail -1)" /usr/lib64/libtinfo.so
+      $sudo ln -s "$(find /usr/lib64/libtinfo.so* | tail -1)" /usr/lib64/libtinfo.so.5
+    fi
+  fi
 }
 
 # Build Dependencies for cabal builds
@@ -654,13 +671,13 @@ parse_args() {
   if [[ -n "${S_ARGS}" ]]; then
     [[ "${S_ARGS}" =~ "p" ]] && INSTALL_OS_DEPS="Y"
     [[ "${S_ARGS}" =~ "b" ]] && INSTALL_OS_DEPS="Y" && WANT_BUILD_DEPS="Y"
-    [[ "${S_ARGS}" =~ "l" ]] && INSTALL_OS_DEPS="Y" && WANT_BUILD_DEPS="Y" && INSTALL_LIBSODIUM_FORK="Y"
+    [[ "${S_ARGS}" =~ "l" ]] && INSTALL_OS_DEPS="Y" && INSTALL_LIBSODIUM_FORK="Y"
     [[ "${S_ARGS}" =~ "m" ]] && INSTALL_MITHRIL="Y"
     [[ "${S_ARGS}" =~ "f" ]] && FORCE_OVERWRITE="Y" && POPULATE_CNODE="F"
     [[ "${S_ARGS}" =~ "d" ]] && INSTALL_CNODEBINS="Y"
     [[ "${S_ARGS}" =~ "c" ]] && INSTALL_CNCLI="Y"
     [[ "${S_ARGS}" =~ "o" ]] && INSTALL_OGMIOS="Y"
-    [[ "${S_ARGS}" =~ "w" ]] && INSTALL_CWHCLI="Y"
+    [[ "${S_ARGS}" =~ "w" ]] && INSTALL_OS_DEPS="Y" && INSTALL_CWHCLI="Y"
     [[ "${S_ARGS}" =~ "x" ]] && INSTALL_CARDANO_SIGNER="Y"
   else
     echo -e "\nNothing to do.."