run

#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
DO_DRY_RUN="false"
HOLOHUB_PY_EXE=${HOLOHUB_PY_EXE:-python3}
HOLOHUB_PY_EXE=$(which ${HOLOHUB_PY_EXE})
HOLOHUB_PY_LIB="${HOLOHUB_PY_EXE/bin/lib}"

#===========================================================================================
# Utilities

YELLOW="\e[1;33m"
RED="\e[1;31m"
NOCOLOR="\e[0m"

# Compare versions
compare_version () {

    if [[ $1 == $2 ]]
    then
        echo "0"
        return 0
    fi

    local IFS=.
    local i ver1=($1) ver2=($2)
    # fill empty fields in ver1 with zeros
    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
    do
        ver1[i]=0
    done

    for ((i=0; i<${#ver1[@]}; i++))
    do
        if [[ -z ${ver2[i]} ]]
        then
            # fill empty fields in ver2 with zeros
            ver2[i]=0
        fi
        if ((10#${ver1[i]} > 10#${ver2[i]}))
        then
            echo "1"
            return 1
        fi
        if ((10#${ver1[i]} < 10#${ver2[i]}))
        then
            echo "2"
            return 2
        fi
    done
    echo "0"
    return 0
}


run_command() {
    local status=0
    local cmd="$*"

    if [ "${DO_DRY_RUN}" != "true" ]; then
        echo -e "${YELLOW}[command]${NOCOLOR} ${cmd}"
    else
        echo -e "${YELLOW}[dryrun]${NOCOLOR} ${cmd}"
    fi

    [ "$(echo -n "$@")" = "" ] && return 1 # return 1 if there is no command available

    if [ "${DO_DRY_RUN}" != "true" ]; then
        eval "$@"
        status=$?
    fi

    return $status
}


get_buildtype_str() {
    local build_type="${1:-}"
    local build_type_str

    case "${build_type}" in
        debug|Debug)
            build_type_str="Debug"
            ;;
        release|Release)
            build_type_str="Release"
            ;;
        rel-debug|RelWithDebInfo)
            build_type_str="RelWithDebInfo"
            ;;
        *)
            build_type_str="${CMAKE_BUILD_TYPE:-Release}"
            ;;
    esac

    echo -n "${build_type_str}"
}


get_app_source_root_dir() {
  # Get source path for a given application according to HoloHub convention.
  # Usage: ./run get_app_source_root_dir <app_name>
  local appname="$1"
  if [[ -z "${appname}" ]]; then
    echo "Missing app name argument for get_app_source_root_dir"
    exit 1
  fi

  holohub_app_source="${SCRIPT_DIR}/applications/${appname}"

  # Check if the application is in a subdirectory
  if [ ! -d "$holohub_app_source" ]; then
    sub_app_path=$(find ${SCRIPT_DIR}/applications -type d -name "${appname}")
    app_rel_path=${sub_app_path#"${SCRIPT_DIR}/applications/"}
    if [[ -n "$app_rel_path" ]]; then
      holohub_app_source="${SCRIPT_DIR}/applications/${app_rel_path}"
    else
      echo "Could not find subproject ${appname}"
      exit 1
    fi
  fi

  echo -n "${holohub_app_source}"
}

get_app_source_lang_dir() {
  # Get source path for a given application language implementation according to HoloHub convention.
  # If a project defines more than one language implementation it must define one subfolder per language,
  # otherwise the implementation may be provided at the top level or in a subfolder at the contributor's discretion.
  # Usage: ./run get_app_source_lang_dir <app_name> <language>
  local appname="$1"
  local language="$2"

  local holohub_app_source=$(get_app_source_root_dir "${appname}")
  if [[ -n "${holohub_app_source}" ]] && [[ -d "${holohub_app_source}/${language}" ]]; then
    holohub_app_source="${holohub_app_source}/${language}"
  fi

  echo -n "${holohub_app_source}"
}

get_app_dockerfile() {
  # Parse the path to a custom application Dockerfile, if provided.
  # Follows the HoloHub convention:
  # 1. Check the metadata.json file for a "dockerfile" entry.
  # 2. Check for a Dockerfile in the application language implementation directory.
  # 3. Check for a Dockerfile in the root application directory.
  # 4. Use the default Dockerfile provided at the top level of the HoloHub repository.
  # Usage: ./run get_app_dockerfile <app_name> <language>
  local appname="$1"
  local language="$2"
  local dockerfile_path=""

  local app_source_root_path=$(get_app_source_root_dir ${appname})
  local app_source_lang_path=$(get_app_source_lang_dir ${appname} ${language})

  local dockerfile_entry=$(cat "${app_source_lang_path}/metadata.json" | grep "dockerfile")
  if [[ -n "${dockerfile_entry}" ]]; then
    pattern="s#<holohub_app_source>#${app_source_lang_path}#g"
    dockerfile_path=$(echo "${dockerfile_entry}" | awk -F'["]' '{ print $4 }' | sed -E ${pattern})
  elif [[ -f "${app_source_lang_path}/Dockerfile" ]]; then
    dockerfile_path="${app_source_lang_path}/Dockerfile"
  elif [[ -f "${app_source_root_path}/Dockerfile" ]]; then
    dockerfile_path="${app_source_root_path}/Dockerfile"
  else
    dockerfile_path="${SCRIPT_DIR}/Dockerfile"
  fi

  echo -n "${dockerfile_path}"
}

get_app_default_language() {
  # If a language was not provided but multiple are available, auto-select
  # the first language matching the app name from the "./run list" command.
  # Yields an empty string if a language folder is not specified.
  # Example `list` string:
  # ultrasound_segmentation (cpp) [Sample Application]
  # Usage: ./run get_app_default_language <app_name>
  appname="$1"

  if [ -z "${language}" ]; then
    language=$(list | grep -m 1 ${appname} | awk -F'[\\(\\)]' '{print $2}')
  fi

  echo -e "${language}"
}


print_error() {
    echo -e "${RED}ERROR${NOCOLOR}:" $*
}

#===========================================================================================
# Helper function to install packages

install_cuda_dependencies_package() {
   package_name=$1
   preferred_version=$2
   optional=false
   if [ -n "$3" ]; then
     optional=true
   fi

   # Checking if compatible packages are already installed
   installed_version=$(apt list --installed ${package_name} 2>/dev/null | grep $package_name)
   if [[ $installed_version != "" ]]; then
      echo "Package $package_name found with version $installed_version"
      return 0
   fi

   available_version=$(apt list -a ${package_name} 2>/dev/null | grep $preferred_version)
   package_version=$(echo $available_version | cut -d' ' -f2)
   package_installed=$(echo $available_version | grep "installed")

   if [[ $package_version == "" ]]; then
    if [[ $optional == "true" ]]; then
      echo "Package $package_name $preferred_version not found. Skipping."
    else
      print_error "$package_name $preferred_version is not installable."
      echo "You might want to try to install a newer version manually and rerun the setup:"
      echo "  sudo apt install $package_name"
      exit 1
    fi
   elif [[ $package_installed == "" ]]; then
    echo "Installing $package_name=$package_version"
    apt install --no-install-recommends -y $package_name=$package_version
   fi
}

# Setup the environment
# Build HoloHub sample apps
setup_desc() {
  echo ""
  echo "Install the required dependencies for HoloHub (sample applications)"
  echo "Usage: ./run setup"
  echo ""
}

setup() {

  # Run apt-get update
  apt-get update

  # Install wget
  wget_version=$(dpkg --status wget | grep -Po '^Version: \K[^-]*')
  if [[ $wget_version == "" ]]; then
    apt-get install -y wget
  fi

  # Install xvfb for running tests/examples headless
  xvfb_version=$(dpkg --status xvfb | grep -Po '^Version: \K[^-]*')
  if [[ $xvfb_version == "" ]]; then
    apt-get install -y xvfb
  fi

  # Check version of CMake otherwise upgrade
  cmake_version=$(dpkg --status cmake | grep -Po '^Version: \K[^-]*')
  ubuntu_codename=$(cat /etc/os-release | grep -Po '^UBUNTU_CODENAME=\K[^ ]*')

  cmake_need_upload=$(compare_version ${cmake_version} "3.24.0")

  # If we should update cmake
  # Install from https://apt.kitware.com/
  if [[ $cmake_version == "" ]] || [[ $cmake_need_upload == 2 ]]; then
    apt install --no-install-recommends -y gpg
    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
    echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ ${ubuntu_codename} main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null
    apt-get update
    apt install --no-install-recommends -y cmake cmake-curses-gui
  fi

  # Install python dev
  python3_dev_version=$(dpkg --status python3-dev  2>/dev/null | grep -Po '^Version: \K[^-]*')
  python3_min_version=$(compare_version ${python3_dev_version} "3.8.0")

  # Python 3.8 to 3.11 are supported
  if [[ $python3_dev_version == "" ]] || [[ $python3_min_version == "2" ]]; then
    echo "Installing python"
    apt install --no-install-recommends -y python3 python3-dev
    python3_dev_version=$(dpkg --status python3-dev  2>/dev/null | grep -Po '^Version: \K[^-]*')
    python3_min_version=$(compare_version ${python3_dev_version} "3.8.0")
  fi

  # If python version is too high
  python3_max_version=$(compare_version ${python3_dev_version} "3.12.0")
  if [[ $python3_max_version == "0" ]] || [[ $python3_max_version == "1" ]]; then
    print_error "Python version ${python3_dev_version} is not supported"
    exit 1
  fi

  # Install ffmpeg
  ffmpeg_version=$(dpkg --status ffmpeg 2>/dev/null | grep -Po '^Version: \K[^-]*')
  if [[ $ffmpeg_version == "" ]]; then
    echo "Installing ffmpeg"
    apt install --no-install-recommends -y ffmpeg
  fi

  # Install libv4l-dev
  libv4l_dev_version=$(dpkg --status libv4l-dev | grep -Po '^Version: \K[^-]*')
  if [[ $libv4l_dev_version == "" ]]; then
    echo "Installing libv4l-dev"
    apt-get install --no-install-recommends -y libv4l-dev
  fi


  # Install ngc-cli
  ngc_version=$(ngc --version 2>/dev/null | grep -Po '^NGC CLI \K[^-]*')
  if [[ $ngc_version == "" ]]; then
    echo "Installing ngc cli"

    if [ $(uname -m) == "aarch64" ]; then
      wget --content-disposition https://ngc.nvidia.com/downloads/ngccli_arm64.zip && unzip ngccli_arm64.zip && chmod u+x ngc-cli/ngc
    fi
    if [ $(uname -m) == "x86_64" ]; then
      wget --content-disposition https://ngc.nvidia.com/downloads/ngccli_linux.zip && unzip ngccli_linux.zip && chmod u+x ngc-cli/ngc
    fi
    ln -s $(pwd)/ngc-cli/ngc /usr/local/bin/
  fi

  # Install the tensorrt development libs
  # Find the newest cudart version installed
  for version in 12-4 12-3 12-2 12-1 12-0 11-8 11-7 11-6 11-4
   do
      cuda_version=$(dpkg --status cuda-cudart-${version} 2>/dev/null | grep -Po '^Version: \K[^-]*')
      # Find the version based on cudart
      if [[ $cuda_version != "" ]]; then
        break
      fi
   done

  short_cuda_version=${cuda_version%.*}
  echo "Cuda version found: $short_cuda_version"

  # Install cudnn9 first
  install_cuda_dependencies_package libcudnn9-cuda-12 cuda$short_cuda_version true
  install_cuda_dependencies_package libcudnn9-dev-cuda-12 cuda$short_cuda_version true

  # Check if cudnn9 is install, if not install cudnn8
  installed_cudnn9_version=$(apt list --installed libcudnn9-cuda-12 2>/dev/null | grep libcudnn9-cuda-12)
  if [[ $installed_cudnn9_version == "" ]]; then
     install_cuda_dependencies_package libcudnn8 cuda$short_cuda_version
     install_cuda_dependencies_package libcudnn8-dev cuda$short_cuda_version
  fi

  # Make sure we do not install any cuda12 packages
  install_cuda_dependencies_package libnvinfer-dev cuda$short_cuda_version
  install_cuda_dependencies_package libnvinfer-plugin-dev cuda$short_cuda_version
  install_cuda_dependencies_package libnvonnxparsers-dev cuda$short_cuda_version

  # Install the autocomplete
  echo "Installing autocomplete"
  cp ${SCRIPT_DIR}/utilities/holohub_autocomplete /etc/bash_completion.d/

  echo "Setup for HoloHub is ready. Happy Holocoding!"
}

#===========================================================================================
# Lint

install_lint_deps_desc() { echo 'Install lint dependencies

This command will install the dependencies required to run the linting tools.

- Python linting: ruff, isort, black, mypy
- CMake linting: cmakelint
- C++ linting: cpplint, clang-format
- Spelling: codespell
'
}
install_lint_deps() {
    # We use $(command) || exit_code=1 to run all linting tools, and exit
    # with failure after all commands were executed if any of them failed
    local exit_code=0

    pushd ${SCRIPT_DIR} > /dev/null

    echo "Install Lint Dependencies for Python"
    if ! command -v ruff &> /dev/null; then
        run_command ${HOLOHUB_PY_EXE} -m pip install ruff || exit_code=1
    fi
    if ! command -v isort &> /dev/null; then
        run_command ${HOLOHUB_PY_EXE} -m pip install isort || exit_code=1
    fi
    if ! command -v black &> /dev/null; then
        run_command ${HOLOHUB_PY_EXE} -m pip install black || exit_code=1
    fi
    if ! command -v mypy &> /dev/null; then
        run_command ${HOLOHUB_PY_EXE} -m pip install mypy || exit_code=1
    fi

    echo "Install Lint Dependencies for C++"
    if ! command -v cpplint &> /dev/null; then
        run_command ${HOLOHUB_PY_EXE} -m pip install cpplint || exit_code=1
    fi
    if ! command -v clang-format &> /dev/null; then
        run_command apt-get install --no-install-recommends -y clang-format || exit_code=1
    fi

    echo "Install Lint Dependencies for Code Spelling"
    if ! command -v codespell &> /dev/null; then
        run_command ${HOLOHUB_PY_EXE} -m pip install codespell || exit_code=1
    fi

    echo "Install Lint Dependencies for CMake"
    if ! command -v cmakelint &> /dev/null; then
        run_command ${HOLOHUB_PY_EXE} -m pip install cmakelint || exit_code=1
    fi

    popd > /dev/null

    exit $exit_code
}

lint_desc() { echo 'Lint the repository

Python linting: black, isort, ruff
C++ linting: cpplint
CMake linting: cmakelint
Spelling: codespell

Arguments:
  $@ - directories to lint (default: .)

Options:
  --fix: to fix all linting errors'
}

lint() {

    # Parse the arguments
    ARGS=("$@")
    local DIR_TO_RUN="."
    local FIX=false

    for i in "${!ARGS[@]}"; do
      arg="${ARGS[i]}"
      if [ "$arg" = "--fix" ]; then
         FIX=true
      elif [[ $arg = -* ]]; then
        print_error "Unknown option $arg"
        exit 1
      else
        DIR_TO_RUN=$arg
      fi
    done

    local exit_code=0

    # If we call with --fix
    if [ $FIX == true ]; then
      # Fix python
      run_command ruff --fix --ignore E712 ${DIR_TO_RUN} || exit_code=1
      # Fix python isort issues, run:
      run_command isort ${DIR_TO_RUN} || exit_code=1
      # Fix python black code formatting issues, run:
      run_command black ${DIR_TO_RUN} || exit_code=1
      run_command codespell -w -i 3 ${DIR_TO_RUN} -L nd,ba,activ,Activ,dne,unexpect,hsi,donot,thirdparty \
                            --skip="*.onnx,*.min.js,*.min.js.map,Contrastive_learning_Notebook.ipynb,./data" || exit_code=1

      # Fix cpplint with clang
      files_to_fix=`set -o pipefail; ${HOLOHUB_PY_EXE} -m cpplint \
            --exclude build \
            --exclude install \
            --exclude build-\* \
            --exclude install-\* \
            --recursive $DIR_TO_RUN 2>&1 | grep -v 'Ignoring\|Done processing\|Total error' | sed 's/:.*//' | sort | uniq`

      # Fix C++ lint issues by running clang-format:
      for file in ${files_to_fix}; do
        run_command clang-format --style=file --sort-includes=0 --lines=20:10000 -i ${file} || exit_code=1
      done

      exit $exit_code
    fi


    # We use $(command) || exit_code=1 to run all linting tools, and exit
    # with failure after all commands were executed if any of them failed
    pushd ${SCRIPT_DIR} > /dev/null

    echo "Linting Python"
    run_command ruff $DIR_TO_RUN --ignore E712 || exit_code=1
    run_command ${HOLOHUB_PY_EXE} -m isort -c $DIR_TO_RUN || exit_code=1
    run_command ${HOLOHUB_PY_EXE} -m black --check $DIR_TO_RUN || exit_code=1

    echo "Linting C++"
    # We use `grep -v` to hide verbose output that drowns actual errors
    # Since we care about the success/failure of cpplitn and not of grep, we:
    #  1. use `set -o pipefail` to fail if `cpplint` fails
    #  2. use `grep -v ... || true` to ignore whether grep hid any output
    run_command "set -o pipefail; ${HOLOHUB_PY_EXE} -m cpplint \
            --exclude build \
            --exclude install \
            --exclude build-\* \
            --exclude install-\* \
            --recursive $DIR_TO_RUN \
        | { grep -v 'Ignoring\|Done processing' || true; } || exit_code=1"


    echo "Code spelling"
    run_command codespell $DIR_TO_RUN --skip="*.onnx,*.min.js,*.min.js.map,Contrastive_learning_Notebook.ipynb,./data" \
                          -L nd,ba,activ,Activ,dne,unexpect,hsi,donot,thirdparty \
                          --exclude-file codespell.txt || exit_code=1

    echo "Linting CMake"
    cmakelint --filter=-whitespace/indent,-linelength,-readability/wonkycase,-convention/filename,-package/stdargs \
              $(find $DIR_TO_RUN '(' -name CMakeLists.txt -o -name *.cmake ')' -not -path "*build/*" -not -path "*./.*" -not -path "*install/*" -not -path "*tmp/*") || exit_code=1

    popd > /dev/null

    if [ $exit_code == 0 ]; then
        echo "Everything looks good!"
    fi

    exit $exit_code
}


#===========================================================================================
# Build HoloHub sample apps

build_desc() {
  echo ""
  echo "Build the application(s) in HoloHub"
  echo "Usage: ./run build <application_name> [options]"
  echo "Options:"
  echo "   --sdk <path_to_holoscan_SDK>  : Provide path to the SDK"
  echo "   --with <list of operators>  : Optional operators that should be built"
  echo "                                 If multiple operators are to be built, list must be"
  echo "                                 in quotes with operators separated by semicolons (;)"
  echo "   --type <debug | release | rel-debug> : Specify the type of build"
  echo "                                          Default: release"
  echo "                                          Associated environment variable: CMAKE_BUILD_TYPE"
  echo "   --buildpath <build_directory>        : Change the build path."
  echo "                                          Default: build"
  echo "                                          Associated environment variable: CMAKE_BUILD_PATH"
  echo "   --configure-args <extra_args>        : Additional configuration arguments"
  echo "                                          multiple arguments can be passed between quotes"
  echo "                                          or using several --configure-args in the command line"
  echo ""
  echo "Prior to the first build use './run setup' to install the required dependencies"
  echo ""
}

# See how many directories under the input path matches "install*" pattern
# If we find exactly one, this is probably the SDK install path the user wants to use
# Print out a warning message if we find multiple install paths and ask user to specify exact path using --sdk option
get_install_path() {
    dir=$1
    install_paths=($(find $dir -maxdepth 1 -type d -name "install*" 2>/dev/null))
    if [ ${#install_paths[@]} -eq 1 ]; then
        echo "${install_paths[0]}"
    elif [ ${#install_paths[@]} -gt 1 ]; then
        echo "Warning: Multiple install directories found under $dir, please specify exact path to --sdk option with one of the following directories" >&2
        for path in "${install_paths[@]}"; do
            echo "- $path" >&2
        done
        echo ""
    else
        echo ""
    fi
}

# Allow users to point "--sdk" to root directory of the SDK installation.
# This is to address the differences in directory structure between
# installations between debian image, docker, or when built from source
resolve_sdk_install_path() {
  sdk_path=$1

  install_path=$(get_install_path "${sdk_path}/public")
  if [[ -n $install_path ]]; then
    echo "${install_path}";
    return
  fi

  install_path=$(get_install_path "${sdk_path}/")
  if [[ -n $install_path ]]; then
    echo "${install_path}";
    return
  fi

  echo "${sdk_path}"
}

build() {
  # Holoscan SDK location
  holoscan_sdk=""

  # CMake configuration args
  configure_args=""

  # Parse the arguments
  ARGS=("$@")
  local app
  local i
  local arg
  local skipnext=0
  local build_type="${CMAKE_BUILD_TYPE:-release}"
  local build_path="${CMAKE_BUILD_PATH}"

  for i in "${!ARGS[@]}"; do
      arg="${ARGS[i]}"
      if [[ $skipnext == "1" ]]; then
         skipnext=0
      elif [ "$arg" = "--sdk" ]; then
         sdk_path=$( resolve_sdk_install_path "${ARGS[i+1]}" )
         holoscan_sdk="-Dholoscan_ROOT=${sdk_path}"
         echo "Setting Holoscan SDK install path to ${sdk_path}"
         skipnext=1
      elif [ "$arg" = "--with" ]; then
         operators="${ARGS[i+1]}"
         configure_args="${configure_args} -DHOLOHUB_BUILD_OPERATORS=\"${operators}\""
         echo "Building with operator(s) ${operators}"
         skipnext=1
      elif [ "$arg" = "--configure-args" ]; then
         configure_args="${configure_args} ${ARGS[i+1]}"
         echo "Adding configuration arguments: ${ARGS[i+1]}"
         skipnext=1
      elif [ "$arg" = "--type" ]; then
         build_type=$(get_buildtype_str "${ARGS[i+1]}")
         skipnext=1
      elif [ "$arg" = "--buildpath" ]; then
         build_path="${ARGS[i+1]}"
         skipnext=1
      elif [[ $arg = -* ]]; then
        print_error "Unknown option $arg"
        exit 1
      else
        app=$arg
      fi
  done

  echo "Building Holohub"

  # Application to build
  if [ "$1" == "" ]; then
    print_error "Please specify which application to build."
    echo "You can run ./run list for a full list of applications."
    exit 1
  fi

  # Assign default build path
  if [ "$build_path" == "" ]; then
    build_path="build/$1"
  fi

  echo "Building $1 application"
  application="-DAPP_$1=1"

  # We define the python path to make sure we grab the right one
  cmake_extra_args="--no-warn-unused-cli -DPython3_EXECUTABLE=${HOLOHUB_PY_EXE} -DPython3_ROOT_DIR=${HOLOHUB_PY_LIB}"

  # We set the data directory to be outside the build directory
  cmake_extra_args="$cmake_extra_args $configure_args -DHOLOHUB_DATA_DIR:PATH=${SCRIPT_DIR}/data"

  # Sets the default path for cuda
  export PATH=$PATH:/usr/local/cuda/bin
  run_command cmake -S . -B ${build_path} ${cmake_extra_args} -DCMAKE_BUILD_TYPE=${build_type} ${holoscan_sdk} ${application}
  run_command cmake --build ${build_path} -j

  echo "Holohub build done."
}


# Launch a sample app
launch_desc() {
  echo ""
  echo "Launch command allowing to run application directly."
  echo "Usage: ./run launch <application_name> [language: cpp|python] [--extra_args <args>]"
  echo "Options:"
  echo "   --extra_args <args>       : additional arguments passed to the application command"
  echo "   --nsys_profile            : profile using Nsight Systems"
  echo "   --verbose                 : output additional run command information to stdout"
  echo ""
  echo ""
  echo "Use './run list' first to list the available applications"
  echo "Note that applications might be listed but not built."
  echo "Make sure you ./run build the application first"
  echo ""
}


launch() {
   local appname=""
   local language=""
   local extra_args=""
   local nsys_profile=false
   local verbose=false
   local holohub_build_dir=""
   local skipnext=0

   for i in "${!ARGS[@]}"; do
     arg="${ARGS[i]}"
     if [[ $skipnext == "1" ]]; then
       skipnext=0
     elif [ "$arg" = "--extra_args" ]; then
       extra_args="${ARGS[i+1]}"
       skipnext=1
     elif [ "$arg" = "--nsys_profile" ]; then
       nsys_profile=true
     elif [ "$arg" = "--buildpath" ]; then
       holohub_build_dir="${ARGS[i+1]}"
       skipnext=1
     elif [ "$arg" = "--verbose" ]; then
       verbose=true
     elif [ -z "${appname}" ]; then
       appname=$arg
     elif [ -z "${language}" ]; then
       language=$arg
     fi
   done

   # Export the python path (by default use the one used to build)
   if [ "$holohub_build_dir" == "" ]; then
      holohub_build_dir="${SCRIPT_DIR}/build/$appname"
   else
     holohub_build_dir=$(readlink --canonicalize $holohub_build_dir)
   fi

   # Do initial sanitary check to make sure build/ is there, to avoid too many other error messages
   if [ ! -d "$holohub_build_dir" ]; then
     print_error "The build directory ${holohub_build_dir} does not exist."
     echo "Did you forget to build your application using './run build $appname' ?"
     build_desc
     exit 1
   fi

   local holoscan_sdk_install=$(grep -Po '^holoscan_DIR:PATH=\K[^ ]*' ${holohub_build_dir}/CMakeCache.txt)
   local holohub_data_dir=$(grep -Po '^HOLOHUB_DATA_DIR:PATH=\K[^ ]*' ${holohub_build_dir}/CMakeCache.txt)

   if [[ -z "${language}" ]]; then
     language=$(get_app_default_language "${appname}")
     language=${language:-cpp}
     if [[ -n "${language}" ]]; then
       echo "Default language for ${appname} selected: ${language}"
     fi
   fi

   holohub_app_source=$(get_app_source_lang_dir "${appname}" "${language}")
   if [[ -z "${holohub_app_source}" ]]; then
     print_error "Could not find app source directory for ${appname} ${language}."
     launch_desc
     exit 1
   fi

   # Check if the metadata file exists
   local metadata_file="${holohub_app_source}/metadata.json"
   if [ ! -f "$metadata_file" ]; then
     print_error "The metadata file for this application does not exist."
     echo "File: ${metadata_file} not found"
     echo "Did you forget to specify the language?"
     exit 1
   fi

   holohub_app_bin="${holohub_build_dir}/${holohub_app_source#"$SCRIPT_DIR/"}"
   # Check if the build directory exists (for C++ apps)
   if [ ! -d "$holohub_app_bin" ] &&  [ "$2" == "cpp" ]; then
     print_error "The build directory for this application does not exist."
     echo "Did you forget to './run build $1' ?"
     build_desc
     exit 1
   fi

   # Use Python to parse json file
   json=$(${HOLOHUB_PY_EXE} -c 'import json,sys
f=open("'${metadata_file}'")
obj=json.load(f)
for k, v in obj["application"]["run"].items():
  print(str(k)+"=\""+str(v)+"\"")
')

   local json_command=$(echo $json | grep -Po 'command="\K[^"]*')

   # replace <holohub_data_dir> by the data dir
   local command=$(echo "${json_command//<holohub_data_dir>/$holohub_data_dir}")

   # replace <holohub_app_bin> by the binary app directory
   command=$(echo "${command//<holohub_app_bin>/$holohub_app_bin}")

   # replace <holohub_app_source> by the source app directory
   command=$(echo "${command//<holohub_app_source>/$holohub_app_source}")

   if [ $nsys_profile == true ]; then
      if [ ! command -v nsys &> /dev/null ]; then
        print_error "Nsight Systems CLI command 'nsys' not found."
        exit 1
      fi
      local perf_event_level=$(cat /proc/sys/kernel/perf_event_paranoid)
      if (( ${perf_event_level} > 2 )); then
        print_error "For Nsight Systems profiling the Linux operating system's perf_event_paranoid level must be 2 or less."
        echo "See https://docs.nvidia.com/nsight-systems/InstallationGuide/index.html#linux-requirements for more information."
        exit 1
      fi
      command="nsys profile --trace=cuda,vulkan,nvtx,osrt ${command}"
   fi

   # default workdir is bin
   local workdir="cd ${holohub_app_bin}"
   local json_workdir=$(echo $json | grep -Po 'workdir="\K[^"]*')
   if [[ $json_workdir == "holohub_app_source" ]]; then
      workdir="cd ${holohub_app_source}"
   elif [[ $json_workdir == "holohub_bin" ]]; then
      workdir="cd ${holohub_build_dir}"
   fi

   local environment="export PYTHONPATH=\${PYTHONPATH}:${holoscan_sdk_install}/../../../python/lib:${holohub_build_dir}/python/lib:${SCRIPT_DIR} && export HOLOHUB_DATA_PATH=${holohub_data_dir}"
   local reset_environment="export PYTHONPATH=${PYTHONPATH} && export HOLOHUB_DATA_PATH=\"${HOLOHUB_DATA_PATH}\""

   if [ ${verbose} ]; then
    echo "Run environment: $environment"
    echo "Run workdir: $workdir"
    echo "Run command: $command"
    echo "Run command args: $extra_args"
   fi

   # Run the command
   run_command $environment
   run_command $workdir
   run_command $command $extra_args
   run_command $reset_environment
}

#===========================================================================================
# Other options

# Build HoloHub sample apps
list_desc() {
  echo ""
  echo "Display the list of applications to run."
  echo "Usage: ./run list"
  echo ""
}

list() {
  echo ""
  echo "***** HoloHub applications ******"

  apps=$(find ${SCRIPT_DIR}/applications -name 'metadata.json' | sort -d)
  for d in ${apps}; do
    local appname=$(dirname $d | grep -Po '^'${SCRIPT_DIR}/applications/'\K[^ ]*')

    local language="${appname##*/}"
    if [[ ${language} != "cpp" ]] && [[ ${language} != "python" ]]; then
      language=""
      filename="${appname}"
    else
      language="(${language})"
      filename="${appname%/*}"
    fi

    application="${filename##*/}"

    echo "- ${application} ${language}"
  done
  echo ""
  echo "Run ./run build <app_name> to build a specific app."
}

# Returns the list of words for autocompletion
autocompletion_list() {
  apps=$(find ${SCRIPT_DIR}/applications -name 'metadata.json')
  for d in ${apps}; do
    local appname=$(dirname $d | grep -Po '^'${SCRIPT_DIR}/applications/'\K[^ ]*')
    filename="${appname%/*}"
    # Deal with subdirectories of applications
    echo "${filename##*/}"
  done
  echo "build launch clear_cache cpp python install_lint_deps lint setup"
}

clear_cache() {
  echo "Clearing cache..."
  run_command rm -rf ${SCRIPT_DIR}/build
  run_command rm -rf ${SCRIPT_DIR}/build-*
}

clear_cache_desc() {
  echo "Remove the build folders"
}

parse_args() {
    local OPTIND
    while getopts 'yh' option;
    do
        case "${option}" in
            y)
                ALWAYS_YES="true"
                ;;
            h)
                print_usage
                exit 1
                ;;
            *)
                ;;
        esac
    done
    shift $((OPTIND-1))

    CMD="$1"
    shift

    ARGS=("$@")
    # Check if the command has `--help`, `-h`, and override the CMD
    local i
    local arg
    extra_args=false
    for i in "${!ARGS[@]}"; do
        arg="${ARGS[i]}"
        # if any argument is --extra_args, then a subsequent "-h/--help" is passed to the application
        if [[ "$arg" == "--extra_args" ]]; then
            extra_args=true
            continue
        fi
        if [[ "$extra_args" == false ]]; then
          if [ "$arg" = "--help" ] || [ "$arg" = "-h" ]; then
              ARGS=("$CMD")
              CMD="help"
              break
          fi
        fi
        if [ "$arg" = "--dryrun" ]; then
            unset_pos=$i
            DO_DRY_RUN="true"  # set to true to print commands to screen without running
        fi
    done
    if [ "${unset_pos}" ]; then
        unset 'ARGS[unset_pos]'
    fi
}

print_usage() {
    set +x
    echo
    echo "USAGE: $0 [command] [arguments]..."
    echo ""
    echo "Global Arguments"
    echo "  --help, -h                                    : Print help messages for [command]"
    echo "  --dryrun                                      : Print commands to screen without running"
    echo ""
    echo "Command List"
    echo "  setup                                         : Install HoloHub main required packages"
    echo "  clear_cache                                   : Clear cache folders"
    echo "  list                                          : List all the applications"
    echo "  build [application] [--sdk holoscan_sdk]      : Build a specific application. Default is sample apps"
    echo "  launch <application> <language>               : Run the application"
}

print_cmd_help_messages() {
    local cmd="$1"
    if [ -n "${cmd}" ]; then
        if type ${cmd}_desc > /dev/null 2>&1; then
            ${cmd}_desc
            exit 0
        else
            echo "Command '${cmd}' doesn't exist!"
            exit 1
        fi
    fi
    print_usage
    return 0
}


main() {
    local ret=0
    parse_args "$@"

    case "$CMD" in
        help|"")
            print_cmd_help_messages "${ARGS[@]}"
            exit 0
            ;;
        *)
            if type ${CMD} > /dev/null 2>&1; then
                "$CMD" "${ARGS[@]}"
            else
                print_usage
                exit 1
            fi
            ;;
    esac
    ret=$?
}

main "$@"