Update test scripts for upcoming Galaxy 22.05

galaxyproject · Apr 11, 2022 · de55247 · de55247
1 parent 2437524
commit de55247
Show file tree

Hide file tree

Showing 5 changed files with 95 additions and 46 deletions.
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -4,8 +4,8 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
         with:
           python-version: '3.10'
       - name: Install dependencies
@@ -18,7 +18,7 @@ jobs:
           twine check dist/*
       - name: Publish to PyPI
         if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && github.repository_owner == 'galaxyproject'
-        uses: pypa/gh-action-pypi-publish@master
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.PYPI_PASSWORD }}
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -10,8 +10,8 @@ jobs:
       matrix:
         python-version: ['3.7']
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install tox

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -65,7 +65,7 @@ jobs:
           #   tox_env: py37
           #   galaxy_version: dev
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Cache pip dir
         uses: actions/cache@v2
         with:
@@ -75,7 +75,7 @@ jobs:
         id: get_bioblend_python_version
         run: echo "::set-output name=bioblend_python_version::$(echo "${{ matrix.tox_env }}" | sed -e 's/^py\([3-9]\)\([0-9]\+\)/\1.\2/')"
       - name: Set up Python for BioBlend
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ steps.get_bioblend_python_version.outputs.bioblend_python_version }}
       - name: Install tox
@@ -100,7 +100,7 @@ jobs:
           esac
           echo "::set-output name=galaxy_python_version::$galaxy_python_version"
       - name: Set up Python for Galaxy
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ steps.get_galaxy_python_version.outputs.galaxy_python_version }}
       - name: Run tests
@@ -112,28 +112,11 @@ jobs:
           # Create a PostgreSQL database for Galaxy. The default SQLite3 database makes test fail randomly because of "database locked" error.
           createdb -U postgres galaxy
           # Install Galaxy
-          wget https://github.com/galaxyproject/galaxy/archive/${{ matrix.galaxy_version }}.tar.gz
-          tar xvzf ${{ matrix.galaxy_version }}.tar.gz | tail
-          cd galaxy-${{ matrix.galaxy_version }}
-          export GALAXY_DIR=$PWD
-          export GALAXY_PYTHON=python${{ steps.get_galaxy_python_version.outputs.galaxy_python_version }}
-          export GALAXY_VERSION=${{ matrix.galaxy_version }}
-          # Export GALAXY_CONFIG_FILE environment variable to be used by run_galaxy.sh
-          export GALAXY_CONFIG_FILE=config/galaxy.ini
-          # Export BIOBLEND_ environment variables to be used in BioBlend tests
-          export BIOBLEND_GALAXY_MASTER_API_KEY=$(LC_ALL=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 32)
-          export BIOBLEND_GALAXY_USER_EMAIL=${USER}@localhost.localdomain
+          GALAXY_DIR=galaxy-${{ matrix.galaxy_version }}
+          git clone --depth=1 -b ${{ matrix.galaxy_version }} https://github.com/galaxyproject/galaxy $GALAXY_DIR
           export DATABASE_CONNECTION=postgresql://postgres:@localhost/galaxy
-          eval "echo \"$(cat "${{ github.workspace }}/tests/template_galaxy.ini")\"" > "$GALAXY_CONFIG_FILE"
-          # Update psycopg2 requirement to a version compatible with glibc 2.26 for Galaxy releases 16.01-18.01, see https://github.com/psycopg/psycopg2-wheels/issues/2
-          sed -i.bak -e 's/psycopg2==2.6.1/psycopg2==2.7.3.1/' lib/galaxy/dependencies/conditional-requirements.txt
-          # Start Galaxy and wait for successful server start
-          export GALAXY_SKIP_CLIENT_BUILD=1
-          GALAXY_RUN_ALL=1 "${{ github.workspace }}/run_galaxy.sh" --daemon --wait
-          export BIOBLEND_GALAXY_URL=http://localhost:8080
-          cd "${{ github.workspace }}"
-          tox -e ${{ matrix.tox_env }}
+          ./run_bioblend_tests.sh -g $GALAXY_DIR -v python${{ steps.get_galaxy_python_version.outputs.galaxy_python_version }} -e ${{ matrix.tox_env }}
       - name: The job has failed
         if: ${{ failure() }}
         run: |
-          cat galaxy-${{ matrix.galaxy_version }}/main.log
+          cat galaxy-${{ matrix.galaxy_version }}/*.log
diff --git a/bioblend/_tests/template_galaxy.yml b/bioblend/_tests/template_galaxy.yml
@@ -0,0 +1,18 @@
+gravity:
+  gunicorn:
+    bind: localhost:${GALAXY_PORT:-8080}
+
+galaxy:
+  managed_config_dir: ${TEMP_DIR:-${GALAXY_DIR}}/config
+  data_dir: ${TEMP_DIR:-${GALAXY_DIR}}/database
+  database_connection: $DATABASE_CONNECTION
+  tool_config_file: ${GALAXY_DIR}/config/tool_conf.xml.sample,${TEMP_DIR:-${GALAXY_DIR}}/config/shed_tool_conf.xml,${GALAXY_DIR}/test/functional/tools/samples_tool_conf.xml
+  # Don't use $TEMP_DIR for tool_dependency_dir to save time on local testing
+  tool_dependency_dir: ${GALAXY_DIR}/database/dependencies
+  allow_path_paste: true
+  admin_users: $BIOBLEND_GALAXY_USER_EMAIL
+  allow_user_deletion: true
+  enable_beta_workflow_modules: true
+  master_api_key: $BIOBLEND_GALAXY_MASTER_API_KEY
+  enable_quotas: true
+  cleanup_job: onsuccess
diff --git a/run_bioblend_tests.sh b/run_bioblend_tests.sh
@@ -21,6 +21,8 @@ Options:
       to all tests.
   -r GALAXY_REV
       Branch or commit of the local Galaxy git repository to checkout.
+  -v GALAXY_PYTHON
+      Python to use for the Galaxy virtual environment.
   -c
       Force removal of the temporary directory created for Galaxy, even if some
       test failed."
@@ -33,7 +35,7 @@ get_abs_dirname () {
 
 e_val=py37
 GALAXY_PORT=8080
-while getopts 'hcg:e:p:t:r:' option; do
+while getopts 'hcg:e:p:t:r:v:' option; do
     case $option in
         h) show_help
            exit;;
@@ -43,6 +45,7 @@ while getopts 'hcg:e:p:t:r:' option; do
         p) GALAXY_PORT=$OPTARG;;
         t) t_val=$OPTARG;;
         r) r_val=$OPTARG;;
+        v) GALAXY_PYTHON=$OPTARG;;
         *) show_help
            exit 1;;
     esac
@@ -56,15 +59,16 @@ fi
 
 # Install BioBlend
 BIOBLEND_DIR=$(get_abs_dirname "$(dirname "$0")")
-cd "${BIOBLEND_DIR}"
-if [ ! -d .venv ]; then
-    virtualenv -p python3 .venv
+if ! command -v tox >/dev/null; then
+    cd "${BIOBLEND_DIR}"
+    if [ ! -d .venv ]; then
+        virtualenv -p python3 .venv
+    fi
+    . .venv/bin/activate
+    python3 -m pip install --upgrade "tox>=1.8.0"
 fi
-. .venv/bin/activate
-python3 setup.py install
-python3 -m pip install --upgrade "tox>=1.8.0"
 
-# Setup Galaxy
+# Setup Galaxy version
 cd "${GALAXY_DIR}"
 if [ -n "${r_val}" ]; then
     # Update repository (may change the sample files or the list of eggs)
@@ -83,24 +87,63 @@ else
             ;;
     esac
 fi
+
+# Setup Galaxy virtualenv
+if [ -n "${GALAXY_PYTHON}" ]; then
+    if [ ! -d .venv ]; then
+        virtualenv -p "${GALAXY_PYTHON}" .venv
+    fi
+    export GALAXY_PYTHON
+fi
+
 # Setup Galaxy master API key and admin user
 TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
 echo "Created temporary directory $TEMP_DIR"
-mkdir "$TEMP_DIR/config"
+mkdir "${TEMP_DIR}/config" "${TEMP_DIR}/database"
 printf "<?xml version=\"1.0\"?>\n<toolbox tool_path=\"%s\">\n</toolbox>\n" "$TEMP_DIR/shed_tools" > "$TEMP_DIR/config/shed_tool_conf.xml"
-# Export GALAXY_CONFIG_FILE environment variable to be used by run_galaxy.sh
-export GALAXY_CONFIG_FILE="$TEMP_DIR/config/galaxy.ini"
 # Export BIOBLEND_ environment variables to be used in BioBlend tests
-BIOBLEND_GALAXY_MASTER_API_KEY=$(LC_ALL=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 32)
-export BIOBLEND_GALAXY_MASTER_API_KEY
+export BIOBLEND_GALAXY_MASTER_API_KEY=$(LC_ALL=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 32)
 export BIOBLEND_GALAXY_USER_EMAIL="${USER}@localhost.localdomain"
-DATABASE_CONNECTION="sqlite:///$TEMP_DIR/universe.sqlite?isolation_level=IMMEDIATE"
-eval "echo \"$(cat "${BIOBLEND_DIR}/tests/template_galaxy.ini")\"" > "$GALAXY_CONFIG_FILE"
+DATABASE_CONNECTION=${DATABASE_CONNECTION:-"sqlite:///${TEMP_DIR}/database/universe.sqlite?isolation_level=IMMEDIATE"}
 # Update psycopg2 requirement to a version compatible with glibc 2.26 for Galaxy releases 16.01-18.01, see https://github.com/psycopg/psycopg2-wheels/issues/2
 sed -i.bak -e 's/psycopg2==2.6.1/psycopg2==2.7.3.1/' lib/galaxy/dependencies/conditional-requirements.txt
 # Start Galaxy and wait for successful server start
 export GALAXY_SKIP_CLIENT_BUILD=1
-GALAXY_RUN_ALL=1 "${BIOBLEND_DIR}/run_galaxy.sh" --daemon --wait
+if grep -q wait_arg_set run.sh ; then
+    # Galaxy 22.01 or earlier.
+    # Export GALAXY_CONFIG_FILE environment variable to be used by run_galaxy.sh
+    export GALAXY_CONFIG_FILE="${TEMP_DIR}/config/galaxy.ini"
+    eval "echo \"$(cat "${BIOBLEND_DIR}/tests/template_galaxy.ini")\"" > "${GALAXY_CONFIG_FILE}"
+    GALAXY_RUN_ALL=1 "${BIOBLEND_DIR}/run_galaxy.sh" --daemon --wait
+else
+    # Galaxy is controlled via gravity, paste/uwsgi are replaced by gunicorn
+    # and the `--wait` option does not work any more.
+    # Export GALAXY_CONFIG_FILE environment variable to be used by run.sh
+    export GALAXY_CONFIG_FILE="${TEMP_DIR}/config/galaxy.yml"
+    eval "echo \"$(cat "${BIOBLEND_DIR}/tests/template_galaxy.yml")\"" > "${GALAXY_CONFIG_FILE}"
+    export GRAVITY_STATE_DIR="${TEMP_DIR}/database/gravity"
+    ./run.sh --daemon
+    if ! .venv/bin/galaxyctl -h > /dev/null; then
+        echo 'galaxyctl status not working'
+        exit 1
+    fi
+    while true; do
+        sleep 1
+        if .venv/bin/galaxyctl status | grep -q 'gunicorn.*RUNNING'; then
+            break
+        else
+            echo 'gunicorn not running yet'
+        fi
+    done
+    while true; do
+        sleep 1
+        if grep -q "serving on http://127.0.0.1:${GALAXY_PORT}" "${GRAVITY_STATE_DIR}/log/gunicorn.log"; then
+            break
+        else
+            echo 'Galaxy not serving yet'
+        fi
+    done
+fi
 export BIOBLEND_GALAXY_URL=http://localhost:${GALAXY_PORT}
 
 # Run the tests
@@ -115,8 +158,13 @@ exit_code=$?
 deactivate
 
 # Stop Galaxy
+echo 'Stopping Galaxy'
 cd "${GALAXY_DIR}"
-GALAXY_RUN_ALL=1 "${BIOBLEND_DIR}/run_galaxy.sh" --daemon stop
+if grep -q wait_arg_set run.sh ; then
+    GALAXY_RUN_ALL=1 "${BIOBLEND_DIR}/run_galaxy.sh" --daemon stop
+else
+    ./run.sh --daemon stop
+fi
 # Remove temporary directory if -c is specified or if all tests passed
 if [ -n "${c_val}" ] || [ $exit_code -eq 0 ]; then
     rm -rf "$TEMP_DIR"