diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index bf3e04bd9..a222a5a33 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -4,8 +4,8 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: python-version: '3.10' - name: Install dependencies @@ -18,7 +18,7 @@ jobs: twine check dist/* - name: Publish to PyPI if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && github.repository_owner == 'galaxyproject' - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 89b53e5f1..512422a0c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -10,8 +10,8 @@ jobs: matrix: python-version: ['3.7'] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install tox diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5499b6bdf..5a620e69f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -65,7 +65,7 @@ jobs: # tox_env: py37 # galaxy_version: dev steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Cache pip dir uses: actions/cache@v2 with: @@ -75,7 +75,7 @@ jobs: id: get_bioblend_python_version run: echo "::set-output name=bioblend_python_version::$(echo "${{ matrix.tox_env }}" | sed -e 's/^py\([3-9]\)\([0-9]\+\)/\1.\2/')" - name: Set up Python for BioBlend - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ steps.get_bioblend_python_version.outputs.bioblend_python_version }} - name: Install tox @@ -100,7 +100,7 @@ jobs: esac echo "::set-output name=galaxy_python_version::$galaxy_python_version" - name: Set up Python for Galaxy - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ steps.get_galaxy_python_version.outputs.galaxy_python_version }} - name: Run tests @@ -112,28 +112,11 @@ jobs: # Create a PostgreSQL database for Galaxy. The default SQLite3 database makes test fail randomly because of "database locked" error. createdb -U postgres galaxy # Install Galaxy - wget https://github.com/galaxyproject/galaxy/archive/${{ matrix.galaxy_version }}.tar.gz - tar xvzf ${{ matrix.galaxy_version }}.tar.gz | tail - cd galaxy-${{ matrix.galaxy_version }} - export GALAXY_DIR=$PWD - export GALAXY_PYTHON=python${{ steps.get_galaxy_python_version.outputs.galaxy_python_version }} - export GALAXY_VERSION=${{ matrix.galaxy_version }} - # Export GALAXY_CONFIG_FILE environment variable to be used by run_galaxy.sh - export GALAXY_CONFIG_FILE=config/galaxy.ini - # Export BIOBLEND_ environment variables to be used in BioBlend tests - export BIOBLEND_GALAXY_MASTER_API_KEY=$(LC_ALL=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 32) - export BIOBLEND_GALAXY_USER_EMAIL=${USER}@localhost.localdomain + GALAXY_DIR=galaxy-${{ matrix.galaxy_version }} + git clone --depth=1 -b ${{ matrix.galaxy_version }} https://github.com/galaxyproject/galaxy $GALAXY_DIR export DATABASE_CONNECTION=postgresql://postgres:@localhost/galaxy - eval "echo \"$(cat "${{ github.workspace }}/tests/template_galaxy.ini")\"" > "$GALAXY_CONFIG_FILE" - # Update psycopg2 requirement to a version compatible with glibc 2.26 for Galaxy releases 16.01-18.01, see https://github.com/psycopg/psycopg2-wheels/issues/2 - sed -i.bak -e 's/psycopg2==2.6.1/psycopg2==2.7.3.1/' lib/galaxy/dependencies/conditional-requirements.txt - # Start Galaxy and wait for successful server start - export GALAXY_SKIP_CLIENT_BUILD=1 - GALAXY_RUN_ALL=1 "${{ github.workspace }}/run_galaxy.sh" --daemon --wait - export BIOBLEND_GALAXY_URL=http://localhost:8080 - cd "${{ github.workspace }}" - tox -e ${{ matrix.tox_env }} + ./run_bioblend_tests.sh -g $GALAXY_DIR -v python${{ steps.get_galaxy_python_version.outputs.galaxy_python_version }} -e ${{ matrix.tox_env }} - name: The job has failed if: ${{ failure() }} run: | - cat galaxy-${{ matrix.galaxy_version }}/main.log + cat galaxy-${{ matrix.galaxy_version }}/*.log diff --git a/bioblend/_tests/template_galaxy.yml b/bioblend/_tests/template_galaxy.yml new file mode 100644 index 000000000..7ad8ba02f --- /dev/null +++ b/bioblend/_tests/template_galaxy.yml @@ -0,0 +1,18 @@ +gravity: + gunicorn: + bind: localhost:${GALAXY_PORT:-8080} + +galaxy: + managed_config_dir: ${TEMP_DIR:-${GALAXY_DIR}}/config + data_dir: ${TEMP_DIR:-${GALAXY_DIR}}/database + database_connection: $DATABASE_CONNECTION + tool_config_file: ${GALAXY_DIR}/config/tool_conf.xml.sample,${TEMP_DIR:-${GALAXY_DIR}}/config/shed_tool_conf.xml,${GALAXY_DIR}/test/functional/tools/samples_tool_conf.xml + # Don't use $TEMP_DIR for tool_dependency_dir to save time on local testing + tool_dependency_dir: ${GALAXY_DIR}/database/dependencies + allow_path_paste: true + admin_users: $BIOBLEND_GALAXY_USER_EMAIL + allow_user_deletion: true + enable_beta_workflow_modules: true + master_api_key: $BIOBLEND_GALAXY_MASTER_API_KEY + enable_quotas: true + cleanup_job: onsuccess diff --git a/run_bioblend_tests.sh b/run_bioblend_tests.sh index a37f9d61b..cb73d44b0 100755 --- a/run_bioblend_tests.sh +++ b/run_bioblend_tests.sh @@ -21,6 +21,8 @@ Options: to all tests. -r GALAXY_REV Branch or commit of the local Galaxy git repository to checkout. + -v GALAXY_PYTHON + Python to use for the Galaxy virtual environment. -c Force removal of the temporary directory created for Galaxy, even if some test failed." @@ -33,7 +35,7 @@ get_abs_dirname () { e_val=py37 GALAXY_PORT=8080 -while getopts 'hcg:e:p:t:r:' option; do +while getopts 'hcg:e:p:t:r:v:' option; do case $option in h) show_help exit;; @@ -43,6 +45,7 @@ while getopts 'hcg:e:p:t:r:' option; do p) GALAXY_PORT=$OPTARG;; t) t_val=$OPTARG;; r) r_val=$OPTARG;; + v) GALAXY_PYTHON=$OPTARG;; *) show_help exit 1;; esac @@ -56,15 +59,16 @@ fi # Install BioBlend BIOBLEND_DIR=$(get_abs_dirname "$(dirname "$0")") -cd "${BIOBLEND_DIR}" -if [ ! -d .venv ]; then - virtualenv -p python3 .venv +if ! command -v tox >/dev/null; then + cd "${BIOBLEND_DIR}" + if [ ! -d .venv ]; then + virtualenv -p python3 .venv + fi + . .venv/bin/activate + python3 -m pip install --upgrade "tox>=1.8.0" fi -. .venv/bin/activate -python3 setup.py install -python3 -m pip install --upgrade "tox>=1.8.0" -# Setup Galaxy +# Setup Galaxy version cd "${GALAXY_DIR}" if [ -n "${r_val}" ]; then # Update repository (may change the sample files or the list of eggs) @@ -83,24 +87,63 @@ else ;; esac fi + +# Setup Galaxy virtualenv +if [ -n "${GALAXY_PYTHON}" ]; then + if [ ! -d .venv ]; then + virtualenv -p "${GALAXY_PYTHON}" .venv + fi + export GALAXY_PYTHON +fi + # Setup Galaxy master API key and admin user TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir') echo "Created temporary directory $TEMP_DIR" -mkdir "$TEMP_DIR/config" +mkdir "${TEMP_DIR}/config" "${TEMP_DIR}/database" printf "\n\n\n" "$TEMP_DIR/shed_tools" > "$TEMP_DIR/config/shed_tool_conf.xml" -# Export GALAXY_CONFIG_FILE environment variable to be used by run_galaxy.sh -export GALAXY_CONFIG_FILE="$TEMP_DIR/config/galaxy.ini" # Export BIOBLEND_ environment variables to be used in BioBlend tests -BIOBLEND_GALAXY_MASTER_API_KEY=$(LC_ALL=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 32) -export BIOBLEND_GALAXY_MASTER_API_KEY +export BIOBLEND_GALAXY_MASTER_API_KEY=$(LC_ALL=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 32) export BIOBLEND_GALAXY_USER_EMAIL="${USER}@localhost.localdomain" -DATABASE_CONNECTION="sqlite:///$TEMP_DIR/universe.sqlite?isolation_level=IMMEDIATE" -eval "echo \"$(cat "${BIOBLEND_DIR}/tests/template_galaxy.ini")\"" > "$GALAXY_CONFIG_FILE" +DATABASE_CONNECTION=${DATABASE_CONNECTION:-"sqlite:///${TEMP_DIR}/database/universe.sqlite?isolation_level=IMMEDIATE"} # Update psycopg2 requirement to a version compatible with glibc 2.26 for Galaxy releases 16.01-18.01, see https://github.com/psycopg/psycopg2-wheels/issues/2 sed -i.bak -e 's/psycopg2==2.6.1/psycopg2==2.7.3.1/' lib/galaxy/dependencies/conditional-requirements.txt # Start Galaxy and wait for successful server start export GALAXY_SKIP_CLIENT_BUILD=1 -GALAXY_RUN_ALL=1 "${BIOBLEND_DIR}/run_galaxy.sh" --daemon --wait +if grep -q wait_arg_set run.sh ; then + # Galaxy 22.01 or earlier. + # Export GALAXY_CONFIG_FILE environment variable to be used by run_galaxy.sh + export GALAXY_CONFIG_FILE="${TEMP_DIR}/config/galaxy.ini" + eval "echo \"$(cat "${BIOBLEND_DIR}/tests/template_galaxy.ini")\"" > "${GALAXY_CONFIG_FILE}" + GALAXY_RUN_ALL=1 "${BIOBLEND_DIR}/run_galaxy.sh" --daemon --wait +else + # Galaxy is controlled via gravity, paste/uwsgi are replaced by gunicorn + # and the `--wait` option does not work any more. + # Export GALAXY_CONFIG_FILE environment variable to be used by run.sh + export GALAXY_CONFIG_FILE="${TEMP_DIR}/config/galaxy.yml" + eval "echo \"$(cat "${BIOBLEND_DIR}/tests/template_galaxy.yml")\"" > "${GALAXY_CONFIG_FILE}" + export GRAVITY_STATE_DIR="${TEMP_DIR}/database/gravity" + ./run.sh --daemon + if ! .venv/bin/galaxyctl -h > /dev/null; then + echo 'galaxyctl status not working' + exit 1 + fi + while true; do + sleep 1 + if .venv/bin/galaxyctl status | grep -q 'gunicorn.*RUNNING'; then + break + else + echo 'gunicorn not running yet' + fi + done + while true; do + sleep 1 + if grep -q "serving on http://127.0.0.1:${GALAXY_PORT}" "${GRAVITY_STATE_DIR}/log/gunicorn.log"; then + break + else + echo 'Galaxy not serving yet' + fi + done +fi export BIOBLEND_GALAXY_URL=http://localhost:${GALAXY_PORT} # Run the tests @@ -115,8 +158,13 @@ exit_code=$? deactivate # Stop Galaxy +echo 'Stopping Galaxy' cd "${GALAXY_DIR}" -GALAXY_RUN_ALL=1 "${BIOBLEND_DIR}/run_galaxy.sh" --daemon stop +if grep -q wait_arg_set run.sh ; then + GALAXY_RUN_ALL=1 "${BIOBLEND_DIR}/run_galaxy.sh" --daemon stop +else + ./run.sh --daemon stop +fi # Remove temporary directory if -c is specified or if all tests passed if [ -n "${c_val}" ] || [ $exit_code -eq 0 ]; then rm -rf "$TEMP_DIR"