Skip to content

Commit

Permalink
Github Action for GlobusComputeExecutor (#3619)
Browse files Browse the repository at this point in the history
* Support for testing GlobusComputeExecutor in a github action
  • Loading branch information
yadudoc authored Oct 16, 2024
1 parent 495d009 commit 85b2298
Show file tree
Hide file tree
Showing 7 changed files with 123 additions and 2 deletions.
108 changes: 108 additions & 0 deletions .github/workflows/gce_test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
name: GlobusComputeExecutor tests

on:
pull_request:
types:
- opened
- synchronize

workflow_dispatch:
inputs:
tags:
description: 'Test scenario tags'
required: false
type: boolean

jobs:
main-test-suite:
strategy:
matrix:
python-version: ["3.11"]
runs-on: ubuntu-20.04
timeout-minutes: 60

steps:
- uses: actions/checkout@master

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: Collect Job Information
id: job-info
run: |
echo "Python Version: ${{ matrix.python-version }}" >> ci_job_info.txt
echo "CI Triggering Event: ${{ github.event_name }}" >> ci_job_info.txt
echo "Triggering Git Ref: ${{ github.ref }}" >> ci_job_info.txt
echo "Triggering Git SHA: ${{ github.sha }}" >> ci_job_info.txt
echo "Workflow Run: ${{ github.run_number }}" >> ci_job_info.txt
echo "Workflow Attempt: ${{ github.run_attempt }}" >> ci_job_info.txt
as_ascii="$(echo "${{ github.ref_name }}" | perl -pe "s/[^A-z0-9-]+/-/g; s/^-+|-+\$//g; s/--+/-/g;")"
echo "as-ascii=$as_ascii" >> $GITHUB_OUTPUT
- name: Non-requirements based install
run: |
# libpython3.5: make workqueue binary installer happy
# mpich: required by radical executor
sudo apt-get update -q
sudo apt-get install -qy libpython3.5 mpich
- name: setup virtual env
run: |
make virtualenv
source .venv/bin/activate
- name: make deps clean_coverage
run: |
source .venv/bin/activate
make deps
make clean_coverage
# Installing parsl into venv required for GCendpoint
pip3 install .
# Temporary fix, until changes make it into compute releases
git clone -b configure_tasks_working_dir https://github.com/globus/globus-compute.git
pip3 install globus-compute/compute_sdk globus-compute/compute_endpoint
- name: start globus_compute_endpoint
env:
GLOBUS_COMPUTE_CLIENT_ID: ${{ secrets.GLOBUS_COMPUTE_CLIENT_ID }}
GLOBUS_COMPUTE_CLIENT_SECRET: ${{ secrets.GLOBUS_COMPUTE_SECRET_KEY }}
run: |
source /home/runner/work/parsl/parsl/.venv/bin/activate
globus-compute-endpoint configure default
cat << EOF > /home/runner/.globus_compute/default/config.yaml
engine:
type: ThreadPoolEngine
max_workers: 4
EOF
cat /home/runner/.globus_compute/default/config.yaml
mkdir ~/.globus_compute/default/tasks_working_dir
globus-compute-endpoint start default
globus-compute-endpoint list
- name: make test
env:
GLOBUS_COMPUTE_CLIENT_ID: ${{ secrets.GLOBUS_COMPUTE_CLIENT_ID }}
GLOBUS_COMPUTE_CLIENT_SECRET: ${{ secrets.GLOBUS_COMPUTE_SECRET_KEY }}
run: |
source .venv/bin/activate
export GLOBUS_COMPUTE_ENDPOINT=$(globus-compute-endpoint list | grep default | cut -c 3-38)
echo "GLOBUS_COMPUTE_ENDPOINT = $GLOBUS_COMPUTE_ENDPOINT"
# temporary; until test-matrixification
export PARSL_TEST_PRESERVE_NUM_RUNS=7
make gce_test
ln -s .pytest/parsltest-current test_runinfo
- name: Archive runinfo logs
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: runinfo-${{ matrix.python-version }}-${{ steps.job-info.outputs.as-ascii }}-${{ github.sha }}
path: |
runinfo/
.pytest/
ci_job_info.txt
compression-level: 9
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ clean_coverage:
mypy: ## run mypy checks
MYPYPATH=$(CWD)/mypy-stubs mypy parsl/

.PHONY: gce_test
gce_test: ## Run tests with GlobusComputeExecutor
pytest -v -k "not shared_fs and not issue_3620 and not staging_required" --config parsl/tests/configs/globus_compute.py parsl/tests/ --random-order --durations 10

.PHONY: local_thread_test
local_thread_test: ## run all tests with local_thread config
pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/local_threads.py --random-order --durations 10
Expand Down
1 change: 1 addition & 0 deletions docs/reference.rst
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ Executors
parsl.executors.taskvine.TaskVineExecutor
parsl.executors.FluxExecutor
parsl.executors.radical.RadicalPilotExecutor
parsl.executors.globus_compute.GlobusComputeExecutor

Launchers
=========
Expand Down
4 changes: 2 additions & 2 deletions parsl/executors/globus_compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ def __init__(
self,
endpoint_id: Optional[UUID_LIKE_T] = None,
task_group_id: Optional[UUID_LIKE_T] = None,
resource_specification: Optional[dict[str, Any]] = None,
user_endpoint_config: Optional[dict[str, Any]] = None,
resource_specification: Optional[Dict[str, Any]] = None,
user_endpoint_config: Optional[Dict[str, Any]] = None,
label: str = "GlobusComputeExecutor",
batch_size: int = 128,
amqp_port: Optional[int] = None,
Expand Down
4 changes: 4 additions & 0 deletions parsl/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ def pytest_configure(config):
'markers',
'shared_fs: Marks tests that require a shared_fs between the workers are the test client'
)
config.addinivalue_line(
'markers',
'issue_3620: Marks tests that do not work correctly on GlobusComputeExecutor (ref: issue 3620)'
)


@pytest.fixture(autouse=True, scope='session')
Expand Down
3 changes: 3 additions & 0 deletions parsl/tests/test_error_handling/test_resource_spec.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import pytest

import parsl
from parsl.app.app import python_app
from parsl.executors import WorkQueueExecutor
Expand All @@ -13,6 +15,7 @@ def double(x, parsl_resource_specification={}):
return x * 2


@pytest.mark.issue_3620
def test_resource(n=2):
executors = parsl.dfk().executors
executor = None
Expand Down
1 change: 1 addition & 0 deletions test-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ types-python-dateutil
types-requests
types-paramiko
mpi4py
globus-compute-sdk>=2.27.1

# sqlalchemy is needed for typechecking, so it's here
# as well as at runtime for optional monitoring execution
Expand Down

0 comments on commit 85b2298

Please sign in to comment.