Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add shared_fs and staging_required test markers #3669

Merged
merged 3 commits into from
Oct 31, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions parsl/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,10 @@ def pytest_configure(config):
'markers',
'executor_supports_std_stream_tuples: Marks tests that require tuple support for stdout/stderr'
)
config.addinivalue_line(
'markers',
'shared_fs: Marks tests that require a shared_fs between the workers are the test client'
)


@pytest.fixture(autouse=True, scope='session')
Expand Down
3 changes: 3 additions & 0 deletions parsl/tests/test_bash_apps/test_basic.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def foo(x, y, z=10, stdout=None, label=None):
return f"echo {x} {y} {z}"


@pytest.mark.shared_fs
def test_command_format_1(tmpd_cwd):
"""Testing command format for BashApps"""

Expand All @@ -38,6 +39,7 @@ def test_command_format_1(tmpd_cwd):
assert so_content == "1 4 10"


@pytest.mark.shared_fs
def test_auto_log_filename_format(caplog):
"""Testing auto log filename format for BashApps
"""
Expand Down Expand Up @@ -66,6 +68,7 @@ def test_auto_log_filename_format(caplog):
assert record.levelno < logging.ERROR


@pytest.mark.shared_fs
def test_parallel_for(tmpd_cwd, n=3):
"""Testing a simple parallel for loop"""
outdir = tmpd_cwd / "outputs/test_parallel"
Expand Down
4 changes: 4 additions & 0 deletions parsl/tests/test_bash_apps/test_error_codes.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ def bad_format(stderr='std.err', stdout='std.out'):
whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')


@pytest.mark.shared_fs
def test_div_0(test_fn=div_0):
err_code = test_matrix[test_fn]['exit_code']
f = test_fn()
Expand All @@ -73,6 +74,7 @@ def test_div_0(test_fn=div_0):
os.remove('std.out')


@pytest.mark.shared_fs
def test_bash_misuse(test_fn=bash_misuse):
err_code = test_matrix[test_fn]['exit_code']
f = test_fn()
Expand All @@ -87,6 +89,7 @@ def test_bash_misuse(test_fn=bash_misuse):
os.remove('std.out')


@pytest.mark.shared_fs
def test_command_not_found(test_fn=command_not_found):
err_code = test_matrix[test_fn]['exit_code']
f = test_fn()
Expand All @@ -103,6 +106,7 @@ def test_command_not_found(test_fn=command_not_found):
return True


@pytest.mark.shared_fs
def test_not_executable(test_fn=not_executable):
err_code = test_matrix[test_fn]['exit_code']
f = test_fn()
Expand Down
1 change: 1 addition & 0 deletions parsl/tests/test_bash_apps/test_kwarg_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ def foo(z=2, stdout=None):
return f"echo {z}"


@pytest.mark.shared_fs
def test_command_format_1(tmpd_cwd):
"""Testing command format for BashApps
"""
Expand Down
8 changes: 2 additions & 6 deletions parsl/tests/test_bash_apps/test_memoize.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,7 @@ def fail_on_presence(outputs=()):
return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])


# This test is an oddity that requires a shared-FS and simply
# won't work if there's a staging provider.
# @pytest.mark.sharedFS_required
@pytest.mark.shared_fs
def test_bash_memoization(tmpd_cwd, n=2):
"""Testing bash memoization
"""
Expand All @@ -29,9 +27,7 @@ def fail_on_presence_kw(outputs=(), foo=None):
return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])


# This test is an oddity that requires a shared-FS and simply
# won't work if there's a staging provider.
# @pytest.mark.sharedFS_required
@pytest.mark.shared_fs
def test_bash_memoization_keywords(tmpd_cwd, n=2):
"""Testing bash memoization
"""
Expand Down
3 changes: 3 additions & 0 deletions parsl/tests/test_bash_apps/test_memoize_ignore_args.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import os

import pytest

import parsl
from parsl.app.app import bash_app

Expand All @@ -21,6 +23,7 @@ def no_checkpoint_stdout_app_ignore_args(stdout=None):
return "echo X"


@pytest.mark.shared_fs
def test_memo_stdout(tmpd_cwd):
path_x = tmpd_cwd / "test.memo.stdout.x"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def no_checkpoint_stdout_app(stdout=None):
return "echo X"


@pytest.mark.shared_fs
def test_memo_stdout(tmpd_cwd):
assert const_list_x == const_list_x_arg

Expand Down
1 change: 1 addition & 0 deletions parsl/tests/test_bash_apps/test_multiline.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ def multiline(inputs=(), outputs=(), stderr=None, stdout=None):
""".format(inputs=inputs, outputs=outputs)


@pytest.mark.shared_fs
def test_multiline(tmpd_cwd):
so, se = tmpd_cwd / "std.out", tmpd_cwd / "std.err"
f = multiline(
Expand Down
3 changes: 3 additions & 0 deletions parsl/tests/test_bash_apps/test_stdout.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def echo_to_streams(msg, stderr=None, stdout=None):
]


@pytest.mark.shared_fs
yadudoc marked this conversation as resolved.
Show resolved Hide resolved
@pytest.mark.parametrize('spec', speclist, ids=testids)
def test_bad_stdout_specs(spec):
"""Testing bad stdout spec cases"""
Expand Down Expand Up @@ -91,6 +92,7 @@ def test_bad_stderr_file():


@pytest.mark.executor_supports_std_stream_tuples
@pytest.mark.shared_fs
def test_stdout_truncate(tmpd_cwd, caplog):
"""Testing truncation of prior content of stdout"""

Expand All @@ -110,6 +112,7 @@ def test_stdout_truncate(tmpd_cwd, caplog):
assert record.levelno < logging.ERROR


@pytest.mark.shared_fs
def test_stdout_append(tmpd_cwd, caplog):
"""Testing appending to prior content of stdout (default open() mode)"""

Expand Down
3 changes: 3 additions & 0 deletions parsl/tests/test_docs/test_from_slides.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import os

import pytest

from parsl.app.app import bash_app, python_app
from parsl.data_provider.files import File

Expand All @@ -15,6 +17,7 @@ def cat(inputs=[]):
return f.readlines()


@pytest.mark.staging_required
def test_slides():
"""Testing code snippet from slides """

Expand Down
3 changes: 3 additions & 0 deletions parsl/tests/test_docs/test_kwargs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Functions used to explain kwargs"""
from pathlib import Path

import pytest

from parsl import File, python_app


Expand All @@ -19,6 +21,7 @@ def reduce_app(inputs=()):
assert reduce_future.result() == 6


@pytest.mark.shared_fs
def test_outputs(tmpd_cwd):
@python_app()
def write_app(message, outputs=()):
Expand Down
1 change: 1 addition & 0 deletions parsl/tests/test_docs/test_workflow1.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ def save(message, outputs=[]):
return 'echo {m} &> {o}'.format(m=message, o=outputs[0])


@pytest.mark.shared_fs
yadudoc marked this conversation as resolved.
Show resolved Hide resolved
@pytest.mark.staging_required
def test_procedural(N=2):
"""Procedural workflow example from docs on
Expand Down
1 change: 1 addition & 0 deletions parsl/tests/test_python_apps/test_outputs.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ def double(x, outputs=[]):
whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')


@pytest.mark.shared_fs
def test_launch_apps(tmpd_cwd, n=2):
outdir = tmpd_cwd / "outputs"
outdir.mkdir()
Expand Down
1 change: 1 addition & 0 deletions parsl/tests/test_regression/test_226.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def test_get_dataframe():
assert res.equals(data), 'Unexpected dataframe'


@pytest.mark.shared_fs
def test_bash_default_arg():
if os.path.exists('std.out'):
os.remove('std.out')
Expand Down
1 change: 1 addition & 0 deletions parsl/tests/test_staging/test_docs_1.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ def convert(inputs=[], outputs=[]):


@pytest.mark.cleannet
@pytest.mark.shared_fs
yadudoc marked this conversation as resolved.
Show resolved Hide resolved
def test():
# create an remote Parsl file
inp = File('ftp://ftp.iana.org/pub/mirror/rirstats/arin/ARIN-STATS-FORMAT-CHANGE.txt')
Expand Down
3 changes: 3 additions & 0 deletions parsl/tests/test_staging/test_output_chain_filenames.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from concurrent.futures import Future

import pytest

from parsl import File
from parsl.app.app import bash_app

Expand All @@ -14,6 +16,7 @@ def app2(inputs=(), outputs=(), stdout=None, stderr=None, mock=False):
return f"echo '{inputs[0]}' > {outputs[0]}"


@pytest.mark.shared_fs
def test_behavior(tmpd_cwd):
expected_path = str(tmpd_cwd / "simple-out.txt")
app1_future = app1(
Expand Down
1 change: 1 addition & 0 deletions parsl/tests/test_staging/test_staging_ftp.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ def sort_strings(inputs=[], outputs=[]):


@pytest.mark.cleannet
@pytest.mark.staging_required
def test_staging_ftp():
"""Test staging for an ftp file

Expand Down
3 changes: 3 additions & 0 deletions parsl/tests/test_staging/test_staging_https.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ def sort_strings_additional_executor(inputs=(), outputs=()):


@pytest.mark.cleannet
@pytest.mark.staging_required
def test_staging_https_cleannet(tmpd_cwd):
unsorted_file = File(_unsorted_url)
sorted_file = File(tmpd_cwd / 'sorted.txt')
Expand All @@ -68,6 +69,7 @@ def test_staging_https_local(tmpd_cwd):


@pytest.mark.cleannet
@pytest.mark.staging_required
def test_staging_https_kwargs(tmpd_cwd):
unsorted_file = File(_unsorted_url)
sorted_file = File(tmpd_cwd / 'sorted.txt')
Expand All @@ -78,6 +80,7 @@ def test_staging_https_kwargs(tmpd_cwd):


@pytest.mark.cleannet
@pytest.mark.staging_required
def test_staging_https_args(tmpd_cwd):
unsorted_file = File(_unsorted_url)
sorted_file = File(tmpd_cwd / 'sorted.txt')
Expand Down
2 changes: 2 additions & 0 deletions parsl/tests/test_staging/test_staging_stdout.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ def output_to_stds(*, stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
return "echo hello ; echo goodbye >&2"


@pytest.mark.shared_fs
yadudoc marked this conversation as resolved.
Show resolved Hide resolved
def test_stdout_staging_file(tmpd_cwd, caplog):
basename = str(tmpd_cwd) + "/stdout.txt"
stdout_file = File("file://" + basename)
Expand All @@ -30,6 +31,7 @@ def test_stdout_staging_file(tmpd_cwd, caplog):
assert record.levelno < logging.ERROR


@pytest.mark.shared_fs
def test_stdout_stderr_staging_zip(tmpd_cwd, caplog):
zipfile_name = str(tmpd_cwd) + "/staging.zip"
stdout_relative_path = "somewhere/test-out.txt"
Expand Down
Loading