Separate out tests that require hypothesis

Signed-off-by: Eduardo Apolinario <[email protected]>

.github/workflows/pythonbuild.yml

@@ -149,6 +149,41 @@ jobs:
           fail_ci_if_error: false
           files: coverage.xml
 
+  test-hypothesis:
+    needs:
+      - detect-python-versions
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements files
+          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
+      - name: Install dependencies
+        run: make setup && pip freeze
+      - name: Test with coverage
+        env:
+          FLYTEKIT_HYPOTHESIS_PROFILE: ci
+        run: |
+          make unit_test_hypothesis
+      - name: Codecov
+        uses: codecov/[email protected]
+        with:
+          fail_ci_if_error: false
+          files: coverage.xml
+
   test-serialization:
     needs:
       - detect-python-versions

Makefile

@@ -62,10 +62,13 @@ unit_test_extras_codecov:
 unit_test:
 	# Skip all extra tests and run them with the necessary env var set so that a working (albeit slower)
 	# library is used to serialize/deserialize protobufs is used.
-	$(PYTEST_AND_OPTS) -m "not (serial or sandbox_test)" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/ --ignore=tests/flytekit/unit/models --ignore=tests/flytekit/unit/extend ${CODECOV_OPTS}
+	$(PYTEST_AND_OPTS) -m "not (serial or sandbox_test or hypothesis)" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/ --ignore=tests/flytekit/unit/models --ignore=tests/flytekit/unit/extend ${CODECOV_OPTS}
 	# Run serial tests without any parallelism
 	$(PYTEST) -m "serial" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/ --ignore=tests/flytekit/unit/models --ignore=tests/flytekit/unit/extend ${CODECOV_OPTS}
 
+.PHONY: unit_test_hypothesis
+unit_test_hypothesis:
+	$(PYTEST_AND_OPTS) -m "hypothesis" tests/flytekit/unit/experimental ${CODECOV_OPTS}
 
 .PHONY: unit_test_extras
 unit_test_extras:

pyproject.toml

@@ -99,6 +99,7 @@ log_cli_level = 20
 markers = [
     "sandbox_test: fake integration tests", # unit tests that are really integration tests that run on a sandbox environment
     "serial: tests to avoid using with pytest-xdist",
+    "hypothesis: tests that use they hypothesis library",
 ]
 
 [tool.coverage.report]

tests/flytekit/unit/conftest.py

@@ -1,4 +1,7 @@
+import os
+
 import pytest
+from hypothesis import settings
 
 from flytekit.image_spec.image_spec import ImageSpecBuilder
 
@@ -11,3 +14,9 @@ def build_image(self, img):
 @pytest.fixture()
 def mock_image_spec_builder():
     return MockImageSpecBuilder()
+
+
+settings.register_profile("ci", max_examples=5, deadline=100_000)
+settings.register_profile("dev", max_examples=10, deadline=10_000)
+
+settings.load_profile(os.getenv("FLYTEKIT_HYPOTHESIS_PROFILE", "dev"))

tests/flytekit/unit/experimental/test_eager_workflows.py

@@ -6,7 +6,7 @@
 
 import hypothesis.strategies as st
 import pytest
-from hypothesis import given, settings
+from hypothesis import given
 
 from flytekit import dynamic, task, workflow
 from flytekit.exceptions.user import FlyteValidationException
@@ -15,7 +15,6 @@
 from flytekit.types.file import FlyteFile
 from flytekit.types.structured import StructuredDataset
 
-DEADLINE = 20000
 INTEGER_ST = st.integers(min_value=-10_000_000, max_value=10_000_000)
 
 
@@ -48,7 +47,7 @@ def dynamic_wf(x: int) -> int:
 
 
 @given(x_input=INTEGER_ST)
-@settings(deadline=DEADLINE, max_examples=5)
+@pytest.mark.hypothesis
 def test_simple_eager_workflow(x_input: int):
     """Testing simple eager workflow with just tasks."""
 
@@ -62,7 +61,7 @@ async def eager_wf(x: int) -> int:
 
 
 @given(x_input=INTEGER_ST)
-@settings(deadline=DEADLINE, max_examples=5)
+@pytest.mark.hypothesis
 def test_conditional_eager_workflow(x_input: int):
     """Test eager workflow with conditional logic."""
 
@@ -80,7 +79,7 @@ async def eager_wf(x: int) -> int:
 
 
 @given(x_input=INTEGER_ST)
-@settings(deadline=DEADLINE, max_examples=5)
+@pytest.mark.hypothesis
 def test_try_except_eager_workflow(x_input: int):
     """Test eager workflow with try/except logic."""
 
@@ -99,7 +98,7 @@ async def eager_wf(x: int) -> int:
 
 
 @given(x_input=INTEGER_ST, n_input=st.integers(min_value=1, max_value=20))
-@settings(deadline=DEADLINE, max_examples=5)
+@pytest.mark.hypothesis
 def test_gather_eager_workflow(x_input: int, n_input: int):
     """Test eager workflow with asyncio gather."""
 
@@ -113,7 +112,7 @@ async def eager_wf(x: int, n: int) -> typing.List[int]:
 
 
 @given(x_input=INTEGER_ST)
-@settings(deadline=DEADLINE, max_examples=5)
+@pytest.mark.hypothesis
 def test_eager_workflow_with_dynamic_exception(x_input: int):
     """Test eager workflow with dynamic workflow is not supported."""
 
@@ -131,7 +130,7 @@ async def nested_eager_wf(x: int) -> int:
 
 
 @given(x_input=INTEGER_ST)
-@settings(deadline=DEADLINE, max_examples=5)
+@pytest.mark.hypothesis
 def test_nested_eager_workflow(x_input: int):
     """Testing running nested eager workflows."""
 
@@ -145,7 +144,7 @@ async def eager_wf(x: int) -> int:
 
 
 @given(x_input=INTEGER_ST)
-@settings(deadline=DEADLINE, max_examples=5)
+@pytest.mark.hypothesis
 def test_eager_workflow_within_workflow(x_input: int):
     """Testing running eager workflow within a static workflow."""
 
@@ -168,7 +167,7 @@ def subworkflow(x: int) -> int:
 
 
 @given(x_input=INTEGER_ST)
-@settings(deadline=DEADLINE, max_examples=5)
+@pytest.mark.hypothesis
 def test_workflow_within_eager_workflow(x_input: int):
     """Testing running a static workflow within an eager workflow."""
 
@@ -182,7 +181,7 @@ async def eager_wf(x: int) -> int:
 
 
 @given(x_input=INTEGER_ST)
-@settings(deadline=DEADLINE, max_examples=5)
+@pytest.mark.hypothesis
 def test_local_task_eager_workflow_exception(x_input: int):
     """Testing simple eager workflow with a local function task doesn't work."""
 
@@ -199,8 +198,8 @@ async def eager_wf_with_local(x: int) -> int:
 
 
 @given(x_input=INTEGER_ST)
-@settings(deadline=DEADLINE, max_examples=5)
 @pytest.mark.filterwarnings("ignore:coroutine 'AsyncEntity.__call__' was never awaited")
+@pytest.mark.hypothesis
 def test_local_workflow_within_eager_workflow_exception(x_input: int):
     """Cannot call a locally-defined workflow within an eager workflow"""
 
@@ -243,6 +242,7 @@ def create_directory() -> FlyteDirectory:
 
 
 @pytest.mark.skipif("pandas" not in sys.modules, reason="Pandas is not installed.")
+@pytest.mark.hypothesis
 def test_eager_workflow_with_offloaded_types():
     """Test eager workflow that eager workflows work with offloaded types."""
     import pandas as pd