Merge pull request #1798 from abhisrkckl/parallel-tox

Run tests in parallel

CHANGELOG-unreleased.md

@@ -11,6 +11,7 @@ the released changes.
 ### Changed
 - Moved the events -> TOAs and photon weights code into the function `load_events_weights` within `event_optimize`.
 - Updated the `maxMJD` argument in `event_optimize` to default to the current mjd
+- Run CI tests in parallel, re-run failed tests (for intermittent failures due to random chance)
 - `maskParameter.__repr__()` output now includes the frozen attribute.
 - Changed default value of `FDJUMPLOG` to `Y`
 - Bumped `black` version to 24.x

requirements_dev.txt

@@ -12,6 +12,8 @@ wheel>=0.29.0
 pytest>=4.3
 pytest-cov>=2.7.1
 pytest-runner>=5.1
+pytest-xdist
+pytest-rerunfailures
 flake8>=3.7
 pep8-naming>=0.8.2
 flake8-docstrings>=1.4

tests/test_noisefit.py

@@ -6,27 +6,38 @@
 
 from io import StringIO
 import numpy as np
+import pytest
 
-par = """
-    ELAT    1.3     1
-    ELONG   2.5     1
-    F0      100     1
-    F1      1e-13   1
-    PEPOCH  55000
-    EPHEM   DE440
-    EFAC mjd 50000 53000 2      1
-    EQUAD mjd 53000 55000 0.8    1
-"""
 
-m = get_model(StringIO(par))
-t = make_fake_toas_uniform(50000, 55000, 200, m, add_noise=True)
+@pytest.fixture
+def model_and_toas_1():
+    par = """
+        ELAT    1.3     1
+        ELONG   2.5     1
+        F0      100     1
+        F1      1e-13   1
+        PEPOCH  55000
+        EPHEM   DE440
+        EFAC mjd 50000 53000 2      1
+        EQUAD mjd 53000 55000 0.8    1
+    """
 
-m2, t2 = get_model_and_toas(
-    datadir / "ecorr_fit_test.par", datadir / "ecorr_fit_test.tim"
-)
+    m = get_model(StringIO(par))
+    t = make_fake_toas_uniform(50000, 55000, 200, m, add_noise=True)
 
+    return m, t
+
+
+@pytest.fixture
+def model_and_toas_2():
+    return get_model_and_toas(
+        datadir / "ecorr_fit_test.par", datadir / "ecorr_fit_test.tim"
+    )
+
+
+def test_white_noise_fit(model_and_toas_1):
+    m, t = model_and_toas_1
 
-def test_white_noise_fit():
     assert m.EFAC1.uncertainty_value == 0 and m.EQUAD1.uncertainty_value == 0
 
     ftr = DownhillWLSFitter(t, m)
@@ -47,7 +58,9 @@ def test_white_noise_fit():
     )
 
 
-def test_white_noise_refit():
+def test_white_noise_refit(model_and_toas_1):
+    m, t = model_and_toas_1
+
     ftr = DownhillWLSFitter(t, m)
 
     ftr.model.EFAC1.value = 1.5
@@ -67,7 +80,9 @@ def test_white_noise_refit():
     )
 
 
-def test_ecorr_fit():
+def test_ecorr_fit(model_and_toas_2):
+    m2, t2 = model_and_toas_2
+
     ftr = DownhillGLSFitter(t2, m2)
     ftr.fit_toas()
 
@@ -79,7 +94,9 @@ def test_ecorr_fit():
     )
 
 
-def test_ecorr_refit():
+def test_ecorr_refit(model_and_toas_2):
+    m2, t2 = model_and_toas_2
+
     ftr = DownhillGLSFitter(t2, m2)
 
     ftr.model.ECORR1.value = 0.75

tests/test_publish.py

@@ -5,24 +5,37 @@
 from pint.scripts import pintpublish
 import os
 
-data_NGC6440E = get_model_and_toas(datadir / "NGC6440E.par", datadir / "NGC6440E.tim")
 
+@pytest.fixture
+def data_NGC6440E():
+    return get_model_and_toas(datadir / "NGC6440E.par", datadir / "NGC6440E.tim")
 
-def test_NGC6440E():
+
+@pytest.fixture
+def data_J0613m0200_NANOGrav_9yv1():
+    return get_model_and_toas(
+        datadir / "J0613-0200_NANOGrav_9yv1.gls.par",
+        datadir / "J0613-0200_NANOGrav_9yv1.tim",
+    )
+
+
+@pytest.fixture
+def data_J1614m2230_NANOGrav_12yv3_wb():
+    return get_model_and_toas(
+        datadir / "J1614-2230_NANOGrav_12yv3.wb.gls.par",
+        datadir / "J1614-2230_NANOGrav_12yv3.wb.tim",
+    )
+
+
+def test_NGC6440E(data_NGC6440E):
     m, t = data_NGC6440E
     output = publish(m, t)
     assert "1748-2021E" in output
     assert "DE421" in output
 
 
-data_J0613m0200_NANOGrav_9yv1 = get_model_and_toas(
-    datadir / "J0613-0200_NANOGrav_9yv1.gls.par",
-    datadir / "J0613-0200_NANOGrav_9yv1.tim",
-)
-
-
 @pytest.mark.parametrize("full", [True, False])
-def test_J0613m0200_NANOGrav_9yv1(full):
+def test_J0613m0200_NANOGrav_9yv1(data_J0613m0200_NANOGrav_9yv1, full):
     m, t = data_J0613m0200_NANOGrav_9yv1
     output = publish(
         m, t, include_dmx=full, include_fd=full, include_noise=full, include_jumps=full
@@ -39,14 +52,8 @@ def test_J0613m0200_NANOGrav_9yv1(full):
     assert not full or "RNAMP" in output
 
 
-data_J1614m2230_NANOGrav_12yv3_wb = get_model_and_toas(
-    datadir / "J1614-2230_NANOGrav_12yv3.wb.gls.par",
-    datadir / "J1614-2230_NANOGrav_12yv3.wb.tim",
-)
-
-
 @pytest.mark.parametrize("full", [True, False])
-def test_J1614m2230_NANOGrav_12yv3_wb(full):
+def test_J1614m2230_NANOGrav_12yv3_wb(data_J1614m2230_NANOGrav_12yv3_wb, full):
     m, t = data_J1614m2230_NANOGrav_12yv3_wb
     output = publish(
         m, t, include_dmx=full, include_fd=full, include_noise=full, include_jumps=full

tests/test_timing_model.py

@@ -42,9 +42,6 @@ def timfile_nojumps():
     return get_TOAs(os.path.join(datadir, "NGC6440E.tim"))
 
 
-len_timfile_nojumps = len(get_TOAs(os.path.join(datadir, "NGC6440E.tim")))
-
-
 class TestModelBuilding:
     def setup_method(self):
         self.parfile = os.path.join(datadir, "J0437-4715.par")

tests/test_toa_shuffle.py

@@ -18,33 +18,22 @@
 import pint.residuals
 from pint.models import get_model
 
-shuffletoas = """FORMAT 1
-test 1234.0 54321 0 pks
-test2 888 59055 0 meerkat
-test3 350 59000 0 gbt
-"""
-
 
 class TOAOrderSetup:
     parfile = os.path.join(datadir, "NGC6440E.par")
     model = get_model(parfile)
     # fake a multi-telescope, multi-frequency data-set and make sure the results don't depend on TOA order
-    fakes = [
+    t = (
         simulation.make_fake_toas_uniform(
             55000, 55500, 30, model=model, freq=1400 * u.MHz, obs="ao"
-        ),
-        simulation.make_fake_toas_uniform(
+        )
+        + simulation.make_fake_toas_uniform(
             55010, 55500, 40, model=model, freq=800 * u.MHz, obs="gbt"
-        ),
-        simulation.make_fake_toas_uniform(
+        )
+        + simulation.make_fake_toas_uniform(
             55020, 55500, 50, model=model, freq=2000 * u.MHz, obs="@"
-        ),
-    ]
-    f = io.StringIO()
-    for t in fakes:
-        t.write_TOA_file(f)
-    f.seek(0)
-    t = toa.get_TOAs(f)
+        )
+    )
     r = pint.residuals.Residuals(t, model, subtract_mean=False)
 
     @classmethod
@@ -95,6 +84,11 @@ def test_resorting_toas_chi2_match(sortkey):
 
 
 class TOALineOrderSetup:
+    shuffletoas = """FORMAT 1
+        test 1234.0 54321 0 pks
+        test2 888 59055 0 meerkat
+        test3 350 59000 0 gbt
+    """
     timfile = io.StringIO(shuffletoas)
     t = toa.get_TOAs(timfile)
     timfile.seek(0)

tox.ini

@@ -18,7 +18,8 @@ envlist =
 skip_missing_interpreters = True
 
 [tool:pytest]
-# pytest docs seem to say that this section should be called just pytest, not tool:pytest; is it working?
+# pytest docs seem to say that this section should be called just pytest, 
+# not tool:pytest; is it working?
 testpaths = tests
 addopts =
     --cov-report=term-missing
@@ -33,6 +34,8 @@ passenv =
 
 deps =
     pytest
+    pytest-xdist
+    pytest-rerunfailures
     cov: coverage
     cov: pytest-cov
     cov: pytest-remotedata
@@ -42,8 +45,9 @@ deps =
     setuptools
 commands =
     pip freeze
-    !cov: pytest 
-    cov: pytest -v --pyargs tests --cov=pint --cov-config={toxinidir}/.coveragerc {posargs}
+    !cov: pytest --reruns 5 --verbose tests/test_toa_selection.py
+    !cov: pytest -n 6 --reruns 5 --verbose --ignore=tests/test_toa_selection.py
+    cov: pytest -n 6 --reruns 5 --verbose --pyargs tests --cov=pint --cov-config={toxinidir}/.coveragerc {posargs}
     cov: coverage xml -o {toxinidir}/coverage.xml
 
 depends =
@@ -70,9 +74,13 @@ deps =
     matplotlib==3.2.0
     scipy==1.4.1
     pytest
+    pytest-xdist
+    pytest-rerunfailures
     coverage
     hypothesis<=6.72.0
-commands = {posargs:pytest}
+commands = 
+    pytest -n 6 --reruns 5 --verbose
+
 
 [testenv:report]
 skip_install = true