From bebdd49ce1d1edeff297a6a564cdf498734c659a Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Tue, 20 Oct 2020 23:01:47 -0700 Subject: [PATCH 01/41] Add CI github actions workflow --- .github/workflows/ci.yml | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..71dcb7ba07 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI +on: + push: + branches: [develop] + pull_request: + +env: + CXXFLAGS: "-fprofile-arcs -ftest-coverage" + +jobs: + test: + name: "Python ${{ matrix.python-version }} / ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: ["2.7", "3.8"] + os: [ubuntu-latest] + compiler: [gcc] + env: + CC: ${{ matrix.compiler }} + PY2: ${{ startsWith(matrix.python-version, 2) && 'on' || 'off' }} + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq libboost swig libhdf5-serial-dev libeigen3-dev + - name: Set-up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: 'x64' + - name: Setup git + run: ./setup_git.py + - name: Build IMP + run: | + mkdir ../build + cd ../build + cmake ../imp -DUSE_PYTHON2=${{ PY2 }} -DCMAKE_CXX_FLAGS="${{ CXXFLAGS }}" + make -k -j 2 + - name: Run tests + run: ctest -j 2 -R XXX + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + fail_ci_if_error: false + verbose: true From 3b7544f7c7202d417c88092c7dc44ca9679cd5f6 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Tue, 20 Oct 2020 23:08:42 -0700 Subject: [PATCH 02/41] Correctly reference environmental variable --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71dcb7ba07..52a45d7276 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: run: | mkdir ../build cd ../build - cmake ../imp -DUSE_PYTHON2=${{ PY2 }} -DCMAKE_CXX_FLAGS="${{ CXXFLAGS }}" + cmake ../imp -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" make -k -j 2 - name: Run tests run: ctest -j 2 -R XXX From a846c358b710e76137b464f4c61facda72f9ac03 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Tue, 20 Oct 2020 23:10:42 -0700 Subject: [PATCH 03/41] Fix boost package name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52a45d7276..9f688dc158 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update -qq - sudo apt-get install -qq libboost swig libhdf5-serial-dev libeigen3-dev + sudo apt-get install -qq libboost-all-dev swig libhdf5-serial-dev libeigen3-dev - name: Set-up Python uses: actions/setup-python@v2 with: From 46f9023676165be087edf85a9cacb9f269c8930a Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Tue, 20 Oct 2020 23:12:21 -0700 Subject: [PATCH 04/41] Use same form of setup --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f688dc158..3725ff2226 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,12 +25,12 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libboost-all-dev swig libhdf5-serial-dev libeigen3-dev - - name: Set-up Python + - name: Set up python uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} architecture: 'x64' - - name: Setup git + - name: Set up git run: ./setup_git.py - name: Build IMP run: | From 10c5ec0a76ea708d2d0b1c7e7767427839300f7c Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Tue, 20 Oct 2020 23:54:31 -0700 Subject: [PATCH 05/41] Remove regex arg to ctest --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3725ff2226..446a36055e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: cmake ../imp -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" make -k -j 2 - name: Run tests - run: ctest -j 2 -R XXX + run: ctest -j 2 - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From c5a84f0f21103c9644de59554a7d400477a0435b Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Tue, 20 Oct 2020 23:54:47 -0700 Subject: [PATCH 06/41] Make ctest output on failure --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 446a36055e..c560481e8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: cmake ../imp -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" make -k -j 2 - name: Run tests - run: ctest -j 2 + run: ctest -j 2 --output-on-failure - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From 0a3dfe12439aa9918c352a77a7ecf6ff224931b1 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Wed, 21 Oct 2020 14:53:00 -0700 Subject: [PATCH 07/41] Install more dependencies --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c560481e8d..be2316f045 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,11 @@ jobs: - name: Install dependencies run: | sudo apt-get update -qq - sudo apt-get install -qq libboost-all-dev swig libhdf5-serial-dev libeigen3-dev + sudo apt-get install -qq libboost-all-dev swig libhdf5-serial-dev libeigen3-dev \ + python-numpy python3-biopython python3-numpy cmake libcgal-dev libcgal-qt5-dev \ + libfftw3-dev libopencv-dev libgsl0-dev python2-dev python3-dev libann-dev \ + libprotobuf-dev protobuf-compiler python-protobuf python3-protobuf \ + libopenmpi-dev - name: Set up python uses: actions/setup-python@v2 with: From f102355996e0d73cbba9fc788bba1eb7b62bfdc8 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Wed, 21 Oct 2020 15:00:49 -0700 Subject: [PATCH 08/41] Make build directory in imp directory --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be2316f045..acaa8e29ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,9 +38,9 @@ jobs: run: ./setup_git.py - name: Build IMP run: | - mkdir ../build - cd ../build - cmake ../imp -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" + mkdir build + cd build + cmake .. -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" make -k -j 2 - name: Run tests run: ctest -j 2 --output-on-failure From d2b07ccd10a4727a65163fadbc2bed8db2e67e19 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Wed, 21 Oct 2020 15:02:45 -0700 Subject: [PATCH 09/41] Replace ctest with nosetests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acaa8e29ff..7aae7c4edb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: cmake .. -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" make -k -j 2 - name: Run tests - run: ctest -j 2 --output-on-failure + run: ./setup_environment.sh ../test/nosetests --with-coverage --cover-branches -e 'probabilistic' ../test/test_*.py ../test/medium_test_*.py test/*/*.py > /dev/null - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From b18174ab285b9f0371134e42c13b72f0382b4df6 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Wed, 21 Oct 2020 15:02:57 -0700 Subject: [PATCH 10/41] Don't run codecov action verbosely --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7aae7c4edb..282b72843e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,4 +48,3 @@ jobs: uses: codecov/codecov-action@v1 with: fail_ci_if_error: false - verbose: true From 37f41219468f29e90c0a41226cd9f6bed522b188 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Wed, 21 Oct 2020 15:15:25 -0700 Subject: [PATCH 11/41] Fix package name python-dev is for Python 2 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 282b72843e..5cadef6646 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: sudo apt-get update -qq sudo apt-get install -qq libboost-all-dev swig libhdf5-serial-dev libeigen3-dev \ python-numpy python3-biopython python3-numpy cmake libcgal-dev libcgal-qt5-dev \ - libfftw3-dev libopencv-dev libgsl0-dev python2-dev python3-dev libann-dev \ + libfftw3-dev libopencv-dev libgsl0-dev python-dev python3-dev libann-dev \ libprotobuf-dev protobuf-compiler python-protobuf python3-protobuf \ libopenmpi-dev - name: Set up python From e8272d62b3302a8d25fb774c88c650529e44fe68 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Wed, 21 Oct 2020 16:14:34 -0700 Subject: [PATCH 12/41] Execute from build directory Any change of directories within a step does not propagate to the next step. --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cadef6646..cf29d5effa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,11 @@ jobs: cmake .. -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" make -k -j 2 - name: Run tests - run: ./setup_environment.sh ../test/nosetests --with-coverage --cover-branches -e 'probabilistic' ../test/test_*.py ../test/medium_test_*.py test/*/*.py > /dev/null + run: | + cd build + ./setup_environment.sh ../test/nosetests --with-coverage --cover-branches -e 'probabilistic' ../test/test_*.py ../test/medium_test_*.py test/*/*.py > /dev/null - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: fail_ci_if_error: false + directory: build From 15ba0e7e215f32b62310ee8f7ab3b6488ae17cea Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Wed, 21 Oct 2020 17:54:46 -0700 Subject: [PATCH 13/41] Add nose as dependency --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf29d5effa..95dfb9ded0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,10 @@ jobs: python-numpy python3-biopython python3-numpy cmake libcgal-dev libcgal-qt5-dev \ libfftw3-dev libopencv-dev libgsl0-dev python-dev python3-dev libann-dev \ libprotobuf-dev protobuf-compiler python-protobuf python3-protobuf \ - libopenmpi-dev + libopenmpi-dev python-nose python3-nose + # IMP tests use sys.argv[0] to determine their location, which won't work if + # we use nosetests, so add a workaround + ln -sf $(which nosetests) test/ - name: Set up python uses: actions/setup-python@v2 with: From 28d19b927a18a5a86154c98b5ed283a8dec2668f Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Wed, 21 Oct 2020 18:39:17 -0700 Subject: [PATCH 14/41] Change comment indentation --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95dfb9ded0..bb0b165b23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,8 +29,8 @@ jobs: libfftw3-dev libopencv-dev libgsl0-dev python-dev python3-dev libann-dev \ libprotobuf-dev protobuf-compiler python-protobuf python3-protobuf \ libopenmpi-dev python-nose python3-nose - # IMP tests use sys.argv[0] to determine their location, which won't work if - # we use nosetests, so add a workaround + # IMP tests use sys.argv[0] to determine their location, which won't work if + # we use nosetests, so add a workaround ln -sf $(which nosetests) test/ - name: Set up python uses: actions/setup-python@v2 From c875f3ad6a0419899d2e0418e38c390a8b78ec64 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Sun, 8 Nov 2020 18:34:08 -0800 Subject: [PATCH 15/41] Create test directory --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb0b165b23..3b011e73c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,7 @@ jobs: libopenmpi-dev python-nose python3-nose # IMP tests use sys.argv[0] to determine their location, which won't work if # we use nosetests, so add a workaround + mkdir test ln -sf $(which nosetests) test/ - name: Set up python uses: actions/setup-python@v2 From 1b1a065cd973257097954f06d11aed65e74598b2 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Sun, 8 Nov 2020 21:48:34 -0800 Subject: [PATCH 16/41] Don't install both Python 2 and 3 --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b011e73c8..76852e3292 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,14 @@ jobs: libfftw3-dev libopencv-dev libgsl0-dev python-dev python3-dev libann-dev \ libprotobuf-dev protobuf-compiler python-protobuf python3-protobuf \ libopenmpi-dev python-nose python3-nose + if [ ${{ env.PY2 }} == 2 ] + then + sudo apt-get install -qq python-dev python-numpy python-protobuf \ + python-nose python-pip + else + sudo apt-get install -qq python3-dev python3-numpy python3-protobuf \ + python3-nose python3-pip + fi # IMP tests use sys.argv[0] to determine their location, which won't work if # we use nosetests, so add a workaround mkdir test From 53c016138cdfd4a975fa166aad8e0b02036feea9 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Sun, 8 Nov 2020 21:49:00 -0800 Subject: [PATCH 17/41] Remove extraneous python install step --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76852e3292..bc19314917 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,11 +41,6 @@ jobs: # we use nosetests, so add a workaround mkdir test ln -sf $(which nosetests) test/ - - name: Set up python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' - name: Set up git run: ./setup_git.py - name: Build IMP From c3d942e773f8c561e091e75a1e10956438e3759c Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Sun, 8 Nov 2020 21:49:06 -0800 Subject: [PATCH 18/41] Install codecov --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc19314917..5b45c181c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: sudo apt-get install -qq python3-dev python3-numpy python3-protobuf \ python3-nose python3-pip fi + pip install codecov # IMP tests use sys.argv[0] to determine their location, which won't work if # we use nosetests, so add a workaround mkdir test From 232dfcd6a5ae77882b10392d172bc29fa68e6a43 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 22 Dec 2020 21:11:53 -0800 Subject: [PATCH 19/41] Fix check for Python 2 env --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b45c181c5..dc78c25c99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: libfftw3-dev libopencv-dev libgsl0-dev python-dev python3-dev libann-dev \ libprotobuf-dev protobuf-compiler python-protobuf python3-protobuf \ libopenmpi-dev python-nose python3-nose - if [ ${{ env.PY2 }} == 2 ] + if [ "${{ env.PY2 }}" == "on" ] then sudo apt-get install -qq python-dev python-numpy python-protobuf \ python-nose python-pip From cefd84884d521728cc1d89e0d77135a445b33c48 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 22 Dec 2020 21:16:09 -0800 Subject: [PATCH 20/41] Set correct Python 3 version Ubuntu 18.04 ships with Python 3.6, not 3.8. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc78c25c99..832fd0d03c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["2.7", "3.8"] - os: [ubuntu-latest] + python-version: ["2.7", "3.6"] + os: [ubuntu-18.04] compiler: [gcc] env: CC: ${{ matrix.compiler }} From 844eb15f3de619b74d44333cd54db8e45b959d31 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 22 Dec 2020 21:19:00 -0800 Subject: [PATCH 21/41] Use correct pip binary When using system Python on Ubuntu, 'pip' will always install Python 2 packages. Use 'pip3' for Python 3. --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 832fd0d03c..c0ab6cbcce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,11 +33,13 @@ jobs: then sudo apt-get install -qq python-dev python-numpy python-protobuf \ python-nose python-pip + export PIP=pip2 else sudo apt-get install -qq python3-dev python3-numpy python3-protobuf \ python3-nose python3-pip + export PIP=pip3 fi - pip install codecov + $PIP install codecov # IMP tests use sys.argv[0] to determine their location, which won't work if # we use nosetests, so add a workaround mkdir test From 0554bd937ca68fb1f4623e967cbddf0c98cfbfc7 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 22 Dec 2020 21:26:38 -0800 Subject: [PATCH 22/41] Use correct nosetests Don't use Python 2's nosetests when building IMP for Python 3, and vice versa. IMP itself should also be more nose-aware these days, so we can drop some of our test workarounds. --- .github/workflows/ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0ab6cbcce..f6b8485ca3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,10 +40,6 @@ jobs: export PIP=pip3 fi $PIP install codecov - # IMP tests use sys.argv[0] to determine their location, which won't work if - # we use nosetests, so add a workaround - mkdir test - ln -sf $(which nosetests) test/ - name: Set up git run: ./setup_git.py - name: Build IMP @@ -54,8 +50,13 @@ jobs: make -k -j 2 - name: Run tests run: | + if [ "${{ env.PY2 }}" == "on" ] + NOSE=nosetests + else + NOSE=nosetests3 + fi cd build - ./setup_environment.sh ../test/nosetests --with-coverage --cover-branches -e 'probabilistic' ../test/test_*.py ../test/medium_test_*.py test/*/*.py > /dev/null + ./setup_environment.sh $NOSE --with-coverage --cover-branches ../test/test_*.py ../test/medium_test_*.py test/*/*.py > /dev/null - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From c2d0e0ccf29b6eba465d11a8ef410bf4b8926452 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 22 Dec 2020 21:30:10 -0800 Subject: [PATCH 23/41] Help CMake to find CGAL Ubuntu puts CGAL in an odd location, and IMP's CMake can't find it. Help it out by setting CGAL_DIR. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6b8485ca3..875ef9d315 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: run: | mkdir build cd build - cmake .. -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" + cmake .. -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" -DCGAL_DIR=/usr/lib/x86_64-linux-gnu/cmake/CGAL/ make -k -j 2 - name: Run tests run: | From 000ee171afc9b28153cf3c267e370077aeabb3f5 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 22 Dec 2020 21:34:57 -0800 Subject: [PATCH 24/41] Run just IMP.kernel tests for now Give nose the correct paths for the IMP.kernel Python tests. Ultimately we will want it to run all modules' tests, although we will need to handle disabled modules appropriately. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 875ef9d315..8e7532820c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: NOSE=nosetests3 fi cd build - ./setup_environment.sh $NOSE --with-coverage --cover-branches ../test/test_*.py ../test/medium_test_*.py test/*/*.py > /dev/null + ./setup_environment.sh $NOSE --with-coverage --cover-branches ../modules/kernel/test/test_*.py ../modules/kernel/test/medium_test_*.py test/*/*.py > /dev/null - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From ca909b780fe05350bc6f1a9ee9e8c9d96f14de1f Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 22 Dec 2020 22:40:34 -0800 Subject: [PATCH 25/41] Add missing "then" to if statement --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e7532820c..e868d719f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: make -k -j 2 - name: Run tests run: | - if [ "${{ env.PY2 }}" == "on" ] + if [ "${{ env.PY2 }}" == "on" ]; then NOSE=nosetests else NOSE=nosetests3 From b10176365b2e4a004c775d76e2e439f23b2633de Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Sat, 9 Jan 2021 13:44:04 -0800 Subject: [PATCH 26/41] Add coverage module --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b517d9edba..1d167792fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,11 +32,13 @@ jobs: if [ "${{ env.PY2 }}" == "on" ] then sudo apt-get install -qq python-dev python-numpy python-protobuf \ - python-nose python-pip python-biopython + python-nose python-coverage python-pip \ + python-biopython export PIP=pip2 else sudo apt-get install -qq python3-dev python3-numpy python3-protobuf \ - python3-nose python3-pip python3-biopython + python3-nose python3-coverage python3-pip \ + python3-biopython export PIP=pip3 fi $PIP install codecov From 3cd828bd0225392f236d153285b90b7fec6b9c30 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Sat, 9 Jan 2021 15:07:19 -0800 Subject: [PATCH 27/41] Remove duplicated Python package installation --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d167792fa..b0a93049e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,10 +25,8 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libboost-all-dev swig libhdf5-serial-dev libeigen3-dev \ - python-numpy python3-biopython python3-numpy cmake libcgal-dev libcgal-qt5-dev \ - libfftw3-dev libopencv-dev libgsl0-dev python-dev python3-dev libann-dev \ - libprotobuf-dev protobuf-compiler python-protobuf python3-protobuf \ - libopenmpi-dev python-nose python3-nose + cmake libcgal-dev libcgal-qt5-dev libfftw3-dev libopencv-dev libgsl0-dev \ + libann-dev libprotobuf-dev protobuf-compiler libopenmpi-dev if [ "${{ env.PY2 }}" == "on" ] then sudo apt-get install -qq python-dev python-numpy python-protobuf \ From d4e40a6c0c3edc87070c032de38d11010c277736 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Sat, 9 Jan 2021 19:11:51 -0800 Subject: [PATCH 28/41] Compile only a few modules for faster iteration --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0a93049e8..b7e4edced0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,8 @@ jobs: env: CC: ${{ matrix.compiler }} PY2: ${{ startsWith(matrix.python-version, 2) && 'on' || 'off' }} + BUILD: "-j 2 IMP.kernel IMP.test" + PERCPP: "kernel" steps: - uses: actions/checkout@v2 - name: Install dependencies @@ -46,8 +48,10 @@ jobs: run: | mkdir build cd build - cmake .. -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" -DCGAL_DIR=/usr/lib/x86_64-linux-gnu/cmake/CGAL/ - make -k -j 2 + cmake .. -DUSE_PYTHON2=${{ env.PY2 }} -DCMAKE_CXX_FLAGS="${{ env.CXXFLAGS }}" \ + -DCGAL_DIR=/usr/lib/x86_64-linux-gnu/cmake/CGAL/ \ + -DIMP_PER_CPP_COMPILATION=${{ env.PERCPP }} + make -k ${{ env.BUILD }} - name: Run tests run: | if [ "${{ env.PY2 }}" == "on" ]; then From 0b376a066e993aa53edd9e95968588bb0798154f Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Sat, 9 Jan 2021 19:59:30 -0800 Subject: [PATCH 29/41] Only test kernel for now --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7e4edced0..c70b77cab0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,7 @@ jobs: NOSE=nosetests3 fi cd build - ./setup_environment.sh $NOSE --with-coverage --cover-branches ../modules/kernel/test/test_*.py ../modules/kernel/test/medium_test_*.py test/*/*.py > /dev/null + ./setup_environment.sh $NOSE --with-coverage --cover-branches ../modules/kernel/test/test_*.py ../modules/kernel/test/medium_test_*.py > /dev/null - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From c677c6260628f2aaa800a882c40f9a81fa0a33d3 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 17:01:21 -0800 Subject: [PATCH 30/41] Use pytest instead of nose nose is no longer maintained, so try running tests with pytest instead. --- .github/workflows/ci.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c70b77cab0..7c55ed0119 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,16 +32,16 @@ jobs: if [ "${{ env.PY2 }}" == "on" ] then sudo apt-get install -qq python-dev python-numpy python-protobuf \ - python-nose python-coverage python-pip \ + python-coverage python-pip \ python-biopython export PIP=pip2 else sudo apt-get install -qq python3-dev python3-numpy python3-protobuf \ - python3-nose python3-coverage python3-pip \ + python3-coverage python3-pip \ python3-biopython export PIP=pip3 fi - $PIP install codecov + $PIP install codecov pytest pytest-cov - name: Set up git run: ./setup_git.py - name: Build IMP @@ -54,13 +54,8 @@ jobs: make -k ${{ env.BUILD }} - name: Run tests run: | - if [ "${{ env.PY2 }}" == "on" ]; then - NOSE=nosetests - else - NOSE=nosetests3 - fi cd build - ./setup_environment.sh $NOSE --with-coverage --cover-branches ../modules/kernel/test/test_*.py ../modules/kernel/test/medium_test_*.py > /dev/null + ./setup_environment.sh pytest --cov=.. --cov-branch ../modules/kernel/test/test_*.py ../modules/kernel/test/medium_test_*.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From 43344979a818167ecf25b44a7193ca147516d71c Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 17:02:12 -0800 Subject: [PATCH 31/41] There are no "medium" tests in IMP.kernel --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c55ed0119..9dab710edf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Run tests run: | cd build - ./setup_environment.sh pytest --cov=.. --cov-branch ../modules/kernel/test/test_*.py ../modules/kernel/test/medium_test_*.py + ./setup_environment.sh pytest --cov=.. --cov-branch ../modules/kernel/test/test_*.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From ef05987e97a0675f8c3374a9b75e4c633fd7d13e Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 17:03:06 -0800 Subject: [PATCH 32/41] Run each test in a separate process If the underlying C++ code segfaults, protect the rest of the test run by running each test in a separate forked process. This should also reduce issues with tests that do things that can only be done once per process, e.g. parsing command line flags. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dab710edf..2443ea909a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: python3-biopython export PIP=pip3 fi - $PIP install codecov pytest pytest-cov + $PIP install codecov pytest pytest-cov pytest-xdist - name: Set up git run: ./setup_git.py - name: Build IMP @@ -55,7 +55,7 @@ jobs: - name: Run tests run: | cd build - ./setup_environment.sh pytest --cov=.. --cov-branch ../modules/kernel/test/test_*.py + ./setup_environment.sh pytest --cov=.. --cov-branch --forked ../modules/kernel/test/test_*.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From 02c3de02d993175982ac876e2c380f861ae6be14 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 17:06:24 -0800 Subject: [PATCH 33/41] Many kernel tests use IMP.core/algebra classes --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2443ea909a..145dc9220b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: env: CC: ${{ matrix.compiler }} PY2: ${{ startsWith(matrix.python-version, 2) && 'on' || 'off' }} - BUILD: "-j 2 IMP.kernel IMP.test" + BUILD: "-j 2 IMP.kernel IMP.test IMP.core IMP.algebra" PERCPP: "kernel" steps: - uses: actions/checkout@v2 From bbe1122dbb2c3806025539063d0f2ec94fd42b58 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 17:59:14 -0800 Subject: [PATCH 34/41] Don't cancel others if one build fails --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 145dc9220b..212250cae4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ jobs: name: "Python ${{ matrix.python-version }} / ${{ matrix.os }}" runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: python-version: ["2.7", "3.6"] os: [ubuntu-18.04] From e4b18c16b1f1fe6a6fe53ba0122da4e29ba26685 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 18:00:25 -0800 Subject: [PATCH 35/41] Use older py.test binary "pytest" only works with newer pytest (not that available for Python 2). Use the older "py.test" binary instead. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 212250cae4..c7c14b0c8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: - name: Run tests run: | cd build - ./setup_environment.sh pytest --cov=.. --cov-branch --forked ../modules/kernel/test/test_*.py + ./setup_environment.sh py.test --cov=.. --cov-branch --forked ../modules/kernel/test/test_*.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From a75a4afc241c3cf06885f065537614a3be68c7cc Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 16:48:49 -0800 Subject: [PATCH 36/41] Run test via unittest The test as currently written doesn't work when run with pytest because it doesn't understand the sys.argv that pytest feeds it. Use unittest instead. --- modules/kernel/test/test_progress.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/modules/kernel/test/test_progress.py b/modules/kernel/test/test_progress.py index 196bd3b2aa..3a61248c55 100644 --- a/modules/kernel/test/test_progress.py +++ b/modules/kernel/test/test_progress.py @@ -1,14 +1,21 @@ import IMP +import IMP.test import sys -IMP.set_deprecation_exceptions(True) -IMP.setup_from_argv(sys.argv, "Test progress bar") -IMP.set_log_level(IMP.PROGRESS) -IMP.set_progress_display("Test progress", 110) -for i in range(0, 110, 5): - IMP.add_to_progress_display(5) +class Tests(IMP.test.TestCase): -IMP.set_progress_display("Test progress round 2", 130) -for i in range(0, 130, 5): - IMP.add_to_progress_display(5) + def test_progress(self): + """Test progress bar""" + IMP.set_log_level(IMP.PROGRESS) + IMP.set_progress_display("Test progress", 110) + for i in range(0, 110, 5): + IMP.add_to_progress_display(5) + + IMP.set_progress_display("Test progress round 2", 130) + for i in range(0, 130, 5): + IMP.add_to_progress_display(5) + + +if __name__ == '__main__': + IMP.test.main() From 6018f191fd945f671668fc648ea432a73ab638c7 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 20:53:38 -0800 Subject: [PATCH 37/41] Install .deb packages for pytest, not pip --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7c14b0c8a..8176fad49c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,15 +34,17 @@ jobs: then sudo apt-get install -qq python-dev python-numpy python-protobuf \ python-coverage python-pip \ - python-biopython + python-biopython python-pytest-xdist \ + python-pytest-cov export PIP=pip2 else sudo apt-get install -qq python3-dev python3-numpy python3-protobuf \ python3-coverage python3-pip \ - python3-biopython + python3-biopython python3-pytest-xdist \ + python3-pytest-cov export PIP=pip3 fi - $PIP install codecov pytest pytest-cov pytest-xdist + $PIP install codecov - name: Set up git run: ./setup_git.py - name: Build IMP @@ -56,7 +58,7 @@ jobs: - name: Run tests run: | cd build - ./setup_environment.sh py.test --cov=.. --cov-branch --forked ../modules/kernel/test/test_*.py + ./setup_environment.sh pytest --cov=.. --cov-branch --forked ../modules/kernel/test/test_*.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From 403966610e6ca04075751be2eb97fae462dc6a51 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 21:39:53 -0800 Subject: [PATCH 38/41] Need IMP.container to run some kernel tests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8176fad49c..10cc9bedc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: env: CC: ${{ matrix.compiler }} PY2: ${{ startsWith(matrix.python-version, 2) && 'on' || 'off' }} - BUILD: "-j 2 IMP.kernel IMP.test IMP.core IMP.algebra" + BUILD: "-j 2 IMP.kernel IMP.test IMP.core IMP.algebra IMP.container" PERCPP: "kernel" steps: - uses: actions/checkout@v2 From ecc8e65aba66fb2436b81cf9bea354ae6261f186 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Wed, 13 Jan 2021 21:41:46 -0800 Subject: [PATCH 39/41] Fix pytest name Ubuntu calls the Python 3 version of pytest "pytest-3". --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10cc9bedc7..5ac233af84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,8 +57,13 @@ jobs: make -k ${{ env.BUILD }} - name: Run tests run: | + if [ "${{ env.PY2 }}" == "on" ]; then + export PYTEST=pytest + else + export PYTEST=pytest-3 + fi cd build - ./setup_environment.sh pytest --cov=.. --cov-branch --forked ../modules/kernel/test/test_*.py + ./setup_environment.sh $PYTEST --cov=.. --cov-branch --forked ../modules/kernel/test/test_*.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From 1db30a4cc16be1784737ae11320752c996d9b3d1 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Thu, 14 Jan 2021 03:18:29 -0800 Subject: [PATCH 40/41] Increase fetch-depth --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ac233af84..29a1a4761a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: PERCPP: "kernel" steps: - uses: actions/checkout@v2 + with: + fetch-depth: 2 - name: Install dependencies run: | sudo apt-get update -qq From 8ffe8da73390a36210e0161b23272997e1157487 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Thu, 14 Jan 2021 03:28:54 -0800 Subject: [PATCH 41/41] Explicitly output xml coverage report --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29a1a4761a..29aa92214c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: export PYTEST=pytest-3 fi cd build - ./setup_environment.sh $PYTEST --cov=.. --cov-branch --forked ../modules/kernel/test/test_*.py + ./setup_environment.sh $PYTEST --cov=.. --cov-report=xml --cov-branch --forked ../modules/kernel/test/test_*.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: