tests: add system tests using pytest-mh framework

Reviewed-by: Jakub Vávra <[email protected]>

.github/workflows/ci.yml

@@ -247,12 +247,136 @@ jobs:
           multihost-install.log
           multihost-pytest.log
 
+  system:
+    needs: [prepare, build]
+    strategy:
+      fail-fast: false
+      matrix:
+        tag: ${{ fromJson(needs.prepare.outputs.matrix).multihost }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - name: Checkout sssd repository
+      uses: actions/checkout@v3
+      with:
+        path: sssd
+
+    - name: Setup containers
+      uses: SSSD/sssd-ci-containers/actions/setup@master
+      with:
+        path: sssd-ci-containers
+        tag: ${{ matrix.tag }}
+        override: |
+          services:
+            client:
+              image: ${REGISTRY}/ci-client-devel:${TAG}
+              shm_size: 4G
+              tmpfs:
+              - /dev/shm
+              volumes:
+              - ../sssd:/sssd:rw
+
+    - name: Build SSSD on the client
+      uses: SSSD/sssd-ci-containers/actions/exec@master
+      with:
+        log-file: build.log
+        working-directory: /sssd
+        script: |
+          #!/bin/bash
+          set -ex
+
+          ./contrib/ci/run --deps-only
+          autoreconf -if
+
+          mkdir -p /dev/shm/sssd
+          pushd /dev/shm/sssd
+          /sssd/configure --enable-silent-rules
+          make rpms
+
+    - name: Install SSSD on the client
+      uses: SSSD/sssd-ci-containers/actions/exec@master
+      with:
+        log-file: install.log
+        user: root
+        script: |
+          #!/bin/bash
+          set -ex
+
+          dnf remove -y --noautoremove sssd\*
+          dnf install -y /dev/shm/sssd/rpmbuild/RPMS/*/*.rpm
+          rm -fr /dev/shm/sssd
+
+          # We need to reenable sssd-kcm since it was disabled by removing sssd not not enabled again
+          systemctl enable --now sssd-kcm.socket
+
+    - name: Install system tests dependencies
+      shell: bash
+      working-directory: ./sssd/src/tests/system
+      run: |
+        set -ex
+
+        sudo apt-get update
+
+        # Install dependencies for python-ldap
+        sudo apt-get install -y libsasl2-dev python3-dev libldap2-dev libssl-dev
+
+        # Virtualenv
+        pip3 install virtualenv
+        python3 -m venv .venv
+        source .venv/bin/activate
+
+        # Install system tests requirements
+        pip3 install -r ./requirements.txt
+
+        # Install yq to parse yaml files
+        sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+        sudo chmod a+x /usr/local/bin/yq
+
+    - name: Remove ad from mhc.yaml
+      shell: bash
+      working-directory: ./sssd/src/tests/system
+      run: |
+        yq -i 'del(.domains[0].hosts.[] | select(.role == "ad"))' mhc.yaml
+
+    - name: Run tests
+      shell: bash
+      working-directory: ./sssd/src/tests/system
+      run: |
+        set -ex -o pipefail
+
+        source .venv/bin/activate
+        pytest \
+          --color=yes \
+          --mh-config=./mhc.yaml \
+          --mh-log-path=$GITHUB_WORKSPACE/mh.log \
+          --mh-artifacts-dir=$GITHUB_WORKSPACE/artifacts \
+          -vvv . |& tee $GITHUB_WORKSPACE/pytest.log
+
+    - name: Upload artifacts
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        if-no-files-found: ignore
+        name: ${{ matrix.tag }}-system
+        path: |
+          sssd/ci-install-deps.log
+          artifacts
+          mh.log
+          build.log
+          install.log
+          pytest.log
+
   result:
     name: All tests are successful
     if: ${{ always() }}
     runs-on: ubuntu-latest
-    needs: [build, intgcheck, multihost]
+    needs: [build, intgcheck, multihost, system]
     steps:
       - name: Fail on failure
-        if: ${{ needs.build.result != 'success' || needs.intgcheck.result != 'success' || needs.multihost.result != 'success' }}
+        if: |
+          needs.build.result != 'success'
+          || needs.intgcheck.result != 'success'
+          || needs.multihost.result != 'success'
+          || needs.system.result != 'success'
         run: exit 1

.github/workflows/docs.yml

@@ -0,0 +1,50 @@
+name: Documentation
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  system-tests:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Setup virtual environment
+      working-directory: ./src/tests/system
+      run: |
+        sudo apt-get update
+
+        # Install dependencies for python-ldap
+        sudo apt-get install -y libsasl2-dev python3-dev libldap2-dev libssl-dev
+
+        pip3 install virtualenv
+        python3 -m venv .venv
+        source .venv/bin/activate
+        pip3 install -r ./requirements.txt
+        pip3 install -r ./docs/requirements.txt
+
+    - name: Build docs
+      working-directory: ./src/tests/system
+      run: |
+        source .venv/bin/activate
+        cd docs
+        make html SPHINXOPTS="-W --keep-going"
+
+  result:
+    name: All documentation builds correctly
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [system-tests]
+    steps:
+      - name: Fail on failure
+        if: ${{ needs.system-tests.result != 'success' }}
+        run: exit 1

.github/workflows/static-code-analysis.yml

@@ -41,6 +41,53 @@ jobs:
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2
 
+  python-system-tests:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Setup virtual environment
+      working-directory: ./src/tests/system
+      run: |
+        sudo apt-get update
+
+        # Install dependencies for python-ldap
+        sudo apt-get install -y libsasl2-dev python3-dev libldap2-dev libssl-dev
+
+        pip3 install virtualenv
+        python3 -m venv .venv
+        source .venv/bin/activate
+        pip3 install -r ./requirements.txt
+        pip3 install flake8 pycodestyle isort mypy black
+
+    - name: flake8
+      if: always()
+      working-directory: ./src/tests/system
+      run: source .venv/bin/activate && flake8 .
+
+    - name: pycodestyle
+      if: always()
+      working-directory: ./src/tests/system
+      run: source .venv/bin/activate && pycodestyle .
+
+    - name: isort
+      if: always()
+      working-directory: ./src/tests/system
+      run: source .venv/bin/activate && isort --check-only .
+
+    - name: black
+      if: always()
+      working-directory: ./src/tests/system
+      run: source .venv/bin/activate && black --check --diff .
+
+    - name: mypy
+      if: always()
+      working-directory: ./src/tests/system
+      run: source .venv/bin/activate && mypy --install-types --non-interactive lib tests
+
   result:
     name: All tests are successful
     if: ${{ always() }}

.gitignore

@@ -112,9 +112,19 @@ sss_ssh_knownhostsproxy
 sssd_ssh
 test-authtok
 /ci-build-*
+.pytest_cache
+__pycache__
+.venv
 
 # multihost tests
 !src/tests/multihost/sssd
 !src/tests/multihost/docs/Makefile
 !src/tests/multihost/setup.py
 !src/tests/multihost/sssd-testlib.spec
+
+# system tests
+src/tests/system/artifacts
+src/tests/system/docs/_build
+src/tests/system/docs/api
+!src/tests/system/docs/Makefile
+!src/tests/system/lib/sssd

src/tests/intg/test_kcm.py

@@ -279,11 +279,13 @@ def kcm_overwrite(testenv):
     assert exp_ccache == testenv.k5util.list_all_princs()
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__kinit_overwrite')
 def test_kcm_mem_overwrite(setup_for_kcm_mem):
     testenv = setup_for_kcm_mem
     kcm_overwrite(testenv)
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__kinit_overwrite')
 def test_kcm_secdb_overwrite(setup_for_kcm_secdb):
     testenv = setup_for_kcm_secdb
     kcm_overwrite(testenv)
@@ -364,11 +366,13 @@ def collection_init_list_destroy(testenv):
     assert testenv.k5util.num_princs() == 0
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__kinit_collection')
 def test_kcm_mem_collection_init_list_destroy(setup_for_kcm_mem):
     testenv = setup_for_kcm_mem
     collection_init_list_destroy(testenv)
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__kinit_collection')
 def test_kcm_secdb_collection_init_list_destroy(setup_for_kcm_secdb):
     testenv = setup_for_kcm_secdb
     collection_init_list_destroy(testenv)
@@ -418,11 +422,13 @@ def exercise_kswitch(testenv):
                                                'host/differenthostname@KCMTEST'])
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__kinit_switch')
 def test_kcm_mem_kswitch(setup_for_kcm_mem):
     testenv = setup_for_kcm_mem
     exercise_kswitch(testenv)
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__kinit_switch')
 def test_kcm_secdb_kswitch(setup_for_kcm_secdb):
     testenv = setup_for_kcm_secdb
     exercise_kswitch(testenv)
@@ -471,11 +477,13 @@ def exercise_subsidiaries(testenv):
                                                'host/differenthostname@KCMTEST'])
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__subsidiaries')
 def test_kcm_mem_subsidiaries(setup_for_kcm_mem):
     testenv = setup_for_kcm_mem
     exercise_subsidiaries(testenv)
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__subsidiaries')
 def test_kcm_secdb_subsidiaries(setup_for_kcm_secdb):
     testenv = setup_for_kcm_secdb
     exercise_subsidiaries(testenv)
@@ -495,11 +503,13 @@ def kdestroy_nocache(testenv):
     assert out == 0
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__kdestroy_nocache')
 def test_kcm_mem_kdestroy_nocache(setup_for_kcm_mem):
     testenv = setup_for_kcm_mem
     exercise_subsidiaries(testenv)
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__kdestroy_nocache')
 def test_kcm_secdb_kdestroy_nocache(setup_for_kcm_secdb):
     testenv = setup_for_kcm_secdb
     exercise_subsidiaries(testenv)
@@ -509,6 +519,7 @@ def get_secrets_socket():
     return os.path.join(config.RUNSTATEDIR, "secrets.socket")
 
 
+@pytest.mark.converted('test_kcm.py', 'test_kcm__tgt_renewal')
 @pytest.mark.skipif(not have_kcm_renewal(),
                     reason="KCM renewal disabled, skipping")
 def test_kcm_renewals(setup_for_kcm_renewals_secdb):

src/tests/intg/test_sudo.py

@@ -232,6 +232,7 @@ def add_common_rules(request, ldap_conn):
     return None
 
 
+@pytest.mark.converted('test_sudo.py', 'test_sudo__user_allowed')
 def test_sudo_rule_for_user(add_common_rules, sudocli_tool):
     """
     Test that user1 is allowed in the rule but user2 is not
@@ -264,6 +265,7 @@ def add_double_qualified_rules(request, ldap_conn):
     return None
 
 
+@pytest.mark.converted('test_sudo.py', 'test_sudo__duplicate_sudo_user')
 def test_sudo_rule_duplicate_sudo_user(add_double_qualified_rules,
                                        sudocli_tool):
     """

src/tests/multihost/ad/test_sudo.py

@@ -36,6 +36,7 @@ def class_setup(cls, multihost):
         client.clear_sssd_cache()
 
     @staticmethod
+    @pytest.mark.converted('test_sudo.py', 'test_sudo__case_sensitive_false')
     def test_001_bz1380436(multihost):
         """test_001_bz1380436
 
@@ -70,6 +71,7 @@ def test_001_bz1380436(multihost):
         assert not failed, f"Rules missing for users: {','.join(failed)}"
 
     @staticmethod
+    @pytest.mark.converted('test_sudo.py', 'test_sudo__sudo_user_is_group')
     def test_002_bz1372440(multihost):
         """test_002_bz1372440
 
@@ -110,6 +112,7 @@ def test_002_bz1372440(multihost):
         assert cmd2.returncode == 0, f"Sudo command failed for user: {user}!"
 
     @staticmethod
+    @pytest.mark.converted('test_sudo.py', 'test_sudo__sudo_user_is_nonposix_group')
     def test_003_support_non_posix_group_in_sudorule(multihost):
         """test_003_support_non_posix_group_in_sudorule
 
@@ -156,6 +159,7 @@ def test_003_support_non_posix_group_in_sudorule(multihost):
         assert test_result, f"Rules missing for user: {user}."
 
     @staticmethod
+    @pytest.mark.converted('test_sudo.py', 'test_sudo__runasuser_shortname')
     def test_004_sudorule_with_short_username(multihost):
         """test_004_sudorule_with_short_username