diff --git a/.github/actions/install_compatibility_layer/entrypoint.sh b/.github/actions/install_compatibility_layer/entrypoint.sh
index b4904ddd..408cbc04 100755
--- a/.github/actions/install_compatibility_layer/entrypoint.sh
+++ b/.github/actions/install_compatibility_layer/entrypoint.sh
@@ -6,7 +6,3 @@ cat << EOF > hosts
 EOF
 
 ansible-playbook -v --connection=local --inventory=hosts -e ansible_python_interpreter=python3 -e gentoo_prefix_path=$1 ${GITHUB_WORKSPACE}/ansible/playbooks/install.yml
-
-# A successful installation should at least have Lmod and archspec,
-# so let's  check if we can use them.
-source $1/usr/share/Lmod/init/profile && module avail && $1/usr/bin/archspec cpu
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml
index 2befa325..e326b647 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml
@@ -31,6 +31,10 @@
 
   - include_tasks: create_host_symlinks.yml
 
+  - include_tasks: test.yml
+    tags:
+      - test
+
   - name: Publish transaction
     command: "cvmfs_server publish {{ cvmfs_repository }}"
     when: cvmfs_start_transaction and cvmfs_publish_transaction
@@ -39,3 +43,7 @@
     - name: Abort transaction
       command: "cvmfs_server abort {{ cvmfs_repository }}"
       when: cvmfs_start_transaction and cvmfs_abort_transaction_on_failures
+
+    - name: Exit because of failure
+      fail:
+        msg: "Task {{ ansible_failed_task }} failed, with result {{ ansible_failed_result }}."
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
new file mode 100644
index 00000000..a60a7937
--- /dev/null
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
@@ -0,0 +1,23 @@
+# Run a ReFrame test suite to validate the correctness 
+# of the compatibility layer installation.
+---
+
+- name: Copy ReFrame test file
+  copy:
+    src: "{{ playbook_dir }}/../../test/compat_layer.py"
+    dest: /tmp/compat_layer.py
+    mode: 0644
+  tags:
+    - test
+
+- name: Run ReFrame tests
+  command: reframe -r -v -c /tmp/compat_layer.py
+  environment:
+    EESSI_VERSION: "{{ eessi_version }}"
+    EESSI_OS: "{{ eessi_host_os }}"
+    EESSI_ARCH: "{{ eessi_host_arch }}"
+  register: reframe_cmd
+  changed_when: false
+  failed_when: reframe_cmd.rc != 0
+  tags:
+    - test
diff --git a/test/compat_layer.py b/test/compat_layer.py
index c45e35c5..72830954 100644
--- a/test/compat_layer.py
+++ b/test/compat_layer.py
@@ -11,7 +11,7 @@ class RunInGentooPrefixTestError(rfm.core.exceptions.ReframeError):
 class RunInGentooPrefixTest(rfm.RunOnlyRegressionTest):
     eessi_repo_dir = '/cvmfs/pilot.eessi-hpc.org'
     eessi_version = parameter(
-        os.environ.get('EESSI_VERSION', os.readlink(os.path.join(eessi_repo_dir, 'latest'))).split(',')
+        os.environ.get('EESSI_VERSION', 'latest').split(',')
     )
     eessi_arch = parameter(
         os.environ.get('EESSI_ARCH', platform.machine()).split(',')
@@ -23,6 +23,9 @@ class RunInGentooPrefixTest(rfm.RunOnlyRegressionTest):
     def __init__(self):
         self.valid_systems = ['*']
         self.valid_prog_environs = ['*']
+        if self.eessi_version == 'latest':
+            # resolve the "latest" symlink to the actual version
+            self.eessi_version = os.readlink(os.path.join(eessi_repo_dir, 'latest'))
         self.compat_dir = os.path.join(
             self.eessi_repo_dir,
             self.eessi_version,