Use base configs for molecule setup

.github/workflows/molecule-firewalld.yml

@@ -10,4 +10,4 @@ jobs:
   molecule-firewalld:
     uses: ./.github/workflows/molecule.yml
     with:
-      run-tags: firewalld
+      tests-path: ansible_collections/mirsg/infrastructure/roles/firewalld

.github/workflows/molecule-postgresql.yml

@@ -10,4 +10,4 @@ jobs:
   molecule-postgresql:
     uses: ./.github/workflows/molecule.yml
     with:
-      run-tags: provision, postgresql
+      tests-path: ansible_collections/mirsg/infrastructure/roles/postgresql

.github/workflows/molecule-provision.yml

@@ -10,4 +10,4 @@ jobs:
   molecule-provision:
     uses: ./.github/workflows/molecule.yml
     with:
-      run-tags: provision
+      tests-path: ansible_collections/mirsg/infrastructure/roles/provision

.github/workflows/molecule-provision_accounts.yml

@@ -8,7 +8,7 @@ on:
       - ".github/workflows/molecule-provision_accounts.yml"
 
 jobs:
-  molecule-provision:
+  molecule-provision-accounts:
     uses: ./.github/workflows/molecule.yml
     with:
-      run-tags: provision_accounts
+      tests-path: ansible_collections/mirsg/infrastructure/roles/provision_accounts

.github/workflows/molecule-python.yml

@@ -10,4 +10,4 @@ jobs:
   molecule-firewalld:
     uses: ./.github/workflows/molecule.yml
     with:
-      run-tags: python
+      tests-path: ansible_collections/mirsg/infrastructure/roles/install_python

.github/workflows/molecule.yml

@@ -5,6 +5,12 @@ on:
       run-tags:
         type: string
         default: all
+      tests-path:
+        required: true
+        type: string
+      base-config-path:
+        default: ../../tests/
+        type: string
 
 jobs:
   molecule:
@@ -13,17 +19,18 @@ jobs:
       fail-fast: true
       matrix:
         scenario:
-          - centos7_roles
-          - rocky9_roles
+          - centos7
+          - rocky9
     env:
       MOLECULE_RUN_TAGS: ${{ inputs.run-tags }}
       PY_COLORS: 1
       ANSIBLE_FORCE_COLOR: 1
 
     steps:
       - name: Run `molecule test`
-        uses: UCL-MIRSG/.github/actions/molecule-test@v0.38.0
+        uses: UCL-MIRSG/.github/actions/molecule-test@v0.40.0
         with:
           scenario: ${{ matrix.scenario }}
+          base_config: ${{ format('{0}{1}_base_config.yml', inputs.base-config-path, matrix.scenario ) }}
           checkout_path: ansible_collections/mirsg/infrastructure
-          tests_path: ansible_collections/mirsg/infrastructure/tests
+          tests_path: ${{ inputs.tests-path }}

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 repos:
   - repo: https://github.com/UCL-MIRSG/.github
-    rev: v0.38.0
+    rev: v0.40.0
     hooks:
       - id: mirsg-hooks

README.md

@@ -76,48 +76,45 @@ python -m pip install molecule 'molecule-plugins[docker]' docker
 
 ### Testing the roles using Molecule
 
-Molecue 6.0 requires that the test configuration is not in the top-level
-directory of the collection. To support running the tests with Molecule 6, the
-Molecule configuration is in `ansible_collections/mirsg/infrastructure/tests`.
-To run the tests you must be in this directory:
+Each role has its own Molecule configuration, which can be found it the `molecule/`
+folder of each role. Molecule base configurations are used to reduce to amount
+of duplication in the setup for testing each role. There are two base configurations
+that correspond to two [Molecule
+scenarios](https://ansible.readthedocs.io/projects/molecule/getting-started/#molecule-scenarios)
+one for testing on CentOS 7 and another for testing on RockyLinux 9. The base configurations
+are in the `ansible_collections/mirsg/infrastructure/tests` folder.
+
+To run the tests for a specific role, first navigate the the role directory, e.g.
 
 ```shell
-cd ansible_collections/mirsg/infrastructure/tests
+cd ansible_collections/mirsg/infrastructure/roles/provision
 ```
 
-This collection contains [molecule
-scenarios](https://ansible.readthedocs.io/projects/molecule/getting-started/#molecule-scenarios)
-that allow testing of the individual roles. There are scenarios that run the
-tests on both CentOS 7 and Rocky 9.
-
-To run the CentOS 7 tests for the roles:
+Then invoke Molecule with a base config and corresponding scenario. To run the
+tests on CentOS 7:
 
 ```shell
-molecule test -s centos7_roles
+molecule --base-config ../../tests/centos7_base_config.yml test --scenario centos7
 ```
 
 This command will:
 
 - install the required Ansible roles and collections
 - create a CentOS 7 container
-- `tests/molecule/resources/shared/prepare.yml` playbook to do any required
-  setup for the roles
-- run the `tests/molecule/resources/roles/converge.yml` playbook, which will run
-  the roles in this collection
-- run `tests/molecule/resources/roles/converge.yml` a second time to check the
-  roles are
+- run the `molecule/prepare.yml` playbook (if it exists) to do any required
+  setup for the role
+- run the `molecule/converge.yml` playbook, which will run
+  the role being tested
+- run `molecule/converge.yml` a second time to check the
+  role is
   [idempotent](https://docs.ansible.com/ansible/latest/reference_appendices/glossary.html#term-Idempotency)
-- run `tests/molecule/resources/roles/verify.yml`
+- run `molecule/verify.yml` playbook (if it exists) to perform verification
 - destroy the CentOS 7 container
 
-### Test a single role in the collection
-
-Individual roles in the collection can be tested by setting the
-`MOLECULE_RUN_TAGS` environment variable:
+To run the tests on Rocky 9 instead:
 
 ```shell
-export MOLECULE_RUN_TAGS=provision
-molecule test -s centos7_roles
+molecule --base-config ../../tests/rocky9_base_config.yml test --scenario rocky9
 ```
 
 ### Inspecting the Container
@@ -130,7 +127,7 @@ If you would like to be able to access the test container, you should instead
 use the `molecule converge` command. To run this on CentOS 7:
 
 ```shell
-molecule converge -s centos7_roles
+molecule --base-config ../../tests/centos7_base_config.yml converge --scenario centos7
 ```
 
 This will install necessary Ansible roles and collections, create the test
@@ -140,17 +137,17 @@ container is not destroyed.
 #### Access the container
 
 Once the command has finished running, you can access the container using the
-name of the scenario. To access the container for the `centos7_roles` scenario:
+name of the scenario. To access the container for the `centos7` scenario:
 
 ```shell
-molecule login -s centos7_roles
+molecule --base-config ../../tests/centos7_base_config.yml login --scenario centos7
 ```
 
 If testing a role or playbook where Molecule creates multiple containers,
 individual hosts can be accessed using the `--host` flag:
 
 ```shell
-molecule login -s centos7_monitoring --host mserv
+molecule --base-config ../../tests/centos7_base_config.yml login --scenario centos7_monitoring --host mserv
 ```
 
 #### Destroy the container
@@ -160,17 +157,22 @@ container, network, and volumes yourself. You can do this using the `molecule
 destroy` command:
 
 ```shell
-molecule destroy -s centos7_roles
+molecule --base-config ../../tests/centos7_base_config.yml  destroy --scenario centos7
 ```
 
 ### Test a playbook
 
-Playbooks in the collection can also be tested using Molecule. An example of how
-this can be done can be seen by looking at the tests for the
-`mirsg.install_monitoring` playbook in this collection. This is tested on CentOS
-7 and RockyLinux 9 using the
+Playbooks in the collection can also be tested using Molecule. The Molecule
+configuration for playbooks is in the
+`ansible_collections/mirsg/infrastructure/tests` folder.
+
+An example of how to setup testing for a playbook can be seen by looking at
+the tests for the `mirsg.install_monitoring` playbook in this collection.
+This is tested on CentOS 7 and RockyLinux 9 using the
 [centos7_monitoring](./tests/molecule/centos7_monitoring/) and
-[rocky9_monitoring](./tests/molecule/rocky9_monitoring/) scenarios. An inventory
+[rocky9_monitoring](./tests/molecule/rocky9_monitoring/) scenarios.
+
+An inventory
 and associated group variables can be found in
 [resources/monitoring/inventory](./tests/molecule/resources/monitoring/inventory/).
 Testing the playbook also requires its own
@@ -179,8 +181,7 @@ it uses the shared [prepare.yml](./tests/molecule/resources/shared/prepare.yml)
 playbook. Running the tests then proceeds as with testing the roles:
 
 ```shell
-export MOLECULE_RUN_TAGS=monitoring
-molecule test -s centos7_monitoring
+molecule --base-config centos7_base_config.yml test --scenario centos7_monitoring
 ```
 
 ### Integration tests

roles/README.md

@@ -7,35 +7,20 @@ XNAT or OMERO.
 
 This collection is tested using Ansible Molecule.
 
-Roles are tested at the collection level rather than as standalone roles, i.e. there is a
-top-level `tests` folder for the collection rather than a `tests/` folder for each role. This
-is to avoid duplicating the Ansible Molecule configuration for each role (especially the
-scenario definitions).
-
-There is a [single converge playbook](../tests/molecule/resources/converge.yml) that by default
-will run all the roles in the playbook. If you would like to run `molecule converge` on a specific
-role instead, you can do this by setting the relevant Ansible tags.
-
-The tags can be set using the
-`MOLECULE_RUN_TAGS` environment variable, which is passed to Ansible Molecule in the
-[scenario definition](../tests/molecule/centos7/molecule.yml) (see the
-`config_options` for the `provisioner` section of the scenario). The accepted tags are defined in the
-[converge playbook](../tests/molecule/resources/converge.yml) for each role.
-
-As an example, to run
-`molecule converge` on the `provision` role only:
-
-```bash
-export MOLECULE_RUN_TAGS=provision
-molecule converge -s centos7
-```
+Each role has its own Molecule configuration, which can be found it the `molecule/`
+folder of each role. Molecule base configurations are used to reduce to amount
+of duplication in the setup for testing each role. There are two base configurations
+that correspond to two [Molecule
+scenarios](https://ansible.readthedocs.io/projects/molecule/getting-started/#molecule-scenarios)
+one for testing on CentOS 7 and another for testing on RockyLinux 9. The base configurations
+are in the `ansible_collections/mirsg/infrastructure/tests` folder.
 
 ## Adding a new role
 
 To add a new role to this collection, you will need to:
 
 1. add the role to the `roles/` folder
-2. add a molecule configuration for the role to the `tests/` folder
+2. add a molecule configuration for the role to the `roles/<my_role>/molecule/` folder
 3. add a workflow for the role to the `.github/workflows` folder
 
 ### Add a new role to the roles folder
@@ -49,8 +34,8 @@ ansible-galaxy init my_role
 Note, roles within a collection cannot contain hyphens in their names - please use underscores
 instead.
 
-You can delete the `meta` and `tests` folders as these are handled at the collection level
-rather than for individual roles:
+You can delete the `meta` and `tests` folders as these metadata is stored at the collection level
+and the molecule configuration should be stored in a `molecule/` folder:
 
 ```bash
 rm -r my_role/meta my_role/tests
@@ -60,32 +45,25 @@ You can then add your tasks, defaults, etc. to this role as usual.
 
 ### Add a molecule configuration
 
-We will want to be able to run `molecule converge` on the new role. To do this, add
-the role to the list of roles in the [converge playbook](../tests/molecule/resources/converge.yml):
-
-```yaml
-- role: mirsg.infrastructure.my_role
-  tags: my_role
-```
-
-We have added a tag `my_role` so that we can optionally tell molecule to only include this role
-in the tests (see [above](#molecule-setup) for more info). To run molecule only on this new role,
-you will need to set the `MOLECULE_RUN_TAGS` environment variable:
+Create a `molecule` folder in your role:
 
 ```bash
-export MOLECULE_RUN_TAGS=my_role
-molecule converge -s centos7
+mkdir my_roles/molecule
 ```
 
-If necessary, add [inventory `group_vars`](../tests/molecule/resources/inventory/group_vars)
+Add the `converge.yml` playbook to the `molecule/` folder,
+and create folders for the `centos7` and `rocky9` scenarios. See
+[`mirsg.infrastructure.firewalld` for an example](./firewalld/).
+
+If necessary, add inventory `group_vars` under a `resources` subfolder: `my-role/molecule/resources/inventory/group_vars`.
 for your role.
 
-You may also need to add [`prepare`](../tests/molecule/resources/prepare.yml) or
-[`verify`](../tests/molecule/resources/verify.yml) steps for your role.
+You may also need to add `prepare` or
+`verify` playbooks for your role. These can be added to the `resources` subfolder if you have multiple scenarios that will share the playbooks, or in the default scenario folder.
 
 ### Add a GitHub Workflow
 
-We have a GitHub workflow for running molecule on each role individually, and doing so
+We have a GitHub workflow for running molecule on each role, and doing so
 only when that role changes.
 
 You should add a GitHub workflow for your new role to `.github/workflows/molecule-my-role.yml`.
@@ -101,12 +79,11 @@ on:
       - ".github/workflows/molecule-my-role.yml"
 
 jobs:
-  molecule-firewalld:
+  molecule-my_role:
     uses: ./.github/workflows/molecule.yml
     with:
-      run-tags: my_role
+      tests-path: ansible_collections/mirsg/infrastructure/roles/my_role
 ```
 
 This uses the [`.github/workflows/molecule.yml` reusable workflow](.github/workflows/molecule.yml) to run
-molecule on the collection. Specifying `run-tags: my_role` will set `MOLECULE_RUN_TAGS` to `my_role` in the
-reusable workflow, and so `mirsg.infrastructure.my_role` will be the only role to run during the tests.
+molecule on the relevant role.

roles/firewalld/molecule/centos7/molecule.yml

@@ -0,0 +1,3 @@
+---
+# test this scenario from the roles/firewalld directory with the command
+# molecule --base-config ../../tests/centos7_base_config.yml test --scenario centos7

roles/firewalld/molecule/resources/converge.yml

@@ -0,0 +1,7 @@
+---
+- name: Setup firewall
+  hosts: all
+  become: true
+  gather_facts: true
+  roles:
+    - role: mirsg.infrastructure.firewalld

roles/firewalld/molecule/resources/inventory/group_vars/all.yml

@@ -0,0 +1,17 @@
+---
+# mirsg.infrastructure.firewalld
+firewalld_allow_public_access: true
+firewalld_internal_zone_open_services:
+  - http
+  - https
+  - ssh
+firewalld_public_zone_open_services:
+  - http
+  - https
+firewalld_work_zone_open_services:
+  - http
+  - https
+firewalld_public_zone_ports:
+  - "8080"
+firewalld_internal_zone_ports:
+  - "5432"

roles/firewalld/molecule/resources/prepare.yml

@@ -0,0 +1,15 @@
+---
+- name: Setup for firewalld role
+  hosts: all
+  gather_facts: true
+  tasks:
+    - name: Install firewalld
+      ansible.builtin.package:
+        name: firewalld
+        state: present
+
+    - name: Change firewalld backend to iptables
+      ansible.builtin.lineinfile:
+        path: /etc/firewalld/firewalld.conf
+        regexp: "^FirewallBackend="
+        line: FirewallBackend=iptables