diff --git a/.github/workflows/molecule-monitoring.yml b/.github/workflows/molecule-monitoring.yml new file mode 100644 index 00000000..d9f661a5 --- /dev/null +++ b/.github/workflows/molecule-monitoring.yml @@ -0,0 +1,53 @@ +name: Test install_monitoring playbook +on: + pull_request: + paths: + - "roles/monitoring_client/**" + - "roles/monitoring_server/**" + - ".github/workflows/molecule.yml" + - ".github/workflows/molecule-monitoring.yml" + +jobs: + molecule-monitoring: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + scenario: + - centos7_monitoring + - rocky9_monitoring + env: + MOLECULE_RUN_TAGS: monitoring + PY_COLORS: 1 + ANSIBLE_FORCE_COLOR: 1 + + steps: + - name: Check out the codebase + uses: actions/checkout@v4 + with: + path: ansible_collections/mirsg/infrastructure + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install test dependencies + shell: bash + run: | + sudo apt-get update && sudo apt-get -y install rsync + python3 -m pip install --upgrade pip + python3 -m pip install \ + ansible \ + molecule \ + molecule-plugins[docker] \ + docker \ + passlib \ + bcrypt==4.0.1 \ + requests + + - name: Test with molecule + shell: bash + run: | + cd ansible_collections/mirsg/infrastructure/tests + molecule test --scenario-name ${{ matrix.scenario }} diff --git a/README.md b/README.md index 3a0d3b17..9288334d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # mirsg.infrastructure Ansible Collection -This repository contains the `mirsg.infrastructure` Ansible Collection. This collection can be used to -configure infrastructure for deploying XNAT and OMERO. +This repository contains the `mirsg.infrastructure` Ansible Collection. This +collection can be used to configure infrastructure for deploying XNAT and OMERO. ## Included content @@ -13,16 +13,17 @@ configure infrastructure for deploying XNAT and OMERO. ## External requirements -Before using this collection and its playbooks, you must install the -[necessary Ansible collections and roles](meta/requirements.yml). +Before using this collection and its playbooks, you must install the [necessary +Ansible collections and roles](meta/requirements.yml). ## Using this collection -You can install this collection using the `ansible-galaxy` command-line tool: +This collection can be installed using the `ansible-galaxy` command-line tool: - ansible-galaxy collection install https://github.com/UCL-MIRSG/ansible-collection-infra.git +ansible-galaxy collection install +https://github.com/UCL-MIRSG/ansible-collection-infra.git -You can also include it in a `requirements.yml` file and install it via +It can also be included in a `requirements.yml` file and install it via `ansible-galaxy collection install -r requirements.yml` using the format: ```yaml @@ -35,54 +36,60 @@ collections: ## Testing this collection -We use [Ansible Molecule](https://ansible.readthedocs.io/projects/molecule/) and its -[Docker plugin](https://github.com/ansible-community/molecule-plugins) to test the roles -and playbooks in this collection. +[Ansible Molecule](https://ansible.readthedocs.io/projects/molecule/) and its +[Docker plugin](https://github.com/ansible-community/molecule-plugins) is used +to test the roles and playbooks in this collection. -If you would like to run the tests locally you will need to: +To run the tests locally you will need to: - clone this repository - install Ansible Molecule and other test requirements -- install and start the [Docker Engine](https://docs.docker.com/engine/install/) if you have not done so already -- if necessary, add your user to the docker group in order to access the Docker Engine +- install and start the [Docker Engine](https://docs.docker.com/engine/install/) + or [Docker Desktop](https://www.docker.com/products/docker-desktop/) on MacOS, + if you have not done so already +- if necessary (i.e. you are running Docker on Linux), add your user to the + docker group in order to access the Docker Engine - run the tests using Molecule ### Clone this repository To test a collection, Molecule requires that it is in the path -`ansible_collections//`. This means when you clone this repository you -must ensure it is in the path `ansible_collections/mirsg/infrastructure`. The simplest way to do this is: +`ansible_collections//`. This means when you clone +this repository you must ensure it is in the path +`ansible_collections/mirsg/infrastructure`. The simplest way to do this is: -``` +```shell git clone git@github.com:UCL-MIRSG/ansible-collection-infra.git ansible_collections/mirsg/infrastructure ``` ### Install Ansible Molecule -Before running the tests you'll need to install Molecule, the Docker plugin, and the Python Docker -Engine API using `pip`: +Before running the tests install Molecule, the Docker plugin, and the Python +Docker Engine API using `pip`: -``` +```shell python -m pip install molecule 'molecule-plugins[docker]' docker ``` -### Run the tests using Molecule +### Testing the roles using Molecule -Molecue 6.0 requires that the test configuration is not in the top-level directory of the -collection. To support running the tests with Molecule 6, the Molecule configuration is in -`ansible_collections/mirsg/infrastructure/tests`. To run the tests you must be in this directory: +Molecue 6.0 requires that the test configuration is not in the top-level +directory of the collection. To support running the tests with Molecule 6, the +Molecule configuration is in `ansible_collections/mirsg/infrastructure/tests`. +To run the tests you must be in this directory: -``` +```shell cd ansible_collections/mirsg/infrastructure/tests ``` -This collection is tested using a -[Molecule scenario](https://ansible.readthedocs.io/projects/molecule/getting-started/#molecule-scenarios) - -that runs on CentOS 7 and Rocky 9. The available roles are in the molecule subdirectory. +This collection contains [molecule +scenarios](https://ansible.readthedocs.io/projects/molecule/getting-started/#molecule-scenarios) +that allow testing of the individual roles. There are scenarios that run the +tests on both CentOS 7 and Rocky 9. -To run the tests for this CentOS 7 sceneraio: +To run the CentOS 7 tests for the roles: -``` +```shell molecule test -s centos7_roles ``` @@ -90,59 +97,102 @@ This command will: - install the required Ansible roles and collections - create a CentOS 7 container -- `tests/molecule/resources/prepare.yml` playbook to do any required setup for the roles -- run the `tests/molecule/resources/converge.yml` playbook, which will run the roles in this collection -- run `tests/molecule/resources/converge.yml` a second time to check the roles are +- `tests/molecule/resources/shared/prepare.yml` playbook to do any required + setup for the roles +- run the `tests/molecule/resources/roles/converge.yml` playbook, which will run + the roles in this collection +- run `tests/molecule/resources/roles/converge.yml` a second time to check the + roles are [idempotent](https://docs.ansible.com/ansible/latest/reference_appendices/glossary.html#term-Idempotency) -- run `tests/molecule/resources/verify.yml` +- run `tests/molecule/resources/roles/verify.yml` - destroy the CentOS 7 container +### Test a single role in the collection + +Individual roles in the collection can be tested by setting the +`MOLECULE_RUN_TAGS` environment variable: + +```shell +export MOLECULE_RUN_TAGS=provision +molecule test -s centos7_roles +``` + ### Inspecting the Container -If the `molecule test` command fails at any stage, the container is immediately destroyed. -This is due to the pre-defined sequence of actions the Molecule take when the `molecule test` -command is invoked. +If the `molecule test` command fails at any stage, the container is immediately +destroyed. This is due to the pre-defined sequence of actions the Molecule take +when the `molecule test` command is invoked. If you would like to be able to access the test container, you should instead use the `molecule converge` command. To run this on CentOS 7: -``` +```shell molecule converge -s centos7_roles ``` -This will install necessary Ansible roles and collections, create the test container, and run the -molecule playbooks. If the deployment fails, the container is not destroyed. +This will install necessary Ansible roles and collections, create the test +container, and run the molecule playbooks. If the deployment fails, the +container is not destroyed. #### Access the container -Once the command has finished running, you can access the container using the name -of the scenario. To access the container for the `centos7` scenario: +Once the command has finished running, you can access the container using the +name of the scenario. To access the container for the `centos7_roles` scenario: -``` +```shell molecule login -s centos7_roles ``` +If testing a role or playbook where Molecule creates multiple containers, +individual hosts can be accessed using the `--host` flag: + +```shell +molecule login -s centos7_monitoring --host mserv +``` + #### Destroy the container -If you use the `molecule converge` command, you must remember to destroy the container, network, -and volumes yourself. You can do this using the `molecule destroy` command: +If you use the `molecule converge` command, you must remember to destroy the +container, network, and volumes yourself. You can do this using the `molecule +destroy` command: -``` +```shell molecule destroy -s centos7_roles ``` +### Test a playbook + +Playbooks in the collection can also be tested using Molecule. An example of how +this can be done can be seen by looking at the tests for the +`mirsg.install_monitoring` playbook in this collection. This is tested on CentOS +7 and RockyLinux 9 using the +[centos7_monitoring](./tests/molecule/centos7_monitoring/) and +[rocky9_monitoring](./tests/molecule/rocky9_monitoring/) scenarios. An inventory +and associated group variables can be found in +[resources/monitoring/inventory](./tests/molecule/resources/monitoring/inventory/). +Testing the playbook also requires its own +[converge.yml](./tests/molecule/resources/monitoring/converge.yml) playbook but +it uses the shared [prepare.yml](./tests/molecule/resources/shared/prepare.yml) +playbook. Running the tests then proceeds as with testing the roles: + +```shell +export MOLECULE_RUN_TAGS=monitoring +molecule test -s centos7_monitoring +``` + ### Integration tests When a PR that modifies a role is opened, the changes are -[tested](.github/workflows/) by deploying that role using GitHub Actions. See the -[`molecule-firewalld` workflow](.github/workflows/molecule-firewalld.yml) +[tested](.github/workflows/) by deploying that role using GitHub Actions. See +the [`molecule-firewalld` workflow](.github/workflows/molecule-firewalld.yml) for an example. ## Code style and formatting [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) -This repo has `pre-commit` hooks enabled, for instructions see . +This repo has `pre-commit` hooks enabled, for instructions see +. ## License diff --git a/playbooks/install_monitoring.yml b/playbooks/install_monitoring.yml new file mode 100644 index 00000000..49357ecd --- /dev/null +++ b/playbooks/install_monitoring.yml @@ -0,0 +1,40 @@ +--- +- name: Set up monitoring server + hosts: monitoring_host + become: true + gather_facts: true + pre_tasks: + - name: Generate list of docker clients from `monitoring_client` group + ansible.builtin.set_fact: + docker_client_hostnames: > + {{ + query('inventory_hostnames', ansible_limit | default('')) | + intersect(groups['monitoring_client']) | + map('extract', hostvars, monitoring_server_hostname_extractor) | + list | default([]) + }} + failed_when: docker_client_hostnames | length == 0 + + roles: + - role: mirsg.infrastructure.provision + - role: mirsg.infrastructure.install_python + - role: mirsg.infrastructure.docker + - role: mirsg.infrastructure.monitoring_server + +- name: Set up monitoring clients + hosts: monitoring_client + become: true + gather_facts: true + + roles: + - role: mirsg.infrastructure.monitoring_client + - role: mirsg.infrastructure.firewalld + vars: + internal_zone_sources: + - "{{ monitoring_client_monitoring_server_ip }}" + rich_rules: + - zone: "internal" + rule: "family=ipv4 source \ + address={{ monitoring_client_monitoring_server_ip }}/32 \ + port protocol=tcp \ + port={{ monitoring_client_node_exporter_port }} accept" diff --git a/roles/create_monitoring_client_csr/tasks/main.yml b/roles/create_monitoring_client_csr/tasks/main.yml deleted file mode 100644 index ea47f43f..00000000 --- a/roles/create_monitoring_client_csr/tasks/main.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- name: Ensure monitoring cert dir exists on client - ansible.builtin.file: - path: "{{ monitoring_client.cert_dir }}" - state: directory - owner: "{{ monitoring_client.owner }}" - group: "{{ monitoring_client.group }}" - mode: "0700" - -- name: Generate OpenSSL private key - community.crypto.openssl_privatekey: - path: "{{ monitoring_client.ssl_key_file }}" - owner: "{{ monitoring_client.owner }}" - group: "{{ monitoring_client.group }}" - mode: "0400" - register: new_monitoring_client_private_key_generated - -- name: Generate OpenSSL CSR - community.crypto.openssl_csr: - path: "{{ monitoring_client.ssl_csr_file }}" - privatekey_path: "{{ monitoring_client.ssl_key_file }}" - common_name: "{{ ansible_host }}" - register: new_monitoring_client_csr_generated - -- name: Copy client CSR - ansible.builtin.fetch: - src: "{{ monitoring_client.ssl_csr_file }}" - dest: "{{ monitoring_temp_files_cert_dir }}/{{ ansible_host }}.csr" - flat: true diff --git a/roles/monitoring_client/defaults/main.yml b/roles/monitoring_client/defaults/main.yml new file mode 100644 index 00000000..18032370 --- /dev/null +++ b/roles/monitoring_client/defaults/main.yml @@ -0,0 +1,24 @@ +--- +monitoring_client_node_exporter_version: 1.7.0 + +monitoring_client_node_exporter_binary: + "https://github.com/prometheus/node_exporter/releases/download/v\ + {{ monitoring_client_node_exporter_version }}/node_exporter-\ + {{ monitoring_client_node_exporter_version }}.linux-amd64.tar.gz" +monitoring_client_node_exporter_install_dir: /usr/bin/node_exporter +monitoring_client_node_exporter_service: /etc/systemd/system/node_exporter.service +monitoring_client_node_exporter_web_config: /usr/bin/node_exporter/web.yml +monitoring_client_node_exporter_port: 9100 +monitoring_client_node_exporter_ssl_key: /usr/bin/node_exporter/node_exporter.key +monitoring_client_node_exporter_ssl_cert: /usr/bin/node_exporter/node_exporter.crt + +monitoring_client_owner: root +monitoring_client_group: root +monitoring_client_cert_dir: /root/monitoring_certs +monitoring_client_ssl_key_file: /root/monitoring_certs/key.pem +monitoring_client_ssl_csr_file: /root/monitoring_certs/monitoring.csr +monitoring_client_ssl_pk8_file: /root/monitoring_certs/monitoring.pk8 +monitoring_client_ssl_cert_file: /root/monitoring_certs/cert.pem +monitoring_client_server_ca_cert_file: /root/monitoring_certs/ca.pem +monitoring_client_exporter_username: prometheus +monitoring_client_exporter_password: "" diff --git a/roles/monitoring_client/tasks/install_node_exporter.yml b/roles/monitoring_client/tasks/install_node_exporter.yml index 952a79fc..59b9c0e7 100644 --- a/roles/monitoring_client/tasks/install_node_exporter.yml +++ b/roles/monitoring_client/tasks/install_node_exporter.yml @@ -1,40 +1,40 @@ --- - name: Create Node exporter folder ansible.builtin.file: - owner: "{{ monitoring_client.owner }}" - group: "{{ monitoring_client.group }}" - path: "{{ node_exporter.install_dir }}" + owner: "{{ monitoring_client_owner }}" + group: "{{ monitoring_client_group }}" + path: "{{ monitoring_client_node_exporter_install_dir }}" state: directory mode: "0777" - name: Download and unarchive node_exporter ansible.builtin.unarchive: - src: "{{ node_exporter.binary_url }}" - dest: "{{ node_exporter.install_dir }}" + src: "{{ monitoring_client_node_exporter_binary }}" + dest: "{{ monitoring_client_node_exporter_install_dir }}" remote_src: true - owner: "{{ monitoring_client.owner }}" - group: "{{ monitoring_client.group }}" + owner: "{{ monitoring_client_owner }}" + group: "{{ monitoring_client_group }}" extra_opts: "--strip-components=1" - creates: "{{ node_exporter.install_dir }}/node_exporter" + creates: "{{ monitoring_client_node_exporter_install_dir }}/node_exporter" - name: Copy monitoring client crt and key files to node_exporter folder ansible.builtin.copy: src: "{{ item.src }}" dest: "{{ item.dest }}" - owner: "{{ monitoring_client.owner }}" - group: "{{ monitoring_client.group }}" + owner: "{{ monitoring_client_owner }}" + group: "{{ monitoring_client_group }}" mode: "0600" remote_src: true with_items: - - src: "{{ monitoring_client.ssl_key_file }}" - dest: "{{ node_exporter.ssl_key_file }}" - - src: "{{ monitoring_client.ssl_cert_file }}" - dest: "{{ node_exporter.ssl_cert_file }}" + - src: "{{ monitoring_client_ssl_key_file }}" + dest: "{{ monitoring_client_node_exporter_ssl_key }}" + - src: "{{ monitoring_client_ssl_cert_file }}" + dest: "{{ monitoring_client_node_exporter_ssl_cert }}" -- name: Copy node_exporter service file +- name: Copy monitoring_client_node_exporter service file ansible.builtin.template: src: templates/node_exporter.service.j2 - dest: "{{ node_exporter.service_config_file }}" + dest: "{{ monitoring_client_node_exporter_service }}" mode: "0644" when: ansible_service_mgr == "systemd" @@ -45,47 +45,15 @@ state: started enabled: true when: ansible_service_mgr == "systemd" - -- name: Check 'passlib' is installed on the Ansible controller - delegate_to: localhost - run_once: true - changed_when: false - ansible.builtin.shell: | - set -o pipefail - {{ ansible_playbook_python }} -m pip freeze | grep passlib - register: passlib_exists - # grep returns error code 2 for exceptions, otherwise don't want failure - # https://stackoverflow.com/a/41010653/7359333 - failed_when: "passlib_exists.rc == 2" - -- name: Fail if 'passlib' is missing on the Ansible controller - run_once: true - ansible.builtin.fail: - msg: > - 'bcrypt' algorithm requires 'passlib' on the Ansible controller, - run `python -m pip install passlib` and re-run the script - when: "passlib_exists.rc == 1" + tags: + - molecule-idempotence-notest - name: Copy node_exporter web config file ansible.builtin.template: src: templates/node_exporter_web.yml.j2 - dest: "{{ node_exporter.web_config_file }}" + dest: "{{ monitoring_client_node_exporter_web_config }}" mode: "0644" notify: - Restart node_exporter - -- name: Add monitoring server to internal zone - ansible.posix.firewalld: - zone: internal - source: "{{ monitoring_service.server.ip }}" - permanent: true - immediate: true - state: enabled - -- name: Open internal node_exporter port to monitoring server - ansible.posix.firewalld: - rich_rule: "rule family=ipv4 source address={{ monitoring_service.server.ip }}/32 port protocol=tcp port={{ node_exporter.port }} accept" - zone: internal - permanent: true - immediate: true - state: enabled + tags: + - molecule-idempotence-notest diff --git a/roles/monitoring_client/tasks/main.yml b/roles/monitoring_client/tasks/main.yml index 951e283f..90882641 100644 --- a/roles/monitoring_client/tasks/main.yml +++ b/roles/monitoring_client/tasks/main.yml @@ -1,26 +1,34 @@ --- - name: Ensure monitoring cert dir exists on client ansible.builtin.file: - path: "{{ monitoring_client.cert_dir }}" + path: "{{ monitoring_client_cert_dir }}" state: directory - owner: "{{ monitoring_client.owner }}" - group: "{{ monitoring_client.group }}" + owner: "{{ monitoring_client_owner }}" + group: "{{ monitoring_client_group }}" mode: "0700" - name: Copy monitoring server certificate to client ansible.builtin.copy: - src: "{{ monitoring_temp_files_cert_dir }}/ca.pem" - dest: "{{ monitoring_client.server_ca_cert_file }}" - owner: "{{ monitoring_client.owner }}" - group: "{{ monitoring_client.group }}" + src: "{{ monitoring_client_certificate_cache_directory }}/ca.pem" + dest: "{{ monitoring_client_server_ca_cert_file }}" + owner: "{{ monitoring_client_owner }}" + group: "{{ monitoring_client_group }}" mode: "0600" - name: Copy signed monitoring client certificate to client ansible.builtin.copy: - src: "{{ monitoring_temp_files_cert_dir }}/{{ ansible_host }}.cert" - dest: "{{ monitoring_client.ssl_cert_file }}" - owner: "{{ monitoring_client.owner }}" - group: "{{ monitoring_client.group }}" + src: "{{ monitoring_client_certificate_cache_directory }}/{{ monitoring_client_certificate_file }}" + dest: "{{ monitoring_client_ssl_cert_file }}" + owner: "{{ monitoring_client_owner }}" + group: "{{ monitoring_client_group }}" + mode: "0600" + +- name: Copy signed monitoring client key to client + ansible.builtin.copy: + src: "{{ monitoring_client_certificate_cache_directory }}/key.pem" + dest: "{{ monitoring_client_ssl_key_file }}" + owner: "{{ monitoring_client_owner }}" + group: "{{ monitoring_client_group }}" mode: "0600" - name: Install and configure Node exporter diff --git a/roles/monitoring_client/templates/node_exporter.service.j2 b/roles/monitoring_client/templates/node_exporter.service.j2 index bce3924e..0ae57fe1 100644 --- a/roles/monitoring_client/templates/node_exporter.service.j2 +++ b/roles/monitoring_client/templates/node_exporter.service.j2 @@ -4,10 +4,10 @@ Documentation=https://prometheus.io/ After=syslog.target network.target [Service] -User= {{ monitoring_client.owner }} -Group= {{ monitoring_client.group }} +User= {{ monitoring_client_owner }} +Group= {{ monitoring_client_group }} -ExecStart={{ node_exporter.install_dir }}/node_exporter --web.config={{ node_exporter.web_config_file }} +ExecStart={{ monitoring_client_node_exporter_install_dir }}/node_exporter --web.config={{ monitoring_client_node_exporter_web_config }} [Install] WantedBy=multi-user.target diff --git a/roles/monitoring_client/templates/node_exporter_web.yml.j2 b/roles/monitoring_client/templates/node_exporter_web.yml.j2 index 10ca62f4..757a0382 100644 --- a/roles/monitoring_client/templates/node_exporter_web.yml.j2 +++ b/roles/monitoring_client/templates/node_exporter_web.yml.j2 @@ -2,4 +2,4 @@ tls_server_config: cert_file: node_exporter.crt key_file: node_exporter.key basic_auth_users: - {{ monitoring_client.exporter_auth.username }}: {{ monitoring_client.exporter_auth.password | password_hash('bcrypt') }} + {{ monitoring_client_exporter_username }}: {{ monitoring_client_exporter_password | password_hash('bcrypt') }} diff --git a/roles/monitoring_server/defaults/main.yml b/roles/monitoring_server/defaults/main.yml new file mode 100644 index 00000000..069f26a7 --- /dev/null +++ b/roles/monitoring_server/defaults/main.yml @@ -0,0 +1,87 @@ +monitoring_server_owner: monitoring +monitoring_server_group: monitoring +monitoring_server_gid: 595 +monitoring_server_uid: 595 +monitoring_server_cert_owner: root +monitoring_server_cert_group: root +monitoring_server_storage_root: /data/monitoring + +# mirsg.monitoring_server CA and server certificate +monitoring_server_certificate_cache_directory: "{{ lookup('env', 'HOME') }}/ansible_persistent_files/monitoring_server_certificates" +monitoring_server_certificate_directory: /root/monitoring_certs +monitoring_server_ca_key: "{{ monitoring_server_certificate_directory }}/ca.key" +monitoring_server_ca_csr: "{{ monitoring_server_certificate_directory }}/ca.csr" +monitoring_server_ca_cert: "{{ monitoring_server_certificate_directory }}/ca.pem" +monitoring_server_server_key: "{{ monitoring_server_certificate_directory }}/server-key.pem" +monitoring_server_server_csr: "{{ monitoring_server_certificate_directory }}/server.csr" +monitoring_server_server_cert: "{{ monitoring_server_certificate_directory }}/server-cert.pem" + +# mirsg.monitoring_server client related +monitoring_server_client_certificate_directory: "{{ monitoring_server_certificate_directory }}/client_certs" +monitoring_server_client_group: "monitoring_client" + +monitoring_server_smtp_enabled: false +monitoring_server_smtp_hostname: "" +monitoring_server_smtp_port: "25" +monitoring_server_smtp_protocol: "smtp" +monitoring_server_smtp_auth: "" +monitoring_server_smtp_username: "" +monitoring_server_smtp_password: "" +monitoring_server_smtp_start_tls: "false" +ssl_trust: "*" + +monitoring_server_hostname_extractor: "ansible_host" + +monitoring_server_alertmanager: + container_name: alertmanager + image: prom/alertmanager + external_data_dir: "{{ monitoring_server_storage_root }}/alertmanager" + volume: /alertmanager + commandline_args: + web.external-url: "https://{{ ansible_host }}/alertmanager/" + storage.path: /alertmanager/data + config.file: /alertmanager/alertmanager.yml + +monitoring_server_cadvisor: + container_name: cadvisor + image: gcr.io/cadvisor/cadvisor:latest + +monitoring_server_grafana: + container_name: grafana + image: grafana/grafana + external_data_dir: "{{ monitoring_server_storage_root }}/grafana" + external_storage_dir: "{{ monitoring_server_storage_root }}/grafana/storage" + volume: /var/lib/grafana + external_datasource: "{{ monitoring_server_storage_root }}/grafana/datasources.yml" + volume_datasource: /etc/grafana/provisioning/datasources/datasources.yml + environment_variables: + GF_SECURITY_ADMIN_USER: "{{ monitoring_server_grafana_username }}" + GF_SECURITY_ADMIN_PASSWORD: "{{ monitoring_server_grafana_password }}" + GF_SERVER_DOMAIN: "{{ monitoring_server_grafana_host }}" + GF_SERVER_ROOT_URL: "%(protocol)s://%(domain)s:%(http_port)s/grafana/" + GF_SERVER_SERVE_FROM_SUB_PATH: "true" + +monitoring_server_nginx: + owner: root + group: root + log_folder: /var/log/nginx + http_port: 80 + https_port: 443 + certs_dir: /etc/nginx/ssl + diffie_helman_size_bits: 2048 + dh_params_file: /etc/nginx/ssl/dhparam.pem + conf_file: /etc/nginx/nginx.conf + ssl_cert_file: /etc/nginx/ssl/server.cert + ssl_key_file: /etc/nginx/ssl/server.key + +monitoring_server_prometheus: + image: prom/prometheus + container_name: prometheus + external_data_dir: "{{ monitoring_server_storage_root }}/prometheus" + volume: /prometheus + ca_cert: /prometheus/ca.pem + rules: /prometheus/rules.yml + commandline_args: + config.file: /prometheus/prometheus.yml + storage.tsdb.path: /prometheus/data + web.external-url: /prometheus/ diff --git a/roles/monitoring_server/handlers/main.yml b/roles/monitoring_server/handlers/main.yml index d99ac0e1..c0c0b745 100644 --- a/roles/monitoring_server/handlers/main.yml +++ b/roles/monitoring_server/handlers/main.yml @@ -1,12 +1,12 @@ - name: Restart prometheus community.docker.docker_container: - name: "{{ prometheus.container_name }}" + name: "{{ monitoring_server_prometheus.container_name }}" state: started restart: true - name: Restart grafana community.docker.docker_container: - name: "{{ grafana.container_name }}" + name: "{{ monitoring_server_grafana.container_name }}" state: started restart: true @@ -18,5 +18,5 @@ - name: Restart docker ansible.builtin.service: - name: "{{ docker.service_name }}" + name: "{{ docker_service_name }}" state: restarted diff --git a/roles/monitoring_server/tasks/install_alertmanager_container.yml b/roles/monitoring_server/tasks/install_alertmanager_container.yml index 160fa14f..1ef3d09f 100644 --- a/roles/monitoring_server/tasks/install_alertmanager_container.yml +++ b/roles/monitoring_server/tasks/install_alertmanager_container.yml @@ -1,31 +1,35 @@ --- -- name: Ensure alertmanager data directory exists - {{ alertmanager.external_data_dir }} +- name: Ensure alertmanager data directory exists - {{ monitoring_server_alertmanager.external_data_dir }} ansible.builtin.file: - path: "{{ alertmanager.external_data_dir }}" - owner: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" + path: "{{ monitoring_server_alertmanager.external_data_dir }}" + owner: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" state: directory mode: "0700" - name: Copy alertmanager config file ansible.builtin.template: src: templates/alertmanager.yml.j2 - dest: "{{ alertmanager.external_data_dir }}/alertmanager.yml" + dest: "{{ monitoring_server_alertmanager.external_data_dir }}/alertmanager.yml" owner: root mode: "0644" - name: Start alertmanager container community.docker.docker_container: - name: "{{ alertmanager.container_name }}" - hostname: "{{ alertmanager.container_name }}" - image: "{{ alertmanager.image }}" + name: "{{ monitoring_server_alertmanager.container_name }}" + hostname: "{{ monitoring_server_alertmanager.container_name }}" + image: "{{ monitoring_server_alertmanager.image }}" state: started - user: "{{ monitoring_server.uid }}:{{ monitoring_server.gid }}" - command: "{% for key in alertmanager.commandline_args %}--{{ key }}={{ alertmanager.commandline_args[key] }} {% endfor %}" + user: "{{ monitoring_server_uid }}:{{ monitoring_server_gid }}" + command: > + "{% for key in + monitoring_server_alertmanager.commandline_args + %}--{{ key }}={{ monitoring_server_alertmanager.commandline_args[key] }} + {% endfor %}" networks: - name: monitor-net volumes: - - "{{ alertmanager.external_data_dir }}:{{ alertmanager.volume }}" + - "{{ monitoring_server_alertmanager.external_data_dir }}:{{ monitoring_server_alertmanager.volume }}" restart_policy: always notify: - Restart prometheus diff --git a/roles/monitoring_server/tasks/install_blackbox_exporter_container.yml b/roles/monitoring_server/tasks/install_blackbox_exporter_container.yml index 80291547..b7510b3e 100644 --- a/roles/monitoring_server/tasks/install_blackbox_exporter_container.yml +++ b/roles/monitoring_server/tasks/install_blackbox_exporter_container.yml @@ -1,30 +1,30 @@ --- -- name: Ensure blackbox_exporter data directory exists - {{ blackbox_exporter.external_data_dir }} +- name: Ensure blackbox_exporter data directory exists ansible.builtin.file: - path: "{{ blackbox_exporter.external_data_dir }}" - owner: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" + path: "{{ monitoring_server_storage_root }}/blackbox-exporter" + owner: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" state: directory mode: "0700" - name: Copy blackbox_exporter config file ansible.builtin.template: src: templates/blackbox-exporter.yml.j2 - dest: "{{ blackbox_exporter.external_data_dir }}/blackbox-exporter.yml" - owner: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" + dest: "{{ monitoring_server_storage_root }}/blackbox-exporter/blackbox-exporter.yml" + owner: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" mode: "0644" - name: Start blackbox_exporter container community.docker.docker_container: - name: "{{ blackbox_exporter.container_name }}" - hostname: "{{ blackbox_exporter.container_name }}" - image: "{{ blackbox_exporter.image }}" + name: blackbox-exporter + hostname: blackbox-exporter + image: prom/blackbox-exporter state: started - user: "{{ monitoring_server.uid }}:{{ monitoring_server.gid }}" - command: "{% for key in blackbox_exporter.commandline_args %}--{{ key }}={{ blackbox_exporter.commandline_args[key] }} {% endfor %}" + user: "{{ monitoring_server_uid }}:{{ monitoring_server_gid }}" + command: --config.file=/config/blackbox-exporter.yml networks: - name: monitor-net volumes: - - "{{ blackbox_exporter.external_data_dir }}/blackbox-exporter.yml:{{ blackbox_exporter.volume }}/blackbox-exporter.yml" + - "{{ monitoring_server_storage_root }}/blackbox-exporter/blackbox-exporter.yml:/config/blackbox-exporter.yml" restart_policy: always diff --git a/roles/monitoring_server/tasks/install_cadvisor_container.yml b/roles/monitoring_server/tasks/install_cadvisor_container.yml index 250e306f..0c775e3e 100644 --- a/roles/monitoring_server/tasks/install_cadvisor_container.yml +++ b/roles/monitoring_server/tasks/install_cadvisor_container.yml @@ -1,11 +1,11 @@ --- - name: Start cadvisor container community.docker.docker_container: - name: "{{ cadvisor.container_name }}" - hostname: "{{ cadvisor.container_name }}" - image: "{{ cadvisor.image }}" + name: "{{ monitoring_server_cadvisor.container_name }}" + hostname: "{{ monitoring_server_cadvisor.container_name }}" + image: "{{ monitoring_server_cadvisor.image }}" state: started - user: "{{ monitoring_server.uid }}:{{ monitoring_server.gid }}" + user: "{{ monitoring_server_uid }}:{{ monitoring_server_gid }}" networks: - name: monitor-net command: diff --git a/roles/monitoring_server/tasks/install_grafana_container.yml b/roles/monitoring_server/tasks/install_grafana_container.yml index 035fd4d3..3e18bc85 100644 --- a/roles/monitoring_server/tasks/install_grafana_container.yml +++ b/roles/monitoring_server/tasks/install_grafana_container.yml @@ -1,9 +1,9 @@ --- -- name: Ensure grafana's directory exists - {{ grafana.external_data_dir }} +- name: Ensure grafana's directory exists - {{ monitoring_server_grafana.external_data_dir }} ansible.builtin.file: - path: "{{ grafana.external_data_dir }}" - owner: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" + path: "{{ monitoring_server_grafana.external_data_dir }}" + owner: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" mode: "0700" state: directory notify: @@ -11,33 +11,33 @@ - name: Ensure grafana's storage directory exists ansible.builtin.file: - path: "{{ grafana.external_storage_dir }}" + path: "{{ monitoring_server_grafana.external_storage_dir }}" state: directory - owner: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" + owner: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" mode: "0744" - name: Copy datasource file to grafana ansible.builtin.template: src: templates/datasources.yml.j2 - dest: "{{ grafana.external_datasource }}" - owner: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" + dest: "{{ monitoring_server_grafana.external_datasource }}" + owner: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" mode: "0644" notify: - Restart grafana - name: Start Grafana container community.docker.docker_container: - name: "{{ grafana.container_name }}" - hostname: "{{ grafana.container_name }}" - image: "{{ grafana.image }}" + name: "{{ monitoring_server_grafana.container_name }}" + hostname: "{{ monitoring_server_grafana.container_name }}" + image: "{{ monitoring_server_grafana.image }}" state: started - user: "{{ monitoring_server.uid }}:{{ monitoring_server.gid }}" + user: "{{ monitoring_server_uid }}:{{ monitoring_server_gid }}" volumes: - - "{{ grafana.external_storage_dir }}:{{ grafana.volume }}" - - "{{ grafana.external_datasource }}:{{ grafana.volume_datasource }}" + - "{{ monitoring_server_grafana.external_storage_dir }}:{{ monitoring_server_grafana.volume }}" + - "{{ monitoring_server_grafana.external_datasource }}:{{ monitoring_server_grafana.volume_datasource }}" networks: - name: monitor-net - env: "{{ grafana.environment_variables }}" + env: "{{ monitoring_server_grafana.environment_variables }}" restart_policy: always diff --git a/roles/monitoring_server/tasks/install_nginx_container.yml b/roles/monitoring_server/tasks/install_nginx_container.yml index b2b1897d..bebec310 100644 --- a/roles/monitoring_server/tasks/install_nginx_container.yml +++ b/roles/monitoring_server/tasks/install_nginx_container.yml @@ -1,24 +1,31 @@ --- -- name: Ensure python-passlib is installed +- name: Ensure passlib is installed on centos7 ansible.builtin.yum: - name: "python-passlib" - state: installed + name: python-passlib + state: present + when: ansible_facts['os_family'] == "RedHat" and ansible_facts['distribution_major_version'] is version("7") + +- name: Ensure passlib is installed on rocky9 + ansible.builtin.yum: + name: python3-passlib + state: present + when: ansible_facts['os_family'] == "RedHat" and ansible_facts['distribution_major_version'] is version("9") - name: Ensure nginx certs directory exists on host ansible.builtin.file: - path: "{{ nginx_monitoring_server.certs_dir }}" - owner: "{{ nginx_monitoring_server.owner }}" - group: "{{ nginx_monitoring_server.group }}" + path: "{{ monitoring_server_nginx.certs_dir }}" + owner: "{{ monitoring_server_nginx.owner }}" + group: "{{ monitoring_server_nginx.group }}" state: directory mode: "0700" - name: Add a user to a password file and ensure permissions are set community.general.htpasswd: path: /etc/nginx/.htpasswd - name: "{{ monitoring_server.admin.username }}" - password: "{{ monitoring_server.admin.password }}" - owner: "{{ nginx_monitoring_server.owner }}" - group: "{{ nginx_monitoring_server.group }}" + name: "{{ monitoring_server_admin_username }}" + password: "{{ monitoring_server_admin_password }}" + owner: "{{ monitoring_server_nginx.owner }}" + group: "{{ monitoring_server_nginx.group }}" mode: "0644" - name: Copy server certificates to nginx @@ -26,30 +33,30 @@ remote_src: true src: "{{ item.src }}" dest: "{{ item.dest }}" - owner: "{{ nginx_monitoring_server.owner }}" - group: "{{ nginx_monitoring_server.group }}" + owner: "{{ monitoring_server_nginx.owner }}" + group: "{{ monitoring_server_nginx.group }}" mode: "0644" with_items: - - src: "{{ monitoring_server.ssl.server_cert }}" - dest: "{{ nginx_monitoring_server.ssl_cert_file }}" - - src: "{{ monitoring_server.ssl.server_key }}" - dest: "{{ nginx_monitoring_server.ssl_key_file }}" + - src: "{{ monitoring_server_ssl_cert }}" + dest: "{{ monitoring_server_nginx.ssl_cert_file }}" + - src: "{{ monitoring_server_ssl_key }}" + dest: "{{ monitoring_server_nginx.ssl_key_file }}" notify: - Restart nginx -- name: Generate Diffie-Hellman (DH) parameters with bits - {{ diffie_helman_size_bits }} +- name: Generate Diffie-Hellman (DH) parameters with bits - {{ monitoring_server_nginx.diffie_helman_size_bits }} community.crypto.openssl_dhparam: - path: "{{ nginx_monitoring_server.dh_params_file }}" - size: "{{ diffie_helman_size_bits }}" + path: "{{ monitoring_server_nginx.dh_params_file }}" + size: "{{ monitoring_server_nginx.diffie_helman_size_bits }}" notify: - Restart nginx - name: Copy nginx config file ansible.builtin.template: - src: "nginx_monitoring_server.j2" - dest: "{{ nginx_monitoring_server.conf_file }}" - owner: "{{ nginx_monitoring_server.owner }}" - group: "{{ nginx_monitoring_server.group }}" + src: monitoring_server_nginx.j2 + dest: "{{ monitoring_server_nginx.conf_file }}" + owner: "{{ monitoring_server_nginx.owner }}" + group: "{{ monitoring_server_nginx.group }}" mode: "0644" force: true notify: @@ -67,8 +74,8 @@ - "80:80" - "443:443" volumes: - - "/etc/nginx/.htpasswd:/etc/nginx/.htpasswd" - - "{{ nginx_monitoring_server.conf_file }}:/etc/nginx/nginx.conf:ro" - - "{{ nginx_monitoring_server.certs_dir }}:/etc/nginx/ssl" - - "{{ nginx_monitoring_server.log_folder }}:/var/log/nginx" + - /etc/nginx/.htpasswd:/etc/nginx/.htpasswd + - "{{ monitoring_server_nginx.conf_file }}:/etc/nginx/nginx.conf:ro" + - "{{ monitoring_server_nginx.certs_dir }}:/etc/nginx/ssl" + - "{{ monitoring_server_nginx.log_folder }}:/var/log/nginx" restart_policy: always diff --git a/roles/monitoring_server/tasks/install_prometheus_container.yml b/roles/monitoring_server/tasks/install_prometheus_container.yml index 4dcb1d8b..de555702 100644 --- a/roles/monitoring_server/tasks/install_prometheus_container.yml +++ b/roles/monitoring_server/tasks/install_prometheus_container.yml @@ -1,18 +1,18 @@ --- -- name: Ensure prometheus data directory exists - {{ prometheus.external_data_dir }} +- name: Ensure prometheus data directory exists - {{ monitoring_server_prometheus.external_data_dir }} ansible.builtin.file: - path: "{{ prometheus.external_data_dir }}" - owner: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" + path: "{{ monitoring_server_prometheus.external_data_dir }}" + owner: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" state: directory mode: "0700" - name: Copy server cert to prometheus bind mount ansible.builtin.copy: - src: "{{ monitoring_temp_files_cert_dir }}/ca.pem" - dest: "{{ prometheus.external_data_dir }}/ca.pem" - owner: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" + src: "{{ monitoring_server_certificate_cache_directory }}/ca.pem" + dest: "{{ monitoring_server_prometheus.external_data_dir }}/ca.pem" + owner: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" mode: "0644" notify: - Restart prometheus @@ -20,9 +20,9 @@ - name: Copy prometheus rules file ansible.builtin.copy: src: templates/prometheus_rules.yml.j2 - dest: "{{ prometheus.external_data_dir }}/rules.yml" - owner: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" + dest: "{{ monitoring_server_prometheus.external_data_dir }}/rules.yml" + owner: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" mode: "0644" notify: - Restart prometheus @@ -30,22 +30,24 @@ - name: Copy prometheus config file ansible.builtin.template: src: templates/prometheus.yml.j2 - dest: "{{ prometheus.external_data_dir }}/prometheus.yml" + dest: "{{ monitoring_server_prometheus.external_data_dir }}/prometheus.yml" owner: root mode: "0644" notify: - Restart prometheus + tags: + - molecule-idempotence-notest - name: Start prometheus container community.docker.docker_container: - name: "{{ prometheus.container_name }}" - hostname: "{{ prometheus.container_name }}" - image: "{{ prometheus.image }}" - user: "{{ monitoring_server.uid }}:{{ monitoring_server.gid }}" + name: "{{ monitoring_server_prometheus.container_name }}" + hostname: "{{ monitoring_server_prometheus.container_name }}" + image: "{{ monitoring_server_prometheus.image }}" + user: "{{ monitoring_server_uid }}:{{ monitoring_server_gid }}" state: started - command: "{% for key in prometheus.commandline_args %}--{{ key }}={{ prometheus.commandline_args[key] }} {% endfor %}" + command: "{% for key in monitoring_server_prometheus.commandline_args %}--{{ key }}={{ monitoring_server_prometheus.commandline_args[key] }} {% endfor %}" networks: - name: monitor-net volumes: - - "{{ prometheus.external_data_dir }}:{{ prometheus.volume }}" + - "{{ monitoring_server_prometheus.external_data_dir }}:{{ monitoring_server_prometheus.volume }}" restart_policy: always diff --git a/roles/monitoring_server/tasks/main.yml b/roles/monitoring_server/tasks/main.yml index 534a7a12..73d6c660 100644 --- a/roles/monitoring_server/tasks/main.yml +++ b/roles/monitoring_server/tasks/main.yml @@ -1,36 +1,40 @@ --- -- name: Find limited XNAT instances that exist in monitoring_service group +- name: Build `monitoring_server_client_hostnames` from `monitoring_client` group ansible.builtin.set_fact: - # loop through the list and concatenate the result - # the default([]) means use the empty list if list doesn't exist - # add to this the ansible group corresponding to the given item - # then perform the intersection with the monitoring_service and web group - # then get the URL from the host - # and prepend https:// to the start of each entry - monitoring_service_xnat_instances: > + # Get hosts in the `monitoring_client` + monitoring_server_client_hostnames: > {{ - monitoring_service_xnat_instances | - default([]) + - groups[item] | - intersect(groups['monitoring_service']) | + query('inventory_hostnames', ansible_limit | default('')) | + intersect(groups['monitoring_client']) | + map('extract', hostvars, monitoring_server_hostname_extractor) | + list | default([]) + }} + failed_when: monitoring_server_client_hostnames | length == 0 + +- name: Find web servers in `monitoring_client` group + ansible.builtin.set_fact: + # Get any hosts in the `monitoring_client` that are + # also in the `web` group + monitoring_server_web_clients: > + {{ + query('inventory_hostnames', ansible_limit | default('')) | + intersect(groups['monitoring_client']) | intersect(groups['web']) | - map('extract', hostvars, 'ansible_host') | + map('extract', hostvars, monitoring_server_hostname_extractor) | map('regex_replace', '^', 'https://') }} - with_items: - # converts the string to a list - - "{{ ansible_limit.split(',') }}" + failed_when: monitoring_server_web_clients | length == 0 - name: Add monitoring_server group ansible.builtin.group: - name: "{{ monitoring_server.group }}" - gid: "{{ monitoring_server.gid }}" + name: "{{ monitoring_server_group }}" + gid: "{{ monitoring_server_gid }}" -- name: Add "monitoring_server" user +- name: Add monitoring_server user ansible.builtin.user: - name: "{{ monitoring_server.owner }}" - group: "{{ monitoring_server.group }}" - uid: "{{ monitoring_server.uid }}" + name: "{{ monitoring_server_owner }}" + group: "{{ monitoring_server_group }}" + uid: "{{ monitoring_server_uid }}" create_home: false system: true @@ -44,7 +48,7 @@ - name: Ensure docker service restarts after Docker SDK is installed ansible.builtin.meta: flush_handlers -- name: "Create a network" +- name: Create a network community.docker.docker_network: name: monitor-net diff --git a/roles/monitoring_server/templates/alertmanager.yml.j2 b/roles/monitoring_server/templates/alertmanager.yml.j2 index d3a5b634..36c04d5b 100644 --- a/roles/monitoring_server/templates/alertmanager.yml.j2 +++ b/roles/monitoring_server/templates/alertmanager.yml.j2 @@ -1,14 +1,14 @@ global: # The smarthost and SMTP sender used for mail notifications. - smtp_smarthost: {{ smtp.hostname }}:{{ smtp.port }} - smtp_from: {{ monitoring_server.admin.email }} - smtp_auth_username: {{ smtp.username }} - smtp_auth_password: {{ smtp.password }} - smtp_require_tls: {{ smtp.start_tls }} + smtp_smarthost: {{ monitoring_server_smtp_hostname }}:{{ monitoring_server_smtp_port }} + smtp_from: {{ monitoring_server_admin_email }} + smtp_auth_username: {{ monitoring_server_smtp_username }} + smtp_auth_password: {{ monitoring_server_smtp_password }} + smtp_require_tls: {{ monitoring_server_smtp_start_tls }} # The directory from which notification templates are read. templates: -- '{{ alertmanager.volume }}/template/*.tmpl' + - '{{ monitoring_server_alertmanager.volume }}/template/*.tmpl' # The root route on which each incoming alert enters. route: @@ -34,6 +34,6 @@ route: receiver: mirsg-admin-email receivers: -- name: 'mirsg-admin-email' - email_configs: - - to: {{ monitoring_server.alerting.notification_email }} + - name: 'mirsg-admin-email' + email_configs: + - to: {{ monitoring_server_notification_email }} diff --git a/roles/monitoring_server/templates/datasources.yml.j2 b/roles/monitoring_server/templates/datasources.yml.j2 index 553c4bfd..44dc0dc9 100644 --- a/roles/monitoring_server/templates/datasources.yml.j2 +++ b/roles/monitoring_server/templates/datasources.yml.j2 @@ -3,4 +3,4 @@ datasources: - name: Prometheus type: prometheus access: proxy - url: http://{{ prometheus.container_name }}:9090/prometheus + url: http://{{ monitoring_server_prometheus.container_name }}:9090/prometheus diff --git a/roles/monitoring_server/templates/nginx_monitoring_server.j2 b/roles/monitoring_server/templates/monitoring_server_nginx.j2 similarity index 65% rename from roles/monitoring_server/templates/nginx_monitoring_server.j2 rename to roles/monitoring_server/templates/monitoring_server_nginx.j2 index 3cb4d5f4..bff65983 100644 --- a/roles/monitoring_server/templates/nginx_monitoring_server.j2 +++ b/roles/monitoring_server/templates/monitoring_server_nginx.j2 @@ -1,7 +1,7 @@ user nginx; worker_processes auto; -error_log {{ nginx_monitoring_server.log_folder }}/error.log warn; -pid /run/nginx_monitoring_server.pid; +error_log {{ monitoring_server_nginx.log_folder }}/error.log warn; +pid /run/monitoring_server_nginx.pid; events { worker_connections 1024; @@ -15,7 +15,7 @@ http { keepalive_timeout 65; types_hash_max_size 2048; default_type application/octet-stream; - access_log {{ nginx_monitoring_server.log_folder }}/access.log; + access_log {{ monitoring_server_nginx.log_folder }}/access.log; # Good security practice is not to expose nginx version server_tokens off; @@ -24,19 +24,18 @@ http { gzip on; gzip_disable "msie6"; -{% if monitoring_server.ssl.use_ssl %} # SSL parameters must be specified outside of the server block. # Otherwise may only use the nginx may end up using parameters specified in # the default_server block even if a different serer is matched - ssl_certificate {{ nginx_monitoring_server.ssl_cert_file }}; - ssl_certificate_key {{ nginx_monitoring_server.ssl_key_file }}; + ssl_certificate {{ monitoring_server_nginx.ssl_cert_file }}; + ssl_certificate_key {{ monitoring_server_nginx.ssl_key_file }}; # TLS 1.0 and TLS 1.1 should be disabled # TLS 1.3 may not be supported by Centos7 ssl_protocols TLSv1.2; - ssl_dhparam {{ nginx_monitoring_server.dh_params_file }}; + ssl_dhparam {{ monitoring_server_nginx.dh_params_file }}; ssl_ecdh_curve secp384r1; # Increase the cache lifetime to improve performance; this requires a larger cache size @@ -52,58 +51,30 @@ http { # Enable HSTS - this requests browsers to only contact the server using TLS add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; -{% endif %} - - # Default server block for connections that do not match the expected host names. # Note that nginx only uses server_name for disambiguation purposes; if # there is no matching server_name then nginx will use the default_server # regardless of its value of server_name server { listen 80 default_server; -{% if monitoring_server.ssl.use_ssl %} listen 443 ssl default_server; -{% endif %} - -{% if ipv6_enabled %} - listen [::]:80 default_server; - {% if monitoring_server.ssl.use_ssl %} - listen [::]:443 ssl default_server; - {% endif %} -{% endif %} - server_name _; - return 444; } -{% if monitoring_server.ssl.use_ssl %} # Redirect to https server { - listen {{ nginx_monitoring_server.http_port }}; + listen {{ monitoring_server_nginx.http_port }}; server_name {{ ansible_host }}; - return 301 https://{{ ansible_host }}:{{ nginx_monitoring_server.https_port }}$request_uri; + return 301 https://{{ monitoring_server_hostname }}:{{ monitoring_server_nginx.https_port }}$request_uri; } -{% endif %} server { - -{% if monitoring_server.ssl.use_ssl %} - listen {{ nginx_monitoring_server.https_port }} ssl http2; - {% if ipv6_enabled %} - listen [::]:{{ nginx_monitoring_server.https_port }} ssl http2; - {% endif %} -{% else %} - listen {{ nginx_monitoring_server.http_port }}; - {% if ipv6_enabled %} - listen [::]:{{ nginx_monitoring_server.http_port }}; - {% endif %} -{% endif %} - + listen {{ monitoring_server_nginx.https_port }} ssl http2; # Note: server_name is only used for disambiguation - server_name {{ ansible_host }}; + server_name {{ monitoring_server_hostname }}; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; @@ -119,29 +90,29 @@ http { client_body_buffer_size 128k; location ~* /cadvisor/.*$ { - proxy_pass http://{{ cadvisor.container_name }}:8080; + proxy_pass http://{{ monitoring_server_cadvisor.container_name }}:8080; auth_basic "Restricted"; auth_basic_user_file /etc/nginx/.htpasswd; } location /grafana/ { - proxy_pass http://{{ grafana.container_name }}:3000/; + proxy_pass http://{{ monitoring_server_grafana.container_name }}:3000/; } location /prometheus/ { - proxy_pass http://{{ prometheus.container_name }}:9090/prometheus/; + proxy_pass http://{{ monitoring_server_prometheus.container_name }}:9090/prometheus/; auth_basic "Restricted"; auth_basic_user_file /etc/nginx/.htpasswd; } location /alertmanager/ { - proxy_pass http://{{ alertmanager.container_name }}:9093/alertmanager/; + proxy_pass http://{{ monitoring_server_alertmanager.container_name }}:9093/alertmanager/; auth_basic "Restricted"; auth_basic_user_file /etc/nginx/.htpasswd; } - access_log {{ nginx_monitoring_server.log_folder }}/monitoring_server.access.log; - error_log {{ nginx_monitoring_server.log_folder }}/monitoring_server.error.log; + access_log {{ monitoring_server_nginx.log_folder }}/monitoring_server.access.log; + error_log {{ monitoring_server_nginx.log_folder }}/monitoring_server.error.log; } } diff --git a/roles/monitoring_server/templates/prometheus.yml.j2 b/roles/monitoring_server/templates/prometheus.yml.j2 index ae2ba6c6..fb530ef9 100644 --- a/roles/monitoring_server/templates/prometheus.yml.j2 +++ b/roles/monitoring_server/templates/prometheus.yml.j2 @@ -1,5 +1,5 @@ global: - scrape_interval: 60s # By default, scrape targets every 60 seconds. + scrape_interval: 60s # By default, scrape targets every 60 seconds. # Attach these labels to any time series or alerts when communicating with # external systems (federation, remote storage, Alertmanager). @@ -7,60 +7,60 @@ global: monitor: 'mirsg-monitor' rule_files: - - {{ prometheus.rules }} + - {{ monitoring_server_prometheus.rules }} scrape_configs: # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'prometheus' - # Override the global default and scrape targets from this job every 5 seconds. - scrape_interval: 5s - metrics_path: /prometheus/metrics - static_configs: - - targets: ['localhost:9090'] + - job_name: 'prometheus' + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + metrics_path: /prometheus/metrics + static_configs: + - targets: ['localhost:9090'] - - job_name: cadvisor - scrape_interval: 5s - metrics_path: /cadvisor/metrics - static_configs: - - targets: - - {{ cadvisor.container_name }}:8080 + - job_name: cadvisor + scrape_interval: 5s + metrics_path: /cadvisor/metrics + static_configs: + - targets: + - {{ monitoring_server_cadvisor.container_name }}:8080 - - job_name: node - scheme: https - tls_config: - ca_file: {{ prometheus.ca_cert }} - basic_auth: - username: {{ monitoring_client.exporter_auth.username }} - password: {{ monitoring_client.exporter_auth.password }} - static_configs: + - job_name: node + scheme: https + tls_config: + ca_file: {{ monitoring_server_prometheus.ca_cert }} + basic_auth: + username: {{ monitoring_server_client_exporter_username }} + password: {{ monitoring_server_client_exporter_password }} + static_configs: + - targets: + {% for client in monitoring_server_client_hostnames %} + - {{ client }}:9100 + {% endfor %} + + - job_name: blackbox + metrics_path: /probe + scrape_interval: 5m + params: + module: [http_2xx] + static_configs: - targets: - {% for client in monitoring_service_clients %} - - {{ client }}:9100 + {% for web in monitoring_server_web_clients %} + - {{ web }} {% endfor %} - - job_name: blackbox - metrics_path: /probe - scrape_interval: 5m - params: - module: [http_2xx] - static_configs: - - targets: - {% for xnat in monitoring_service_xnat_instances %} - - {{ xnat }} - {% endfor %} - - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: {{ blackbox_exporter.container_name }}:9115 # Blackbox exporter. + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 # Blackbox exporter. alerting: alertmanagers: - path_prefix: "alertmanager/" static_configs: - targets: - - {{ alertmanager.container_name }}:9093 + - {{ monitoring_server_alertmanager.container_name }}:9093 diff --git a/roles/monitoring_server/templates/prometheus_rules.yml.j2 b/roles/monitoring_server/templates/prometheus_rules.yml.j2 index e91d07eb..43f93c3c 100644 --- a/roles/monitoring_server/templates/prometheus_rules.yml.j2 +++ b/roles/monitoring_server/templates/prometheus_rules.yml.j2 @@ -1,4 +1,14 @@ groups: + - name: tls-expiry + rules: + - alert: SSLCertExpiringSoon + expr: probe_ssl_earliest_cert_expiry - time() < 60 * 60 * 24 * 14 + for: 1m + labels: + severity: warning + annotations: + description: "TLS certificate will expire in {{ $value | humanizeDuration }} (instance {{ $labels.instance }})" + - name: down-detector rules: - alert: InstanceDown diff --git a/roles/sign_monitoring_client_cert/tasks/main.yml b/roles/sign_monitoring_client_cert/tasks/main.yml deleted file mode 100644 index 6c40874e..00000000 --- a/roles/sign_monitoring_client_cert/tasks/main.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -- name: Find limited hosts that exist in monitoring_service group - ansible.builtin.set_fact: - # loop through the list and concatenate the result - # the default([]) means use the empty list if list doesn't exist - # add to this the ansible group corresponding to the given item - # then perform the intersection with the monitoring_service group - # the final step gets the URL from the host - monitoring_service_clients: > - {{ - monitoring_service_clients | - default([]) + - groups[item] | - intersect(groups['monitoring_service']) | - map('extract', hostvars, 'ansible_host') - }} - with_items: - # converts the string to a list - - "{{ ansible_limit.split(',') }}" - -- name: Ensure monitoring cert dir exists on server - ansible.builtin.file: - path: "{{ monitoring_server.client_cert_dir }}" - state: directory - owner: "{{ monitoring_server.cert_owner }}" - group: "{{ monitoring_server.cert_group }}" - mode: "0700" - -- name: Copy monitoring client CSRs to server - ansible.builtin.copy: - src: "{{ monitoring_temp_files_cert_dir }}/{{ item }}.csr" - dest: "{{ monitoring_server.client_cert_dir }}/{{ item }}.csr" - owner: "{{ monitoring_server.cert_owner }}" - group: "{{ monitoring_server.cert_group }}" - mode: "0600" - loop: "{{ monitoring_service_clients }}" - -- name: Generate client certificates signed by server CA - community.crypto.x509_certificate: - path: "{{ monitoring_server.client_cert_dir }}/{{ item }}.cert" - csr_path: "{{ monitoring_server.client_cert_dir }}/{{ item }}.csr" - provider: ownca - ownca_path: "{{ docker.ca_cert }}" - ownca_privatekey_path: "{{ docker.ca_key }}" - mode: "0400" - owner: "{{ monitoring_server.cert_owner }}" - group: "{{ monitoring_server.cert_group }}" - loop: "{{ monitoring_service_clients }}" - -- name: Copy signed client certificates - ansible.builtin.fetch: - src: "{{ monitoring_server.client_cert_dir }}/{{ item }}.cert" - dest: "{{ monitoring_temp_files_cert_dir }}/{{ item }}.cert" - flat: true - loop: "{{ monitoring_service_clients }}" - -- name: Copy server certificate - ansible.builtin.fetch: - src: "{{ docker.ca_cert }}" - dest: "{{ monitoring_temp_files_cert_dir }}/ca.pem" - owner: "{{ monitoring_server.cert_owner }}" - group: "{{ monitoring_server.cert_group }}" - mode: "0600" - flat: true diff --git a/tests/molecule/centos7_monitoring/molecule.yml b/tests/molecule/centos7_monitoring/molecule.yml new file mode 100644 index 00000000..51dbb467 --- /dev/null +++ b/tests/molecule/centos7_monitoring/molecule.yml @@ -0,0 +1,91 @@ +--- +dependency: + name: galaxy + options: + force: true + role-file: ${MOLECULE_SCENARIO_DIRECTORY}/../../../meta/requirements.yml + requirements-file: ${MOLECULE_SCENARIO_DIRECTORY}/../../../meta/requirements.yml + +driver: + name: docker + +platforms: + - name: mclient + hostname: molecule.mclient.local + image: ${MOLECULE_DOCKER_IMAGE:-geerlingguy/docker-centos7-ansible:latest} + required: true + command: "" + cgroupns_mode: host + privileged: true + pre_build_image: ${MOLECULE_PRE_BUILD_IMAGE:-true} + volumes: + - ./molecule-data:/storage/molecule + keep_volumes: false + groups: + - all + - monitoring_client + - monitoring_service + docker_networks: + - name: monitoring + ipam_config: + - subnet: 192.168.56.0/24 + gateway: 192.168.56.1 + networks: + - name: monitoring + ipv4_address: 192.168.56.2 + etc_hosts: + molecule.mserv.local: 192.168.56.3 + + - name: mserv + hostname: molecule.mserv.local + image: ${MOLECULE_DOCKER_IMAGE:-geerlingguy/docker-centos7-ansible:latest} + required: true + command: "" + cgroupns_mode: host + privileged: true + pre_build_image: ${MOLECULE_PRE_BUILD_IMAGE:-true} + volumes: + - ./molecule-data:/storage/molecule + keep_volumes: false + groups: + - all + - monitoring_host + - monitoring_service + networks: + - name: monitoring + ipv4_address: 192.168.56.3 + exposed_ports: + - 80 + - 443 + etc_hosts: + molecule.mclient.local: 192.168.56.2 + +provisioner: + name: ansible + log: true + config_options: + defaults: + callbacks_enabled: profile_tasks, timer, yaml + tags: + run: ${MOLECULE_RUN_TAGS:-all} + inventory: + links: + hosts: ../resources/monitoring/inventory/hosts.yml + group_vars: ../resources/monitoring/inventory/group_vars/ + host_vars: ../resources/monitoring/inventory/host_vars/ + playbooks: + converge: ../resources/monitoring/converge.yml + prepare: ../resources/shared/prepare.yml + verify: ../resources/monitoring/verify.yml + env: + ANSIBLE_VERBOSITY: "1" + +verifier: + name: ansible + env: + ANSIBLE_VERBOSITY: "1" + +lint: | + set -e + yamllint . + ansible-lint . diff --git a/tests/molecule/centos7_roles/molecule.yml b/tests/molecule/centos7_roles/molecule.yml index 1d17a4f8..defd4bb5 100644 --- a/tests/molecule/centos7_roles/molecule.yml +++ b/tests/molecule/centos7_roles/molecule.yml @@ -44,12 +44,12 @@ provisioner: run: ${MOLECULE_RUN_TAGS:-all} inventory: links: - hosts: ../resources/inventory/hosts.yml - group_vars: ../resources/inventory/group_vars/ + hosts: ../resources/roles/inventory/hosts.yml + group_vars: ../resources/roles/inventory/group_vars/ playbooks: - converge: ../resources/converge.yml - prepare: ../resources/prepare.yml - verify: ../resources/verify.yml + converge: ../resources/roles/converge.yml + prepare: ../resources/shared/prepare.yml + verify: ../resources/roles/verify.yml env: ANSIBLE_VERBOSITY: "1" diff --git a/tests/molecule/resources/monitoring/converge.yml b/tests/molecule/resources/monitoring/converge.yml new file mode 100644 index 00000000..48badf9d --- /dev/null +++ b/tests/molecule/resources/monitoring/converge.yml @@ -0,0 +1,26 @@ +--- +- name: Create monitoring server certs + hosts: monitoring_host + become: true + gather_facts: true + tags: monitoring + pre_tasks: + - name: Check the major Python version for the OS + block: + - name: Check default version + ansible.builtin.include_role: + name: mirsg.infrastructure.install_python + tasks_from: check_default_version + + - name: Set install_python variable based on the version to be installed + ansible.builtin.set_fact: + install_python: "{{ install_python2 if default_python_version is version('2') else install_python3 }}" + + roles: + - role: mirsg.infrastructure.ssl_certificates + vars: + ssl_certificate: "{{ monitoring_server_ssl_certificate }}" + +- name: Install Monitoring Service + ansible.builtin.import_playbook: mirsg.infrastructure.install_monitoring + tags: monitoring diff --git a/tests/molecule/resources/monitoring/inventory/group_vars/all.yml b/tests/molecule/resources/monitoring/inventory/group_vars/all.yml new file mode 100644 index 00000000..9ffa9e95 --- /dev/null +++ b/tests/molecule/resources/monitoring/inventory/group_vars/all.yml @@ -0,0 +1,33 @@ +--- +ansible_cache_dir: "{{ lookup('env', 'HOME') }}/ansible_persistent_files" + +external_storage_drive: "/storage/molecule" +selinux_enabled: false + +# mirsg.infrastructure.provision +server_locale: "en_GB.UTF-8" + +# mirsg.infrastructure.install_python +install_python3: + version: "3" + pip_version: "21.3.1" + pip_executable: /usr/local/bin/pip3 + system_packages: + - python3 + - python3-pip + - python3-setuptools + pip_packages: + - cryptography + +install_python2: + version: "2" + pip_version: "20.3.4" + pip_executable: /usr/bin/pip + system_packages: + - python + - python-pip + - python-setuptools + pip_packages: + - cryptography + +install_python: "{{ install_python3 }}" # default to Python 3 diff --git a/tests/molecule/resources/monitoring/inventory/group_vars/monitoring_client.yml b/tests/molecule/resources/monitoring/inventory/group_vars/monitoring_client.yml new file mode 100644 index 00000000..b1676eab --- /dev/null +++ b/tests/molecule/resources/monitoring/inventory/group_vars/monitoring_client.yml @@ -0,0 +1,11 @@ +# mirsg.infrastructure.monitoring_client +monitoring_client_certificate_file: "{{ hostvars[inventory_hostname]['hostname'] }}.cert" +monitoring_client_monitoring_server_ip: "{{ hostvars['mserv']['ansible_ip'] }}" +monitoring_client_owner: root +monitoring_client_group: root +monitoring_client_cert_dir: /root/monitoring_certs +monitoring_client_ssl_key_file: /root/monitoring_certs/key.pem +monitoring_client_ssl_csr_file: /root/monitoring_certs/monitoring.csr +monitoring_client_ssl_pk8_file: /root/monitoring_certs/monitoring.pk8 +monitoring_client_ssl_cert_file: /root/monitoring_certs/cert.pem +monitoring_client_server_ca_cert_file: /root/monitoring_certs/ca.pem diff --git a/tests/molecule/resources/monitoring/inventory/group_vars/monitoring_host.yml b/tests/molecule/resources/monitoring/inventory/group_vars/monitoring_host.yml new file mode 100644 index 00000000..b36c1517 --- /dev/null +++ b/tests/molecule/resources/monitoring/inventory/group_vars/monitoring_host.yml @@ -0,0 +1,21 @@ +monitoring_server_hostname: "{{ hostvars['mserv']['hostname'] }}" +monitoring_server_ssl_cert: "/etc/ssl/certs/{{ monitoring_server_hostname }}.cert" +monitoring_server_ssl_key: "/etc/ssl/certs/{{ monitoring_server_hostname }}.key" +monitoring_server_admin_username: mirsg_service +monitoring_server_admin_password: password +monitoring_server_admin_email: admin@monitoring.org +monitoring_server_notification_email: test@monitoring.org +monitoring_server_storage_root: "{{ external_storage_drive }}/monitoring" +monitoring_server_owner: monitoring +monitoring_server_group: monitoring +monitoring_server_uid: 595 +monitoring_server_gid: 595 +monitoring_server_validate_certs: false +monitoring_server_cert_owner: root +monitoring_server_cert_group: root +monitoring_server_cert_dir: /root/monitoring_certs +monitoring_server_client_cert_dir: /root/monitoring_certs/client_certs +monitoring_server_hostname_extractor: "hostname" +monitoring_server_grafana_username: grafana +monitoring_server_grafana_password: grafana +monitoring_server_grafana_host: "{{ monitoring_server_hostname }}" diff --git a/tests/molecule/resources/monitoring/inventory/group_vars/monitoring_service.yml b/tests/molecule/resources/monitoring/inventory/group_vars/monitoring_service.yml new file mode 100644 index 00000000..91ef30e7 --- /dev/null +++ b/tests/molecule/resources/monitoring/inventory/group_vars/monitoring_service.yml @@ -0,0 +1,18 @@ +# mirsg.infrastructure.docker +docker_client_certificate_cache_directory: "{{ lookup('env', 'HOME') }}/ansible_persistent_files/monitoring_server_certificates" +docker_server_hostname: "{{ hostvars['mserv']['hostname'] }}" +docker_server_ip: "{{ hostvars['mserv']['ansible_ip'] }}" +docker_server_port: "2376" +docker_service_name: docker +docker_generate_certificates: true # generate TLS certs for clients +docker_tls_verify: true + +# mirsg.infrastructure.monitoring_client +monitoring_client_exporter_username: prometheus +monitoring_client_exporter_password: prometheus +monitoring_client_certificate_cache_directory: "{{ docker_client_certificate_cache_directory }}" + +# mirsg.infrastructure.monitoring_server +monitoring_server_client_exporter_username: "{{ monitoring_client_exporter_username }}" +monitoring_server_client_exporter_password: "{{ monitoring_client_exporter_password }}" +monitoring_server_certificate_cache_directory: "{{ docker_client_certificate_cache_directory }}" diff --git a/tests/molecule/resources/monitoring/inventory/host_vars/mserv.yml b/tests/molecule/resources/monitoring/inventory/host_vars/mserv.yml new file mode 100644 index 00000000..50437929 --- /dev/null +++ b/tests/molecule/resources/monitoring/inventory/host_vars/mserv.yml @@ -0,0 +1,10 @@ +monitoring_server_ssl_certificate: + owner: "{{ monitoring_server_cert_owner }}" + group: "{{ monitoring_server_cert_group }}" + certificate_directory: /etc/ssl/certs + privatekey_filename: "/etc/ssl/certs/{{ hostvars['mserv']['hostname'] }}.key" + use_pk8: false + csr_filename: "/etc/ssl/certs/{{ hostvars['mserv']['hostname'] }}.csr" + csr_common_name: "{{ hostvars['mserv']['hostname'] }}" + certificate_filename: "/etc/ssl/certs/{{ hostvars['mserv']['hostname'] }}.cert" + provider: selfsigned diff --git a/tests/molecule/resources/monitoring/inventory/hosts.yml b/tests/molecule/resources/monitoring/inventory/hosts.yml new file mode 100644 index 00000000..a5e01eef --- /dev/null +++ b/tests/molecule/resources/monitoring/inventory/hosts.yml @@ -0,0 +1,39 @@ +--- +all: + # List of all servers, defining their alias and IP (hostname) + hosts: + mclient: + hostname: molecule.mclient.local + ansible_ip: 192.168.56.2 + ansible_port: 22 + + mserv: + hostname: molecule.mserv.local + ansible_ip: 192.168.56.3 + ansible_port: 22 + + # Ansible groups. Groups allow configuration and variables to be shared between hosts + # Variables in group_vars/all will be shared between all hosts + children: + # All the monitoring service (docker) servers. Variables in group_vars/monitoring_server will be shared + # between these hosts + monitoring_host: + hosts: + mserv: + + # All the monitoring service (web) clients. Variables in group_vars/monitoring_client will be shared + # between these hosts + monitoring_client: + hosts: + mclient: + + # All the monitoring service hosts and clients. Variables in group_vars/monitoring_service will be shared + # between these hosts + monitoring_service: + hosts: + mclient: + mserv: + + web: + hosts: + mclient: diff --git a/tests/molecule/resources/monitoring/verify.yml b/tests/molecule/resources/monitoring/verify.yml new file mode 100644 index 00000000..ceb3f341 --- /dev/null +++ b/tests/molecule/resources/monitoring/verify.yml @@ -0,0 +1,40 @@ +- name: Monitoring verify + hosts: monitoring_host + become: true + gather_facts: true + tasks: + - name: Check alertmanager exporter container is running + community.docker.docker_container_info: + name: "{{ monitoring_server_alertmanager.container_name }}" + register: result + failed_when: "'exists' not in result.exists" + + - name: Check blackbox exporter container is running + community.docker.docker_container_info: + name: blackbox-exporter + register: result + failed_when: "'exists' not in result.exists" + + - name: Check cadvisor container is running + community.docker.docker_container_info: + name: "{{ monitoring_server_cadvisor.container_name }}" + register: result + failed_when: "'exists' not in result.exists" + + - name: Check Grafana container is running + community.docker.docker_container_info: + name: "{{ monitoring_server_grafana.container_name }}" + register: result + failed_when: "'exists' not in result.exists" + + - name: Check nginx container is running + community.docker.docker_container_info: + name: nginx + register: result + failed_when: "'exists' not in result.exists" + + - name: Check Prometheus container is running + community.docker.docker_container_info: + name: "{{ monitoring_server_prometheus.container_name }}" + register: result + failed_when: "'exists' not in result.exists" diff --git a/tests/molecule/resources/converge.yml b/tests/molecule/resources/roles/converge.yml similarity index 100% rename from tests/molecule/resources/converge.yml rename to tests/molecule/resources/roles/converge.yml diff --git a/tests/molecule/resources/inventory/group_vars/all.yml b/tests/molecule/resources/roles/inventory/group_vars/all.yml similarity index 95% rename from tests/molecule/resources/inventory/group_vars/all.yml rename to tests/molecule/resources/roles/inventory/group_vars/all.yml index 6af68a01..e4188f22 100644 --- a/tests/molecule/resources/inventory/group_vars/all.yml +++ b/tests/molecule/resources/roles/inventory/group_vars/all.yml @@ -28,7 +28,7 @@ install_python2: pip_packages: - cryptography -install_python: "{{ install_python_3 }}" # default to Python 3 +install_python: "{{ install_python3 }}" # default to Python 3 # mirsg.infrastructure.firewalld allow_public_access: true diff --git a/tests/molecule/resources/inventory/hosts.yml b/tests/molecule/resources/roles/inventory/hosts.yml similarity index 100% rename from tests/molecule/resources/inventory/hosts.yml rename to tests/molecule/resources/roles/inventory/hosts.yml diff --git a/tests/molecule/resources/verify.yml b/tests/molecule/resources/roles/verify.yml similarity index 100% rename from tests/molecule/resources/verify.yml rename to tests/molecule/resources/roles/verify.yml diff --git a/tests/molecule/resources/verify/firewalld.yml b/tests/molecule/resources/roles/verify/firewalld.yml similarity index 100% rename from tests/molecule/resources/verify/firewalld.yml rename to tests/molecule/resources/roles/verify/firewalld.yml diff --git a/tests/molecule/resources/prepare.yml b/tests/molecule/resources/shared/prepare.yml similarity index 93% rename from tests/molecule/resources/prepare.yml rename to tests/molecule/resources/shared/prepare.yml index d0e4b210..b13ed28f 100644 --- a/tests/molecule/resources/prepare.yml +++ b/tests/molecule/resources/shared/prepare.yml @@ -4,7 +4,9 @@ gather_facts: true tasks: - name: Install and configure firewalld - tags: firewalld + tags: + - firewalld + - monitoring block: - name: Install firewalld ansible.builtin.package: diff --git a/tests/molecule/rocky9_monitoring/molecule.yml b/tests/molecule/rocky9_monitoring/molecule.yml new file mode 100644 index 00000000..09218dbf --- /dev/null +++ b/tests/molecule/rocky9_monitoring/molecule.yml @@ -0,0 +1,93 @@ +--- +dependency: + name: galaxy + options: + force: true + role-file: ${MOLECULE_SCENARIO_DIRECTORY}/../../../meta/requirements.yml + requirements-file: ${MOLECULE_SCENARIO_DIRECTORY}/../../../meta/requirements.yml + +driver: + name: docker + +platforms: + - name: mclient + hostname: molecule.mclient.local + image: ${MOLECULE_DOCKER_IMAGE:-geerlingguy/docker-rockylinux9-ansible:latest} + required: true + command: "" + cgroupns_mode: host + privileged: true + pre_build_image: ${MOLECULE_PRE_BUILD_IMAGE:-true} + volumes: + - ./molecule-data:/storage/molecule + - /sys/fs/cgroup:/sys/fs/cgroup:rw + keep_volumes: false + groups: + - all + - monitoring_client + - monitoring_service + docker_networks: + - name: monitoring + ipam_config: + - subnet: 192.168.56.0/24 + gateway: 192.168.56.1 + networks: + - name: monitoring + ipv4_address: 192.168.56.2 + etc_hosts: + molecule.mserv.local: 192.168.56.3 + + - name: mserv + hostname: molecule.mserv.local + image: geerlingguy/docker-rockylinux9-ansible:latest + required: true + command: "" + cgroupns_mode: host + privileged: true + pre_build_image: true + volumes: + - ./molecule-data:/storage/molecule + - /sys/fs/cgroup:/sys/fs/cgroup:rw + keep_volumes: false + groups: + - all + - monitoring_service + - monitoring_host + networks: + - name: monitoring + ipv4_address: 192.168.56.3 + exposed_ports: + - 80 + - 443 + etc_hosts: + molecule.mclient.local: 192.168.56.2 + +provisioner: + name: ansible + log: true + config_options: + defaults: + callbacks_enabled: profile_tasks, timer, yaml + tags: + run: ${MOLECULE_RUN_TAGS:-all} + inventory: + links: + hosts: ../resources/monitoring/inventory/hosts.yml + group_vars: ../resources/monitoring/inventory/group_vars/ + host_vars: ../resources/monitoring/inventory/host_vars/ + playbooks: + converge: ../resources/monitoring/converge.yml + prepare: ../resources/shared/prepare.yml + verify: ../resources/monitoring/verify.yml + env: + ANSIBLE_VERBOSITY: "1" + +verifier: + name: ansible + env: + ANSIBLE_VERBOSITY: "1" + +lint: | + set -e + yamllint . + ansible-lint . diff --git a/tests/molecule/rocky9_roles/molecule.yml b/tests/molecule/rocky9_roles/molecule.yml index 885471d5..e28e2b18 100644 --- a/tests/molecule/rocky9_roles/molecule.yml +++ b/tests/molecule/rocky9_roles/molecule.yml @@ -45,12 +45,12 @@ provisioner: run: ${MOLECULE_RUN_TAGS:-all} inventory: links: - hosts: ../resources/inventory/hosts.yml - group_vars: ../resources/inventory/group_vars/ + hosts: ../resources/roles/inventory/hosts.yml + group_vars: ../resources/roles/inventory/group_vars/ playbooks: - converge: ../resources/converge.yml - prepare: ../resources/prepare.yml - verify: ../resources/verify.yml + converge: ../resources/roles/converge.yml + prepare: ../resources/shared/prepare.yml + verify: ../resources/roles/verify.yml env: ANSIBLE_VERBOSITY: "1"