Secondary HTCondor cluster playbook (container setup and demonstration)

Add a playbook that uses the kysrpex.systemd_nspawn role to set up a container that can be managed using Ansible.

An empty playbook (just the Ansible task that gathers facts) is run on the container as a demonstration.

group_vars/central-manager-secondary-host.yml

@@ -0,0 +1,62 @@
+---
+nspawn_name: htcondor
+nspawn_distro: "rocky"
+nspawn_release: "8"
+nspawn_packages:
+  - dhcp-client
+  - dnf
+  - glibc-langpack-en
+  - iproute
+  - iputils
+  - less
+  - passwd
+  - systemd
+  - dbus
+  - vim-minimal
+  - openssh-server
+
+nspawn_config: |
+  [Files]
+  {% for mount in jwd.values() %}
+  Bind={{ mount.path }}
+  {% endfor %}
+
+  [Exec]
+  NotifyReady=yes
+  PrivateUsers=no
+
+  [Network]
+  VirtualEthernet=no
+
+nspawn_enable: false
+nspawn_start: false
+
+nspawn_ssh: yes
+nspawn_ssh_config_path: /etc/ssh/sshd_config
+nspawn_ssh_config:
+  Port: "2222"
+  ListenAddress: "127.0.0.1"
+  PermitRootLogin: "yes"
+  PubkeyAuthentication: "yes"
+  AuthenticationMethods: "publickey"
+  PasswordAuthentication: "no"
+  PermitEmptyPasswords: "no"
+  ChallengeResponseAuthentication: "no"
+  PermitTunnel: "no"
+  AllowTcpForwarding: "no"
+  AllowAgentForwarding: "no"
+  GatewayPorts: "no"
+nspawn_ssh_host_keys:
+  ecdsa: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF9MMtDEfPPSaHSNUFz3b1BBtasjGT1XB2my45Lhtp4NLJv/jPxpVQ8sxRSS3+8fc9+1EZXF0AGj4D1NjqpabmQ="
+  ed25519: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHMTH60CkuT9moO66XVEaUP1YelPv/aOQdqSImNNiQFh"
+  rsa: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDxIgnu5/BIIzVbWM442uzwyuXPQx0jqaVfffjexwewfc2mMNnVAG7mRzcWcgr+o/lFiogkqGulkvO/a+EARs5WsWrsq5bGFcHx9DPgaT+7smu+K3t0WNiqPEykXUJppyMtaYftZSWaCSl51pg3QT29pvl95sjC3URFrQhoaLHJ2fESYJI4WuMBig/rCLaRsF6xaR38LIVfrZdA7UjG5Ncr+FXF6cpNtdvDvJXkGyuYft2p9LMh7vycdiSUL/CluC6tptgQ82JGxgyHDaxwJvyzzDukjKTUkR59ctuGthq9M/M384sCn8Z3PsGE4hkRGdL/HJZhasPIkkwJK8ekucnBR2LzlCqiTrn7HFbndqgePhJWHq0tBN53o2Vmczrbq6UVROAFDCpPKeaWa6hcdqzp0fpHrgSZEyKmMxThV8sj69E/mCh2PEgWjCXsPoJLwrW9TuPUMeXx2+Cwf4/JaFSvLBT19ADFKkzgAPeEVdJg2ZkjWZVO+xxb9e6Ahux+g7E="
+nspawn_ssh_host_certs:
+  # valid from 2023-10-12T14:46:00 to 2033-09-29T14:47:45
+  ecdsa: "[email protected] AAAAKGVjZHNhLXNoYTItbmlzdHAyNTYtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgIkfzNxdybaCbJ+rziM3zQy/grFSitfDOJWQS2zH4I4MAAAAIbmlzdHAyNTYAAABBBF9MMtDEfPPSaHSNUFz3b1BBtasjGT1XB2my45Lhtp4NLJv/jPxpVQ8sxRSS3+8fc9+1EZXF0AGj4D1NjqpabmQAAAAAAAAAAAAAAAIAAAAXbWFuYWdlci1zZWNvbmRhcnktZWNkc2EAAAAAAAAAAGUoBqgAAAAAd+bbEQAAAAAAAAAAAAAAAAAAARcAAAAHc3NoLXJzYQAAAAMBAAEAAAEBAMtAPp8bfy7AWP3UZJ7pidGcG5ONdZk3o9yTUlEJNa25/ApXRJk1zMP6pR2cV9msLH/AFUrxHrMvZVZL2FwUrUu4Twcrzp/yob+BwHrIW3/Qj89ahKNxtQw9BPMUjLzLG90ig9OWP8/r642DB5O8wUjj5Q64IjvUiEH+WJMefX/Z1dAz3hIV3hVB6D7dNzqAPESpFJLp0sirS69Hz/+W4imhbfzKHajPAeaMwpO8edt24aoeURLirP1xT2JmRag/NKn6qjEI6KMe/DMvm8PT+TiFn+InJPMMdAsOtrcpprzKRZ0RELGOkynXL11VDbeVXGn5RQC3dEc4JTX8q40z0+sAAAEUAAAADHJzYS1zaGEyLTUxMgAAAQBbd3PNc1Ezs5BDCF/Ca02Oea4hL/L3Z1G4PO1W7y3IHAVW2hFmHBmJTd2r5tO/G+n/o03j+wr59ZfKYr4sj9c99oMXA19IRl+3a7ECnG6xpnepVA8FMo8qatcTyeOlECDbS8VI8Kc+cBuSu4UZjdCIZCFGzIj9dupK22id7D/UdeqDNTZ0b+KB0NOQM6lzyLaXD75U9kOFFzWmJqRYtkFns71dhzzr1i04V+ZaPlmSz2QZ20WIqt2hX6ecU+w3U2fK0PkFy/cy3YtSzhnNRiKnmJk1U4MOCbxWQ3rFOTopX514elUZAUCw3HCU+vBIOwwDIrVlbNuCEecuMtQUfLyW /etc/ssh/ssh_host_ecdsa_key.pub"
+  # valid from 2023-10-12T14:46:00 to 2033-09-29T14:47:55
+  ed25519: "[email protected] AAAAIHNzaC1lZDI1NTE5LWNlcnQtdjAxQG9wZW5zc2guY29tAAAAILMcMkZV0MfVGEZghy0DD2dOMTtpjE5z0Hj6Cus/C8mjAAAAIHMTH60CkuT9moO66XVEaUP1YelPv/aOQdqSImNNiQFhAAAAAAAAAAAAAAACAAAAGW1hbmFnZXItc2Vjb25kYXJ5LWVjMjU1MTkAAAAAAAAAAGUoBqgAAAAAd+bbGwAAAAAAAAAAAAAAAAAAARcAAAAHc3NoLXJzYQAAAAMBAAEAAAEBAMtAPp8bfy7AWP3UZJ7pidGcG5ONdZk3o9yTUlEJNa25/ApXRJk1zMP6pR2cV9msLH/AFUrxHrMvZVZL2FwUrUu4Twcrzp/yob+BwHrIW3/Qj89ahKNxtQw9BPMUjLzLG90ig9OWP8/r642DB5O8wUjj5Q64IjvUiEH+WJMefX/Z1dAz3hIV3hVB6D7dNzqAPESpFJLp0sirS69Hz/+W4imhbfzKHajPAeaMwpO8edt24aoeURLirP1xT2JmRag/NKn6qjEI6KMe/DMvm8PT+TiFn+InJPMMdAsOtrcpprzKRZ0RELGOkynXL11VDbeVXGn5RQC3dEc4JTX8q40z0+sAAAEUAAAADHJzYS1zaGEyLTUxMgAAAQAno9IlnZ6TLhK83xx1+qjTL2+DjYsFTVeH0D97P7QdvZ3l4VnOWJH62V79VSNA4VQ7MRgoUZkeVY6Y2OBBItGGNxn0Z/9O/yE8csoqKE0L0LJNWoEU8Xbw/rs7rr4gIUqTNqzT42dSt7p6chht8+9iCSku8bKHmmnokQOnC/tF29zC5Iq4FQoVYDMgm0YO2MN9mJ6R9zEzkcu/cPWTjPM0FY9qs/RlK7yGQXcINAxaW4a5xdlMp85LWygGRLBn+gKiCwUATW1EPx7MkuHUx+vHzeKwRDsc904KxYD9TWsrSygFAfesLY2Xh6TDVdrPCnw096K7l9EcovgSCVWPV1mM /etc/ssh/ssh_host_ed25519_key.pub"
+  # valid from 2023-10-12T14:47:00 to 2033-09-29T14:48:11
+  rsa: "[email protected] AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgQuoZdMdY7Boli6KRlNmUk2zQar66du6kmLR4PkWoHI8AAAADAQABAAABgQDxIgnu5/BIIzVbWM442uzwyuXPQx0jqaVfffjexwewfc2mMNnVAG7mRzcWcgr+o/lFiogkqGulkvO/a+EARs5WsWrsq5bGFcHx9DPgaT+7smu+K3t0WNiqPEykXUJppyMtaYftZSWaCSl51pg3QT29pvl95sjC3URFrQhoaLHJ2fESYJI4WuMBig/rCLaRsF6xaR38LIVfrZdA7UjG5Ncr+FXF6cpNtdvDvJXkGyuYft2p9LMh7vycdiSUL/CluC6tptgQ82JGxgyHDaxwJvyzzDukjKTUkR59ctuGthq9M/M384sCn8Z3PsGE4hkRGdL/HJZhasPIkkwJK8ekucnBR2LzlCqiTrn7HFbndqgePhJWHq0tBN53o2Vmczrbq6UVROAFDCpPKeaWa6hcdqzp0fpHrgSZEyKmMxThV8sj69E/mCh2PEgWjCXsPoJLwrW9TuPUMeXx2+Cwf4/JaFSvLBT19ADFKkzgAPeEVdJg2ZkjWZVO+xxb9e6Ahux+g7EAAAAAAAAAAAAAAAIAAAAVbWFuYWdlci1zZWNvbmRhcnktcnNhAAAAAAAAAABlKAbkAAAAAHfm2ysAAAAAAAAAAAAAAAAAAAEXAAAAB3NzaC1yc2EAAAADAQABAAABAQDLQD6fG38uwFj91GSe6YnRnBuTjXWZN6Pck1JRCTWtufwKV0SZNczD+qUdnFfZrCx/wBVK8R6zL2VWS9hcFK1LuE8HK86f8qG/gcB6yFt/0I/PWoSjcbUMPQTzFIy8yxvdIoPTlj/P6+uNgweTvMFI4+UOuCI71IhB/liTHn1/2dXQM94SFd4VQeg+3Tc6gDxEqRSS6dLIq0uvR8//luIpoW38yh2ozwHmjMKTvHnbduGqHlES4qz9cU9iZkWoPzSp+qoxCOijHvwzL5vD0/k4hZ/iJyTzDHQLDra3Kaa8ykWdERCxjpMp1y9dVQ23lVxp+UUAt3RHOCU1/KuNM9PrAAABFAAAAAxyc2Etc2hhMi01MTIAAAEAOXRiZFs14HScCvi+9Hp51z310t0r3SHtHhE50vEjdAsnej+/yHUtlmaYJ1l4B8SWYHHUV94KxZbQP79D8PYQextXoYldA4Vp29ow33sYF8Mr4p7Jbjyyr4GVP9RAymiZz9cuiiRqjs8qwn/UCOw4A6yaix4k0EYTJa9bpbD3MsCxJbKEdZxxL72kIKf4sGuDrd77hEtRwJFUSqz+XD96TYG+BIomnrk35Oy+oyMGhoIlSM0//cS0pHUW8fpGnRIJJbBjDETwOmaMEF5yynRSeJyWnEBPpjcA+1i7wILSxYxZb5ckGxCABYxNvlIhF8Bvdcyb4eo7B/WeEdzr5C+Pzg== /etc/ssh/ssh_host_rsa_key.pub"
+nspawn_ssh_authorized_keys:
+  - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDV7gfNbNN5O8vH6/tM/iOFXKBP2YKRHXOmdfV8ogvu9BdVV0IPmDzk2EooVpThDE1VMv1hz3811tvBhHRJ6IgNhVIV/61w/+RazQD/AU27X8bX+Hb9EQ/bP4DW+6ySd/z5vdDLzpH5dbiMhzPEDkXVsylUT+hkQnas6cHspDhHmtKQ5MWOgDe3D/IEudTDJQe8hxxaU4TaZUmFzn7eYp9HvuK8qW0yCy4NWOxJJHA+G5wSCyLuKnaKo4AitUIzSKF1AB94oq7b96KONhPxgRptAk4OYIUTdNFbrI5HDaSNzHLnF5FbjQvG+Eu6m5nY5yvJMogE+jiuWeIXCZTCFljg287FUo0ohmbZpd802L6VXun14VumRC+rRgPrvBALo/CsyCsPIoBSTKhVElxKVOcRjmTLNfrUZM0GQxqJhIvah8BV+JTExkipPwkrKTdMAWIXvCoehxV+WMpBWqtEEzAzEoqJpaiec7HfriwsHTGESZWAPYEbFjzbHXQZtqBkbOvtokPMRmTWfWKxaplCMN6ddJeeY6faorD0w/e6lszWES1Q1ieajiPKDy37UvybKKvPTk4o3MzyzYOS4c8HQj+jnGeR5Q3ETuyz4psLyOfuBtIrfOeuxV42rFDmkYM3IrrRR+F9oklFG6Ig8DVfgQEzSG36NkgvpF4OdFvigYqXvw== cloud@vgcn"
+nspawn_ssh_host_trust_container: yes

hosts

@@ -78,3 +78,9 @@ ansible_ssh_user=centos
 
 [central-manager]
 manager.vgcn.galaxyproject.eu ansible_ssh_user=root
+
+[central-manager-secondary]
+manager-secondary.galaxyproject.eu ansible_host=127.0.0.1 ansible_port=2222 ansible_ssh_user=root ansible_ssh_common_args='-o HostKeyAlias=manager-secondary.galaxyproject.eu -o ProxyCommand="ssh -W %h:%p -q [email protected]"'
+
+[central-manager-secondary-host]
+sn06.galaxyproject.eu ansible_ssh_user=root

htcondor-secondary.yml

@@ -0,0 +1,181 @@
+---
+- name: Systemd-nspawn container aimed at running a second HTCondor installation.
+  hosts: central-manager-secondary-host
+  handlers:
+    - name: Reload sshd  # (in the container)
+      when: nspawn_ssh | default(no)
+      become: true
+      ansible.builtin.shell:
+        executable: /bin/bash
+        cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl reload sshd"
+      changed_when: true
+  vars_files:
+    - mounts/mountpoints.yml
+    - secret_group_vars/central-manager-secondary-host.yml
+  pre_tasks:
+    # Because it is already disabled for sn06 and this setup is needed just
+    # temporarily.
+    - name: Disable SELinux.
+      become: true
+      ansible.posix.selinux:
+        state: disabled
+  roles:
+    - kysrpex.systemd_nspawn
+  post_tasks:
+    - name: Configure the container's sshd.
+      when: nspawn_ssh and (nspawn_ssh_config is defined or nspawn_ssh_config_path is defined)
+      become: true
+      block:
+        - name: Get the location of the container image.
+          ansible.builtin.shell:
+            executable: /bin/bash
+            cmd: |
+              set -o pipefail
+              machinectl image-status htcondor | grep "Path: " | awk '{$1=$1};1' | cut -d' ' -f2
+          register: nspawn_image
+          changed_when: false
+
+        - name: Write the sshd configuration to sshd_config.
+          ansible.builtin.lineinfile:
+            path: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}"
+            regexp: '^\s*(?:#)?\s*{{ item.key | regex_escape() }}'
+            line: "{{ item.key }} {{ item.value }}"
+          with_dict: "{{ nspawn_ssh_config }}"
+          notify: Reload sshd
+
+        - name: Replace ssh host keys (private).
+          when: nspawn_ssh_host_keys is defined and nspawn_ssh_host_keys_private is defined
+          ansible.builtin.copy:
+            dest: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2') | dirname, 'ssh_host_' + item.key + '_key') | path_join }}"
+            content: "{{ item.value }}"
+            mode: "0600"
+          with_dict: "{{ nspawn_ssh_host_keys_private }}"
+          notify: Reload sshd
+
+        - name: Replace ssh host keys (public).
+          when: nspawn_ssh_host_keys is defined and nspawn_ssh_host_keys_private is defined
+          ansible.builtin.copy:
+            dest: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2') | dirname, 'ssh_host_' + item.key + '_key.pub') | path_join }}"
+            content: "{{ item.value }}"
+            mode: "0644"
+          with_dict: "{{ nspawn_ssh_host_keys }}"
+          notify: Reload sshd
+
+        - name: Replace ssh host keys (certs).
+          when: nspawn_ssh_host_keys is defined and nspawn_ssh_host_keys_private is defined and nspawn_ssh_host_certs is defined
+          ansible.builtin.copy:
+            dest: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2') | dirname, 'ssh_host_' + item.key + '_key-cert.pub') | path_join }}"
+            content: "{{ item.value }}"
+            mode: "0644"
+          with_dict: "{{ nspawn_ssh_host_certs }}"
+          notify: Reload sshd
+
+        - name: Add certs to sshd_config.
+          when: nspawn_ssh_host_keys is defined and nspawn_ssh_host_keys_private is defined and nspawn_ssh_host_certs is defined
+          ansible.builtin.lineinfile:
+            path: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}"
+            regexp: '^\s*(?:#)?\s*HostCertificate\s+.*ssh_host_{{ item.key }}_key-cert.pub'
+            line: "HostCertificate {{ nspawn_ssh_config_path | dirname }}/ssh_host_{{ item.key }}_key-cert.pub"
+          with_dict: "{{ nspawn_ssh_host_certs }}"
+          notify: Reload sshd
+
+        - name: Ensure the ssh configuration directory exists (for root).
+          when: nspawn_ssh_authorized_keys is defined
+          ansible.builtin.file:
+            path: "{{ (nspawn_image.stdout, '/root/.ssh' | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}"
+            state: directory
+            owner: root
+            group: root
+            mode: "0700"
+
+        - name: Ensure the authorized_keys file exists (for root).
+          when: nspawn_ssh_authorized_keys is defined
+          ansible.builtin.file:
+            path: "{{ (nspawn_image.stdout, '/root/.ssh/authorized_keys' | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}"
+            state: touch
+            owner: root
+            group: root
+            mode: "0600"
+
+        - name: Authorize specific users log-in as root.
+          when: nspawn_ssh_authorized_keys is defined
+          ansible.builtin.lineinfile:
+            path: "{{ (nspawn_image.stdout, '/root/.ssh/authorized_keys' | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}"
+            regexp: '^\s*(?:#)?\s*{{ item | regex_escape() }}'
+            line: "{{ item }}"
+          loop: "{{ nspawn_ssh_authorized_keys }}"
+
+    - name: Enable and start the container.
+      become: true
+      block:
+        - name: Enable the container.
+          ansible.builtin.shell:
+            executable: /bin/bash
+            cmd: "machinectl enable {{ nspawn_name }}"
+          register: nspawn_container_enable
+          changed_when: nspawn_container_enable.rc == 0 and nspawn_container_enable.stderr != ''
+
+        - name: Check if the container is already running.
+          ansible.builtin.shell:
+            executable: /bin/bash
+            cmd: "machinectl show {{ nspawn_name }} -p State --value"
+          register: nspawn_status
+          changed_when: false
+          failed_when: false
+
+        - name: Start the container.
+          ansible.builtin.shell:
+            executable: /bin/bash
+            cmd: "machinectl start {{ nspawn_name }}"
+          register: nspawn_container_enable
+          changed_when: nspawn_status.stdout != 'running'
+
+    - name: Enable and start sshd in the container.
+      when: nspawn_ssh | default(no)
+      become: true
+      block:
+        - name: Check if sshd is enabled in the container.
+          ansible.builtin.shell:
+            executable: /bin/bash
+            cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl is-enabled sshd"
+          register: nspawn_ssh_enabled
+          changed_when: false
+          failed_when: false
+
+        - name: Enable sshd in the container.
+          become: true
+          ansible.builtin.shell:
+            executable: /bin/bash
+            cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl enable sshd"
+          changed_when: nspawn_ssh_enabled.rc != 0
+
+        - name: Check if sshd is active in the container.
+          ansible.builtin.shell:
+            executable: /bin/bash
+            cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl is-active sshd"
+          register: nspawn_ssh_active
+          changed_when: false
+          failed_when: false
+
+        - name: Start sshd in the container.
+          become: true
+          ansible.builtin.shell:
+            executable: /bin/bash
+            cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl start sshd"
+          changed_when: nspawn_ssh_active.rc != 0
+
+    - name: Read the container's host key.
+      become: true
+      ansible.builtin.slurp:
+        src: "{{ (nspawn_image.stdout, '/etc/ssh/ssh_host_ecdsa_key.pub' | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}"
+      register: nspawn_ssh_host_key
+      when: nspawn_ssh_host_trust_container
+
+    - name: Trust the container's host key.
+      ansible.builtin.known_hosts:
+        name: "[127.0.0.1]:{{ nspawn_ssh_config.Port }}"
+        key: "[127.0.0.1]:{{ nspawn_ssh_config.Port }} {{ nspawn_ssh_host_key.content | b64decode }}"
+      when: nspawn_ssh_host_trust_container
+
+- name: Secondary HTCondor 10 cluster.
+  hosts: central-manager-secondary