From ce8df70e87dbbfce39d9c2599c85f7c62e1805fb Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Fri, 15 Mar 2024 10:50:53 +0000 Subject: [PATCH 1/2] Unify HTCondor secondary with HTCondor and cleanup HTCondor secondary configs --- group_vars/htcondor-secondary-submit-host.yml | 86 -------- group_vars/htcondor-secondary-submit.yml | 12 - group_vars/htcondor-secondary/vars.yml | 8 - group_vars/htcondor-secondary/vault.yml | 10 - group_vars/htcondor-submit.yml | 13 ++ group_vars/htcondor/vars.yml | 30 +-- group_vars/htcondor/vault.yml | 18 +- .../nspawn-htcondor.sn06.galaxyproject.eu.yml | 2 - host_vars/sn06.galaxyproject.eu.yml | 12 +- hosts | 40 +--- htcondor.yml | 208 +----------------- sn06.yml | 1 + templates/htcondor/condor_config.local.j2 | 6 +- 13 files changed, 41 insertions(+), 405 deletions(-) delete mode 100644 group_vars/htcondor-secondary-submit-host.yml delete mode 100644 group_vars/htcondor-secondary-submit.yml delete mode 100644 group_vars/htcondor-secondary/vars.yml delete mode 100644 group_vars/htcondor-secondary/vault.yml delete mode 100644 host_vars/nspawn-htcondor.sn06.galaxyproject.eu.yml diff --git a/group_vars/htcondor-secondary-submit-host.yml b/group_vars/htcondor-secondary-submit-host.yml deleted file mode 100644 index c1664fe71..000000000 --- a/group_vars/htcondor-secondary-submit-host.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- -nspawn_name: htcondor -nspawn_distro: "rocky" -nspawn_release: "8" -nspawn_packages: - - dhcp-client - - dnf - - glibc-langpack-en - - iproute - - iputils - - less - - passwd - - systemd - - dbus - - vim-minimal - - openssh-server - -nspawn_config: | - [Files] - {% for mount in jwd.values() if mount.name != "birna01" %} - Bind={{ mount.path }} - {% endfor %} - Bind=/data/dnb01/maintenance - Bind={{ galaxy_log_dir }} - BindReadOnly={{ galaxy_config_dir }} - - [Exec] - NotifyReady=yes - PrivateUsers=no - - [Network] - VirtualEthernet=no - -nspawn_galaxy_environment_file: "{{ galaxy_user.home }}/env" -nspawn_galaxy_environment_vars: | - HOME={{ galaxy_root }} - VIRTUAL_ENV={{ galaxy_venv_dir }} - PATH={{ galaxy_venv_dir }}/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin - DOCUTILSCONFIG='' - PYTHONPATH={{ galaxy_server_dir }}/lib/galaxy/jobs/rules - LANG=en_US.UTF-8 - {% for var in galaxy_systemd_handler_env | split %} - {{ var }} - {% endfor %} - -nspawn_enable: false -nspawn_start: false - -nspawn_ssh: yes -nspawn_ssh_config_path: /etc/ssh/sshd_config -nspawn_ssh_config: - Port: "2222" - ListenAddress: "127.0.0.1" - PermitRootLogin: "yes" - PubkeyAuthentication: "yes" - AuthenticationMethods: "publickey" - PasswordAuthentication: "no" - PermitEmptyPasswords: "no" - ChallengeResponseAuthentication: "no" - PermitTunnel: "no" - AllowTcpForwarding: "no" - AllowAgentForwarding: "no" - GatewayPorts: "no" -nspawn_ssh_host_keys: - ecdsa: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF9MMtDEfPPSaHSNUFz3b1BBtasjGT1XB2my45Lhtp4NLJv/jPxpVQ8sxRSS3+8fc9+1EZXF0AGj4D1NjqpabmQ=" - ed25519: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHMTH60CkuT9moO66XVEaUP1YelPv/aOQdqSImNNiQFh" - rsa: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDxIgnu5/BIIzVbWM442uzwyuXPQx0jqaVfffjexwewfc2mMNnVAG7mRzcWcgr+o/lFiogkqGulkvO/a+EARs5WsWrsq5bGFcHx9DPgaT+7smu+K3t0WNiqPEykXUJppyMtaYftZSWaCSl51pg3QT29pvl95sjC3URFrQhoaLHJ2fESYJI4WuMBig/rCLaRsF6xaR38LIVfrZdA7UjG5Ncr+FXF6cpNtdvDvJXkGyuYft2p9LMh7vycdiSUL/CluC6tptgQ82JGxgyHDaxwJvyzzDukjKTUkR59ctuGthq9M/M384sCn8Z3PsGE4hkRGdL/HJZhasPIkkwJK8ekucnBR2LzlCqiTrn7HFbndqgePhJWHq0tBN53o2Vmczrbq6UVROAFDCpPKeaWa6hcdqzp0fpHrgSZEyKmMxThV8sj69E/mCh2PEgWjCXsPoJLwrW9TuPUMeXx2+Cwf4/JaFSvLBT19ADFKkzgAPeEVdJg2ZkjWZVO+xxb9e6Ahux+g7E=" -nspawn_ssh_host_certs: - # valid from 2023-10-12T14:46:00 to 2033-09-29T14:47:45 - ecdsa: "ecdsa-sha2-nistp256-cert-v01@openssh.com AAAAKGVjZHNhLXNoYTItbmlzdHAyNTYtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgIkfzNxdybaCbJ+rziM3zQy/grFSitfDOJWQS2zH4I4MAAAAIbmlzdHAyNTYAAABBBF9MMtDEfPPSaHSNUFz3b1BBtasjGT1XB2my45Lhtp4NLJv/jPxpVQ8sxRSS3+8fc9+1EZXF0AGj4D1NjqpabmQAAAAAAAAAAAAAAAIAAAAXbWFuYWdlci1zZWNvbmRhcnktZWNkc2EAAAAAAAAAAGUoBqgAAAAAd+bbEQAAAAAAAAAAAAAAAAAAARcAAAAHc3NoLXJzYQAAAAMBAAEAAAEBAMtAPp8bfy7AWP3UZJ7pidGcG5ONdZk3o9yTUlEJNa25/ApXRJk1zMP6pR2cV9msLH/AFUrxHrMvZVZL2FwUrUu4Twcrzp/yob+BwHrIW3/Qj89ahKNxtQw9BPMUjLzLG90ig9OWP8/r642DB5O8wUjj5Q64IjvUiEH+WJMefX/Z1dAz3hIV3hVB6D7dNzqAPESpFJLp0sirS69Hz/+W4imhbfzKHajPAeaMwpO8edt24aoeURLirP1xT2JmRag/NKn6qjEI6KMe/DMvm8PT+TiFn+InJPMMdAsOtrcpprzKRZ0RELGOkynXL11VDbeVXGn5RQC3dEc4JTX8q40z0+sAAAEUAAAADHJzYS1zaGEyLTUxMgAAAQBbd3PNc1Ezs5BDCF/Ca02Oea4hL/L3Z1G4PO1W7y3IHAVW2hFmHBmJTd2r5tO/G+n/o03j+wr59ZfKYr4sj9c99oMXA19IRl+3a7ECnG6xpnepVA8FMo8qatcTyeOlECDbS8VI8Kc+cBuSu4UZjdCIZCFGzIj9dupK22id7D/UdeqDNTZ0b+KB0NOQM6lzyLaXD75U9kOFFzWmJqRYtkFns71dhzzr1i04V+ZaPlmSz2QZ20WIqt2hX6ecU+w3U2fK0PkFy/cy3YtSzhnNRiKnmJk1U4MOCbxWQ3rFOTopX514elUZAUCw3HCU+vBIOwwDIrVlbNuCEecuMtQUfLyW /etc/ssh/ssh_host_ecdsa_key.pub" - # valid from 2023-10-12T14:46:00 to 2033-09-29T14:47:55 - ed25519: "ssh-ed25519-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQtdjAxQG9wZW5zc2guY29tAAAAILMcMkZV0MfVGEZghy0DD2dOMTtpjE5z0Hj6Cus/C8mjAAAAIHMTH60CkuT9moO66XVEaUP1YelPv/aOQdqSImNNiQFhAAAAAAAAAAAAAAACAAAAGW1hbmFnZXItc2Vjb25kYXJ5LWVjMjU1MTkAAAAAAAAAAGUoBqgAAAAAd+bbGwAAAAAAAAAAAAAAAAAAARcAAAAHc3NoLXJzYQAAAAMBAAEAAAEBAMtAPp8bfy7AWP3UZJ7pidGcG5ONdZk3o9yTUlEJNa25/ApXRJk1zMP6pR2cV9msLH/AFUrxHrMvZVZL2FwUrUu4Twcrzp/yob+BwHrIW3/Qj89ahKNxtQw9BPMUjLzLG90ig9OWP8/r642DB5O8wUjj5Q64IjvUiEH+WJMefX/Z1dAz3hIV3hVB6D7dNzqAPESpFJLp0sirS69Hz/+W4imhbfzKHajPAeaMwpO8edt24aoeURLirP1xT2JmRag/NKn6qjEI6KMe/DMvm8PT+TiFn+InJPMMdAsOtrcpprzKRZ0RELGOkynXL11VDbeVXGn5RQC3dEc4JTX8q40z0+sAAAEUAAAADHJzYS1zaGEyLTUxMgAAAQAno9IlnZ6TLhK83xx1+qjTL2+DjYsFTVeH0D97P7QdvZ3l4VnOWJH62V79VSNA4VQ7MRgoUZkeVY6Y2OBBItGGNxn0Z/9O/yE8csoqKE0L0LJNWoEU8Xbw/rs7rr4gIUqTNqzT42dSt7p6chht8+9iCSku8bKHmmnokQOnC/tF29zC5Iq4FQoVYDMgm0YO2MN9mJ6R9zEzkcu/cPWTjPM0FY9qs/RlK7yGQXcINAxaW4a5xdlMp85LWygGRLBn+gKiCwUATW1EPx7MkuHUx+vHzeKwRDsc904KxYD9TWsrSygFAfesLY2Xh6TDVdrPCnw096K7l9EcovgSCVWPV1mM /etc/ssh/ssh_host_ed25519_key.pub" - # valid from 2023-10-12T14:47:00 to 2033-09-29T14:48:11 - rsa: "ssh-rsa-cert-v01@openssh.com AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgQuoZdMdY7Boli6KRlNmUk2zQar66du6kmLR4PkWoHI8AAAADAQABAAABgQDxIgnu5/BIIzVbWM442uzwyuXPQx0jqaVfffjexwewfc2mMNnVAG7mRzcWcgr+o/lFiogkqGulkvO/a+EARs5WsWrsq5bGFcHx9DPgaT+7smu+K3t0WNiqPEykXUJppyMtaYftZSWaCSl51pg3QT29pvl95sjC3URFrQhoaLHJ2fESYJI4WuMBig/rCLaRsF6xaR38LIVfrZdA7UjG5Ncr+FXF6cpNtdvDvJXkGyuYft2p9LMh7vycdiSUL/CluC6tptgQ82JGxgyHDaxwJvyzzDukjKTUkR59ctuGthq9M/M384sCn8Z3PsGE4hkRGdL/HJZhasPIkkwJK8ekucnBR2LzlCqiTrn7HFbndqgePhJWHq0tBN53o2Vmczrbq6UVROAFDCpPKeaWa6hcdqzp0fpHrgSZEyKmMxThV8sj69E/mCh2PEgWjCXsPoJLwrW9TuPUMeXx2+Cwf4/JaFSvLBT19ADFKkzgAPeEVdJg2ZkjWZVO+xxb9e6Ahux+g7EAAAAAAAAAAAAAAAIAAAAVbWFuYWdlci1zZWNvbmRhcnktcnNhAAAAAAAAAABlKAbkAAAAAHfm2ysAAAAAAAAAAAAAAAAAAAEXAAAAB3NzaC1yc2EAAAADAQABAAABAQDLQD6fG38uwFj91GSe6YnRnBuTjXWZN6Pck1JRCTWtufwKV0SZNczD+qUdnFfZrCx/wBVK8R6zL2VWS9hcFK1LuE8HK86f8qG/gcB6yFt/0I/PWoSjcbUMPQTzFIy8yxvdIoPTlj/P6+uNgweTvMFI4+UOuCI71IhB/liTHn1/2dXQM94SFd4VQeg+3Tc6gDxEqRSS6dLIq0uvR8//luIpoW38yh2ozwHmjMKTvHnbduGqHlES4qz9cU9iZkWoPzSp+qoxCOijHvwzL5vD0/k4hZ/iJyTzDHQLDra3Kaa8ykWdERCxjpMp1y9dVQ23lVxp+UUAt3RHOCU1/KuNM9PrAAABFAAAAAxyc2Etc2hhMi01MTIAAAEAOXRiZFs14HScCvi+9Hp51z310t0r3SHtHhE50vEjdAsnej+/yHUtlmaYJ1l4B8SWYHHUV94KxZbQP79D8PYQextXoYldA4Vp29ow33sYF8Mr4p7Jbjyyr4GVP9RAymiZz9cuiiRqjs8qwn/UCOw4A6yaix4k0EYTJa9bpbD3MsCxJbKEdZxxL72kIKf4sGuDrd77hEtRwJFUSqz+XD96TYG+BIomnrk35Oy+oyMGhoIlSM0//cS0pHUW8fpGnRIJJbBjDETwOmaMEF5yynRSeJyWnEBPpjcA+1i7wILSxYxZb5ckGxCABYxNvlIhF8Bvdcyb4eo7B/WeEdzr5C+Pzg== /etc/ssh/ssh_host_rsa_key.pub" -nspawn_ssh_authorized_keys: - - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDV7gfNbNN5O8vH6/tM/iOFXKBP2YKRHXOmdfV8ogvu9BdVV0IPmDzk2EooVpThDE1VMv1hz3811tvBhHRJ6IgNhVIV/61w/+RazQD/AU27X8bX+Hb9EQ/bP4DW+6ySd/z5vdDLzpH5dbiMhzPEDkXVsylUT+hkQnas6cHspDhHmtKQ5MWOgDe3D/IEudTDJQe8hxxaU4TaZUmFzn7eYp9HvuK8qW0yCy4NWOxJJHA+G5wSCyLuKnaKo4AitUIzSKF1AB94oq7b96KONhPxgRptAk4OYIUTdNFbrI5HDaSNzHLnF5FbjQvG+Eu6m5nY5yvJMogE+jiuWeIXCZTCFljg287FUo0ohmbZpd802L6VXun14VumRC+rRgPrvBALo/CsyCsPIoBSTKhVElxKVOcRjmTLNfrUZM0GQxqJhIvah8BV+JTExkipPwkrKTdMAWIXvCoehxV+WMpBWqtEEzAzEoqJpaiec7HfriwsHTGESZWAPYEbFjzbHXQZtqBkbOvtokPMRmTWfWKxaplCMN6ddJeeY6faorD0w/e6lszWES1Q1ieajiPKDy37UvybKKvPTk4o3MzyzYOS4c8HQj+jnGeR5Q3ETuyz4psLyOfuBtIrfOeuxV42rFDmkYM3IrrRR+F9oklFG6Ig8DVfgQEzSG36NkgvpF4OdFvigYqXvw== cloud@vgcn" -nspawn_ssh_host_trust_container: yes - -nspawn_condor_systemd_run: "/usr/bin/systemd-run -p EnvironmentFile={{ nspawn_galaxy_environment_file }} --uid={{ galaxy_user.uid }} --gid={{ galaxy_group.gid }} --pipe --quiet --machine {{ nspawn_name }}" -nspawn_condor_rm_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_rm" -nspawn_condor_ssh_to_job_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_ssh_to_job" -nspawn_condor_submit_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_submit" - -ssh_allow_tcp_forwarding: "local" -sshd_custom_options: - - "PermitOpen 127.0.0.1:{{ nspawn_ssh_config.Port }}" diff --git a/group_vars/htcondor-secondary-submit.yml b/group_vars/htcondor-secondary-submit.yml deleted file mode 100644 index 5de34cd9b..000000000 --- a/group_vars/htcondor-secondary-submit.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Configure HTCondor submit nodes (secondary cluster only). ---- -galaxy_root: /opt/galaxy -galaxy_venv_dir: "{{ galaxy_root }}/venv" -galaxy_server_dir: "{{ galaxy_root }}/server" -galaxy_config_dir: "{{ galaxy_root }}/config" -galaxy_config_file: "{{ galaxy_config_dir }}/galaxy.yml" -galaxy_mutable_config_dir: "{{ galaxy_root }}/mutable-config" -galaxy_log_dir: "/var/log/galaxy" -galaxy_config: - galaxy: - job_working_directory: /data/jwd04/main diff --git a/group_vars/htcondor-secondary/vars.yml b/group_vars/htcondor-secondary/vars.yml deleted file mode 100644 index 76b5c7a08..000000000 --- a/group_vars/htcondor-secondary/vars.yml +++ /dev/null @@ -1,8 +0,0 @@ -# Configure nodes in the secondary HTCondor cluster. -# -# Nodes in the secondary HTCondor cluster belong both to the -# "htcondor-secondary" (with group priority > 1) and "htcondor" groups. They -# thus inherit variables from the latter. ---- -htcondor_server: "build.galaxyproject.eu" -htcondor_port: 9628 diff --git a/group_vars/htcondor-secondary/vault.yml b/group_vars/htcondor-secondary/vault.yml deleted file mode 100644 index 728b73381..000000000 --- a/group_vars/htcondor-secondary/vault.yml +++ /dev/null @@ -1,10 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -31353533313831356632376636636564653732313930623263376437313362386632623732306136 -3465326632326138646330353164336363653764396237370a393562613834343765313835656362 -66633030353534663831323939386335316130343137396139633038366438613731376130663564 -6635643366613463390a663637643834366632643730666131323737633966393335343734663731 -63346138623034333265633465376633313537313062633633353261623934333037646532303132 -63643364633136613265333461623036313964383932336335623236623462316437303964346163 -32386236303765353936333563303934323964383039626233613333396431383936326530343931 -33636531343831663864373365613036333964343534616664356462383066623238326138373435 -3566 diff --git a/group_vars/htcondor-submit.yml b/group_vars/htcondor-submit.yml index 625f0012d..5479471b9 100644 --- a/group_vars/htcondor-submit.yml +++ b/group_vars/htcondor-submit.yml @@ -2,6 +2,19 @@ --- htcondor_role_submit: true +# Role: hxr.postgres-connection postgres_user: galaxy postgres_host: sn05.galaxyproject.eu postgres_port: 5432 + +# MISC +galaxy_root: /opt/galaxy +galaxy_venv_dir: "{{ galaxy_root }}/venv" +galaxy_server_dir: "{{ galaxy_root }}/server" +galaxy_config_dir: "{{ galaxy_root }}/config" +galaxy_config_file: "{{ galaxy_config_dir }}/galaxy.yml" +galaxy_mutable_config_dir: "{{ galaxy_root }}/mutable-config" +galaxy_log_dir: "/var/log/galaxy" +galaxy_config: + galaxy: + job_working_directory: /data/jwd04/main diff --git a/group_vars/htcondor/vars.yml b/group_vars/htcondor/vars.yml index 2a7a3f102..61631a1e4 100644 --- a/group_vars/htcondor/vars.yml +++ b/group_vars/htcondor/vars.yml @@ -1,11 +1,12 @@ # Configure nodes in the HTCondor cluster. --- -htcondor_server: "condor-cm.galaxyproject.eu" +htcondor_server: "build.galaxyproject.eu" htcondor_domain: bi.uni-freiburg.de -htcondor_port: 9618 +htcondor_server_port: 9628 +htcondor_shared_port: 9628 htcondor_version: 23.0 htcondor_channel: 23.0 -htcondor_firewall_condor: "{{ true if htcondor_port == 9618 else false }}" +htcondor_firewall_condor: false htcondor_firewall_nfs: false htcondor_role_execute: false htcondor_role_manager: false @@ -30,29 +31,6 @@ htcondor_job_start_delay: 0 htcondor_claim_worklife: 120 htcondor_negotiator_post_job_rank: "isUndefined(RemoteOwner) * (10000 - TotalLoadAvg)" -# Settings specific to the `usegalaxy_eu.htcondor` role (to be replaced with -# `grycap.htcondor`). -condor_host: "{{ htcondor_server }}" -condor_fs_domain: "{{ htcondor_domain }}" -condor_uid_domain: "{{ htcondor_domain }}" -condor_allow_write: "{{ htcondor_allow_write }}" -# condor_daemons -> Defined per-host in host_vars. -condor_allow_negotiator: "{{ htcondor_allow_negotiator }}" -condor_allow_administrator: "{{ htcondor_allow_administrator }}" -condor_system_periodic_hold: "{{ htcondor_system_periodic_hold }}" -condor_system_periodic_remove: "{{ htcondor_system_periodic_remove }}" -condor_network_interface: "{{ htcondor_network_interface }}" -condor_extra: | - MASTER_UPDATE_INTERVAL = {{ htcondor_master_update_interval }} - CLASSAD_LIFETIME = {{ htcondor_classad_lifetime }} - NEGOTIATOR_INTERVAL = {{ htcondor_negotiator_interval }} - NEGOTIATOR_UPDATE_INTERVAL = {{ htcondor_negotiator_update_interval }} - SCHEDD_INTERVAL = {{ htcondor_schedd_interval }} - JOB_START_COUNT = {{ htcondor_job_start_count }} - JOB_START_DELAY = {{ htcondor_job_start_delay }} - CLAIM_WORKLIFE = {{ htcondor_claim_worklife }} - NEGOTIATOR_POST_JOB_RANK = {{ htcondor_negotiator_post_job_rank }} - # Configuration of `usegalaxy_eu.handy.os_setup`. enable_create_user: true enable_remap_user: true diff --git a/group_vars/htcondor/vault.yml b/group_vars/htcondor/vault.yml index af51099b9..728b73381 100644 --- a/group_vars/htcondor/vault.yml +++ b/group_vars/htcondor/vault.yml @@ -1,10 +1,10 @@ $ANSIBLE_VAULT;1.1;AES256 -36336166336332656436376537343036353234366164616236393139313932343538313133373639 -3064333637333539353566396361666362666539353231360a646430356366343632633637326462 -39333232646363656438316533666664613935353336313064323038313564383734373433656330 -3161396636623764660a636332303565396630666134626235636363636434623537333933653537 -37383165643433633630353961623930653139653132303235306539613332346662323764356563 -65303062333738616266383339366165643264633038323533306365623034656563333731393465 -66386263353433303832363936323138386637636366663338336263323835663730616639393831 -32333161633131323534306565626530616364386261646439336436303834386265396161333133 -3130 +31353533313831356632376636636564653732313930623263376437313362386632623732306136 +3465326632326138646330353164336363653764396237370a393562613834343765313835656362 +66633030353534663831323939386335316130343137396139633038366438613731376130663564 +6635643366613463390a663637643834366632643730666131323737633966393335343734663731 +63346138623034333265633465376633313537313062633633353261623934333037646532303132 +63643364633136613265333461623036313964383932336335623236623462316437303964346163 +32386236303765353936333563303934323964383039626233613333396431383936326530343931 +33636531343831663864373365613036333964343534616664356462383066623238326138373435 +3566 diff --git a/host_vars/nspawn-htcondor.sn06.galaxyproject.eu.yml b/host_vars/nspawn-htcondor.sn06.galaxyproject.eu.yml deleted file mode 100644 index d32f2b473..000000000 --- a/host_vars/nspawn-htcondor.sn06.galaxyproject.eu.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -htcondor_network_interface: ens802f0.223 diff --git a/host_vars/sn06.galaxyproject.eu.yml b/host_vars/sn06.galaxyproject.eu.yml index ef317b889..b28263ca1 100644 --- a/host_vars/sn06.galaxyproject.eu.yml +++ b/host_vars/sn06.galaxyproject.eu.yml @@ -1,9 +1,9 @@ --- htcondor_network_interface: ens802f0.223 -# Settings specific to the `usegalaxy_eu.htcondor` role. -condor_daemons: - - COLLECTOR - - NEGOTIATOR - - MASTER - - SCHEDD +# 15/03/2024: On sn06 HTCondor conf was manually adjusted to use port 9618 since the +# HTCondor container was using 9628. Changing this now will require a restart of the +# HTCondor service on sn06. So this needs to be combined with a maintenance window in the +# future. Rest of the schedulers are using 9628 including the manager. +# Adding it to the host_vars for the dedicated host sn06 thus it has the higher precedence. +htcondor_shared_port: 9618 diff --git a/hosts b/hosts index f97bfcb92..554b2deb6 100644 --- a/hosts +++ b/hosts @@ -79,48 +79,16 @@ ansible_ssh_user=centos [htcondor:children] htcondor-manager htcondor-submit -htcondor-secondary [htcondor-manager] -sn06.galaxyproject.eu - -[htcondor-manager:children] -htcondor-secondary-manager - -[htcondor-manager:vars] -ansible_group_priority=2 - -[htcondor-submit] -sn06.galaxyproject.eu - -[htcondor-submit:children] -htcondor-secondary-submit - -[htcondor-submit:vars] -ansible_group_priority=2 - -[htcondor-secondary:children] -htcondor-secondary-manager -htcondor-secondary-submit - -[htcondor-secondary:vars] -ansible_group_priority=3 - -[htcondor-secondary-manager] build.galaxyproject.eu ansible_ssh_user=root -[htcondor-secondary-manager:vars] +[htcondor-manager:vars] ansible_group_priority=4 -[htcondor-secondary-submit] -nspawn-htcondor.sn06.galaxyproject.eu ansible_host=127.0.0.1 ansible_port=2222 ansible_ssh_user=root ansible_ssh_common_args='-o HostKeyAlias=nspawn-htcondor.sn06.galaxyproject.eu -o ProxyCommand="ssh -W %h:%p -q centos@sn06.galaxyproject.eu"' +[htcondor-submit] maintenance.galaxyproject.eu - -[htcondor-secondary-submit:vars] -ansible_group_priority=4 - -[htcondor-secondary-submit-host] sn06.galaxyproject.eu -[htcondor-secondary-submit-host:vars] -ansible_group_priority=2 +[htcondor-submit:vars] +ansible_group_priority=4 \ No newline at end of file diff --git a/htcondor.yml b/htcondor.yml index dbf3dd89b..c6027650d 100644 --- a/htcondor.yml +++ b/htcondor.yml @@ -1,210 +1,6 @@ --- -- name: Create a systemd-nspawn container aimed at running a second HTCondor installation. - hosts: htcondor-secondary-submit-host - handlers: - - name: Reload sshd # (in the container) - when: nspawn_ssh | default(no) - become: true - ansible.builtin.shell: - executable: /bin/bash - cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl reload sshd" - changed_when: true - vars_files: - - mounts/mountpoints.yml - - secret_group_vars/all.yml - - secret_group_vars/htcondor-secondary-submit-host.yml - pre_tasks: - # Because it is already disabled for sn06 and this setup is needed just - # temporarily. - - name: Disable SELinux. - become: true - ansible.posix.selinux: - state: disabled - roles: - - kysrpex.systemd_nspawn - post_tasks: - - name: Get the location of the container image. - ansible.builtin.shell: - executable: /bin/bash - cmd: | - set -o pipefail - machinectl image-status htcondor | grep "Path: " | awk '{$1=$1};1' | cut -d' ' -f2 - register: nspawn_image - changed_when: false - - - name: Configure the container's sshd. - when: nspawn_ssh and (nspawn_ssh_config is defined or nspawn_ssh_config_path is defined) - become: true - block: - - name: Write the sshd configuration to sshd_config. - ansible.builtin.lineinfile: - path: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" - regexp: '^\s*(?:#)?\s*{{ item.key | regex_escape() }}' - line: "{{ item.key }} {{ item.value }}" - with_dict: "{{ nspawn_ssh_config }}" - notify: Reload sshd - - - name: Replace ssh host keys (private). - when: nspawn_ssh_host_keys is defined and nspawn_ssh_host_keys_private is defined - ansible.builtin.copy: - dest: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2') | dirname, 'ssh_host_' + item.key + '_key') | path_join }}" - content: "{{ item.value }}" - mode: "0600" - with_dict: "{{ nspawn_ssh_host_keys_private }}" - notify: Reload sshd - - - name: Replace ssh host keys (public). - when: nspawn_ssh_host_keys is defined and nspawn_ssh_host_keys_private is defined - ansible.builtin.copy: - dest: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2') | dirname, 'ssh_host_' + item.key + '_key.pub') | path_join }}" - content: "{{ item.value }}" - mode: "0644" - with_dict: "{{ nspawn_ssh_host_keys }}" - notify: Reload sshd - - - name: Replace ssh host keys (certs). - when: nspawn_ssh_host_keys is defined and nspawn_ssh_host_keys_private is defined and nspawn_ssh_host_certs is defined - ansible.builtin.copy: - dest: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2') | dirname, 'ssh_host_' + item.key + '_key-cert.pub') | path_join }}" - content: "{{ item.value }}" - mode: "0644" - with_dict: "{{ nspawn_ssh_host_certs }}" - notify: Reload sshd - - - name: Add certs to sshd_config. - when: nspawn_ssh_host_keys is defined and nspawn_ssh_host_keys_private is defined and nspawn_ssh_host_certs is defined - ansible.builtin.lineinfile: - path: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" - regexp: '^\s*(?:#)?\s*HostCertificate\s+.*ssh_host_{{ item.key }}_key-cert.pub' - line: "HostCertificate {{ nspawn_ssh_config_path | dirname }}/ssh_host_{{ item.key }}_key-cert.pub" - with_dict: "{{ nspawn_ssh_host_certs }}" - notify: Reload sshd - - - name: Ensure the ssh configuration directory exists (for root). - when: nspawn_ssh_authorized_keys is defined - ansible.builtin.file: - path: "{{ (nspawn_image.stdout, '/root/.ssh' | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" - state: directory - owner: root - group: root - mode: "0700" - - - name: Ensure the authorized_keys file exists (for root). - when: nspawn_ssh_authorized_keys is defined - ansible.builtin.file: - path: "{{ (nspawn_image.stdout, '/root/.ssh/authorized_keys' | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" - state: touch - owner: root - group: root - mode: "0600" - - - name: Authorize specific users log-in as root. - when: nspawn_ssh_authorized_keys is defined - ansible.builtin.lineinfile: - path: "{{ (nspawn_image.stdout, '/root/.ssh/authorized_keys' | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" - regexp: '^\s*(?:#)?\s*{{ item | regex_escape() }}' - line: "{{ item }}" - loop: "{{ nspawn_ssh_authorized_keys }}" - - - name: Enable and start the container. - become: true - block: - - name: Enable the container. - ansible.builtin.shell: - executable: /bin/bash - cmd: "machinectl enable {{ nspawn_name }}" - register: nspawn_container_enable - changed_when: nspawn_container_enable.rc == 0 and nspawn_container_enable.stderr != '' - - - name: Check if the container is already running. - ansible.builtin.shell: - executable: /bin/bash - cmd: "machinectl show {{ nspawn_name }} -p State --value" - register: nspawn_status - changed_when: false - failed_when: false - - - name: Start the container. - ansible.builtin.shell: - executable: /bin/bash - cmd: "machinectl start {{ nspawn_name }}" - register: nspawn_container_enable - changed_when: nspawn_status.stdout != 'running' - - - name: Enable and start sshd in the container. - when: nspawn_ssh | default(no) - become: true - block: - - name: Check if sshd is enabled in the container. - ansible.builtin.shell: - executable: /bin/bash - cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl is-enabled sshd" - register: nspawn_ssh_enabled - changed_when: false - failed_when: false - - - name: Enable sshd in the container. - become: true - ansible.builtin.shell: - executable: /bin/bash - cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl enable sshd" - changed_when: nspawn_ssh_enabled.rc != 0 - - - name: Check if sshd is active in the container. - ansible.builtin.shell: - executable: /bin/bash - cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl is-active sshd" - register: nspawn_ssh_active - changed_when: false - failed_when: false - - - name: Start sshd in the container. - become: true - ansible.builtin.shell: - executable: /bin/bash - cmd: "systemd-run --pipe --machine {{ nspawn_name }} /bin/systemctl start sshd" - changed_when: nspawn_ssh_active.rc != 0 - - - name: Read the container's host key. - become: true - ansible.builtin.slurp: - src: "{{ (nspawn_image.stdout, '/etc/ssh/ssh_host_ecdsa_key.pub' | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" - register: nspawn_ssh_host_key - when: nspawn_ssh_host_trust_container - - - name: Trust the container's host key. - ansible.builtin.known_hosts: - name: "[127.0.0.1]:{{ nspawn_ssh_config.Port }}" - key: "[127.0.0.1]:{{ nspawn_ssh_config.Port }} {{ nspawn_ssh_host_key.content | b64decode }}" - when: nspawn_ssh_host_trust_container - - - name: Allow the Galaxy user to run HTCondor commands in the container. - # Uses /etc/sudoers. Ideally this would be solved using what is requested - # in this issue https://github.com/systemd/systemd/issues/10997, but the - # issue is still open. - become: true - community.general.sudoers: - name: htcondor-nspawn - user: "{{ galaxy_user.name }}" - nopassword: true - validation: required - setenv: true - commands: - - "{{ nspawn_condor_rm_command }} *" - - "{{ nspawn_condor_ssh_to_job_command }} *" - - "{{ nspawn_condor_submit_command }} *" - - - name: Make the environment variables available to the Galaxy handlers also available to the container. - become: true - ansible.builtin.copy: - content: "{{ nspawn_galaxy_environment_vars }}" - dest: "{{ (nspawn_image.stdout, nspawn_galaxy_environment_file | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" - owner: "{{ galaxy_user.name }}" - group: "{{ galaxy_group.name }}" - mode: "0440" - - name: HTCondor cluster. - hosts: htcondor:!sn06.galaxyproject.eu + hosts: htcondor become: true handlers: - name: Reload HTCondor @@ -280,7 +76,7 @@ - name: Open HTCondor shared port in the firewall. become: true ansible.posix.firewalld: - port: "{{ htcondor_port }}/tcp" + port: "{{ htcondor_server_port }}/tcp" state: enabled permanent: true immediate: true diff --git a/sn06.yml b/sn06.yml index 44bab37af..4bf454013 100644 --- a/sn06.yml +++ b/sn06.yml @@ -48,6 +48,7 @@ - "*.eirene.usegalaxy.eu" - "*.interactivetoolentrypoint.interactivetool.eirene.usegalaxy.eu" vars_files: + - group_vars/sn06.yml - group_vars/tiaas.yml # All of the training infrastructure - group_vars/gxconfig.yml # The base galaxy configuration - group_vars/toolbox.yml # User controlled toolbox diff --git a/templates/htcondor/condor_config.local.j2 b/templates/htcondor/condor_config.local.j2 index 7dab786b8..6f9371be3 100644 --- a/templates/htcondor/condor_config.local.j2 +++ b/templates/htcondor/condor_config.local.j2 @@ -1,12 +1,10 @@ # Networking CONDOR_HOST = {{ htcondor_server }} -COLLECTOR_HOST = $(CONDOR_HOST):{{ htcondor_port }} -SHARED_PORT_PORT = {{ htcondor_port }} -{% if "htcondor-secondary" in group_names %} +COLLECTOR_HOST = $(CONDOR_HOST):{{ htcondor_server_port }} +SHARED_PORT_PORT = {{ htcondor_shared_port }} WANT_UDP_COMMAND_SOCKET = False UPDATE_COLLECTOR_WITH_TCP = True UPDATE_VIEW_COLLECTOR_WITH_TCP = True -{% endif %} {% if htcondor_network_interface is defined %} NETWORK_INTERFACE = {{ htcondor_network_interface }} {% endif %} From a9a391116ded166a9d23d920e2ee9b7caf621342 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Fri, 15 Mar 2024 10:58:44 +0000 Subject: [PATCH 2/2] Add missing new line to the inventory file --- hosts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts b/hosts index 554b2deb6..76ee15fb0 100644 --- a/hosts +++ b/hosts @@ -91,4 +91,4 @@ maintenance.galaxyproject.eu sn06.galaxyproject.eu [htcondor-submit:vars] -ansible_group_priority=4 \ No newline at end of file +ansible_group_priority=4