usegalaxy-au · cat-bro · Oct 28, 2024 · Oct 29, 2024
diff --git a/dev-handler_playbook.yml b/dev-handler_playbook.yml
@@ -8,7 +8,7 @@
     - group_vars/dev_slurm.yml
     - host_vars/dev.gvl.org.au.yml
     - host_vars/dev-handlers.gvl.org.au.yml
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
     - secret_group_vars/stats_server_vault
     - secret_group_vars/dev_secrets
     - secret_group_vars/sentry_vault

diff --git a/dev-pulsar_playbook.yml b/dev-pulsar_playbook.yml
@@ -7,7 +7,7 @@
     - group_vars/dev_pulsar_slurm.yml
     - group_vars/VAULT
     - host_vars/dev-pulsar.gvl.org.au.yml
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
     - secret_group_vars/stats_server_vault
   pre_tasks:
     - name: apt update

diff --git a/dev_playbook.yml b/dev_playbook.yml
@@ -7,7 +7,7 @@
     - group_vars/galaxyservers.yml
     - group_vars/dev_slurm.yml
     - host_vars/dev.gvl.org.au.yml
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
     - secret_group_vars/stats_server_vault
     - secret_group_vars/dev_secrets
     - secret_group_vars/sentry_vault

diff --git a/dev_update_configs_playbook.yml b/dev_update_configs_playbook.yml
@@ -8,7 +8,7 @@
       - group_vars/dev_slurm.yml
       - host_vars/dev.gvl.org.au.yml
       - host_vars/dev-handlers.gvl.org.au.yml
-      - secret_group_vars/ubuntu_maintenance_key
+      - secret_group_vars/ssh_keys
       - secret_group_vars/dev_secrets
   handlers:
     - include: handlers/galaxy_handlers.yml

diff --git a/files/keys/pulsar_internal_hop_key.pub b/files/keys/pulsar_internal_hop_key.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDZHp4U2WsvJMYyholic8UId8YLNX+D5fYDrqKVlUIz95yeEK/drTGp6Qjng0uHz7976TdLRaVKQfarcMwHiVPu/HufdcR8LBuzgpgrWinTrahAXfmejGEHl3cJ+PC2nTpUY5nMhhtIPHBDUXFGz2tup8HTfyt3JkDLwc+4XF7SyQ== pulsar_internal_hop_key
diff --git a/galaxy-backup_playbook.yml b/galaxy-backup_playbook.yml
@@ -6,7 +6,7 @@
       - group_vars/VAULT
       - host_vars/galaxy-backup.yml
       - secret_group_vars/stats_server_vault
-      - secret_group_vars/ubuntu_maintenance_key
+      - secret_group_vars/ssh_keys
   pre_tasks:
     - name: Attach volume to instance
       include_role:

diff --git a/galaxy-db_playbook.yml b/galaxy-db_playbook.yml
@@ -7,7 +7,7 @@
       - group_vars/dbservers.yml
       - host_vars/galaxy-db.usegalaxy.org.au.yml
       - secret_group_vars/stats_server_vault
-      - secret_group_vars/ubuntu_maintenance_key
+      - secret_group_vars/ssh_keys
   pre_tasks:
       - name: Attach volume to instance
         include_role:

diff --git a/galaxy-handlers_playbook.yml b/galaxy-handlers_playbook.yml
@@ -9,7 +9,7 @@
     - host_vars/galaxy.usegalaxy.org.au.yml
     - host_vars/galaxy-handlers.usegalaxy.org.au.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
     - secret_group_vars/etca_vault.yml
   pre_tasks:
     - name: Attach volume to instance

diff --git a/galaxy-job-nfs_playbook.yml b/galaxy-job-nfs_playbook.yml
@@ -7,7 +7,7 @@
     - group_vars/nfs_servers.yml
     - host_vars/galaxy-job-nfs.usegalaxy.org.au.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   pre_tasks:
     - name: Attach volume to instance
       include_role:

diff --git a/galaxy-misc-nfs_playbook.yml b/galaxy-misc-nfs_playbook.yml
@@ -6,7 +6,7 @@
     - group_vars/VAULT
     - group_vars/nfs_servers.yml
     - host_vars/galaxy-misc-nfs.usegalaxy.org.au.yml
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
     - secret_group_vars/stats_server_vault
   pre_tasks:
     - name: Create nfs directories

diff --git a/galaxy-queue_playbook.yml b/galaxy-queue_playbook.yml
@@ -9,7 +9,7 @@
       - group_vars/galaxy_etca_slurm.yml
       - secret_group_vars/stats_server_vault
       - secret_group_vars/etca_vault.yml
-      - secret_group_vars/ubuntu_maintenance_key
+      - secret_group_vars/ssh_keys
   roles:
       - common
       - insspb.hostname

diff --git a/galaxy-user-nfs_playbook.yml b/galaxy-user-nfs_playbook.yml
@@ -7,7 +7,7 @@
     - group_vars/nfs_servers.yml
     - host_vars/galaxy-user-nfs.usegalaxy.org.au.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   pre_tasks:
     # - name: Attach volumes to instance # handle user nfs mounting manually
     #   include_role:

diff --git a/galaxy-workers_playbook.yml b/galaxy-workers_playbook.yml
@@ -6,7 +6,7 @@
       - group_vars/VAULT
       - group_vars/galaxy_etca_slurm.yml
       - group_vars/galaxy_workers.yml
-      - secret_group_vars/ubuntu_maintenance_key  # TODO: when branches are merged, rename this file to internal_keys
+      - secret_group_vars/ssh_keys  # TODO: when branches are merged, rename this file to internal_keys
       - secret_group_vars/stats_server_vault
   pre_tasks:
       - name: Attach volume to instance

diff --git a/galaxy_playbook.yml b/galaxy_playbook.yml
@@ -8,7 +8,7 @@
     - group_vars/galaxy_etca_slurm.yml
     - host_vars/galaxy.usegalaxy.org.au.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
     - secret_group_vars/etca_vault.yml
   pre_tasks:
     - name: Attach volume to instance

diff --git a/group_vars/all.yml b/group_vars/all.yml
@@ -252,6 +252,10 @@ internal_ssh_keys:
     private_key_content: "{{ vault_internal_hop_key }}"
     public_key_src: files/keys/internal_hop_key.pub
     user: ubuntu
+  pulsar_internal_hop_key:
+    private_key_content: "{{ vault_pulsar_internal_hop_key }}"
+    public_key_src: files/keys/pulsar_internal_hop_key.pub
+    user: ubuntu
 
 # Golang
 golang_version: '1.15.6' # default for role version 2.6.3

diff --git a/group_vars/pulsar-nci-test/pulsar-nci-test_workers.yml b/group_vars/pulsar-nci-test/pulsar-nci-test_workers.yml
@@ -32,3 +32,8 @@ cvmfs_quota_limit: 20000
 # Docker
 docker_daemon_options:
   data-root: /mnt/docker-data
+
+extra_keys:
+  - id: pulsar_internal_hop_key
+    type: public
+    from: "{{ hostvars['pulsar-nci-test']['internal_ip'] }}"
diff --git a/group_vars/pulsar_nci_training/pulsar-nci-training_workers.yml b/group_vars/pulsar_nci_training/pulsar-nci-training_workers.yml
@@ -31,3 +31,8 @@ attached_volumes:
 # docker
 docker_daemon_options:
   data-root: /mnt/tmpdisk/docker-data
+
+extra_keys:
+  - id: pulsar_internal_hop_key
+    type: public
+    from: "{{ hostvars['pulsar-nci-training']['internal_ip'] }}"
diff --git a/host_vars/pulsar-nci-test/pulsar-nci-test.usegalaxy.org.au.yml b/host_vars/pulsar-nci-test/pulsar-nci-test.usegalaxy.org.au.yml
@@ -30,3 +30,12 @@ pulsar_conda_exec: "mamba"
 # Docker
 docker_daemon_options:
   data-root: /mnt/docker-data
+
+extra_keys:
+  - id: pulsar_internal_hop_key
+    type: private
+
+# ssh config for ubuntu to ssh to worker machines
+ssh_config_id_file: "/home/{{ ssh_config_user }}/.ssh/pulsar_internal_hop_key"
+ssh_config_user: ubuntu
+ssh_config_hosts: "{{ groups['pulsar_nci_test_workers'] }}"
diff --git a/host_vars/pulsar-nci-training/pulsar-nci-training.usegalaxy.org.au.yml b/host_vars/pulsar-nci-training/pulsar-nci-training.usegalaxy.org.au.yml
@@ -28,7 +28,13 @@ extra_keys:
   - id: ubuntu_maintenance_key
     type: public
     from: "{{ hostvars['galaxy']['ansible_ssh_host'] }},{{ hostvars['galaxy-handlers']['ansible_ssh_host'] }},{{ hostvars['galaxy-backup']['ansible_ssh_host'] }}"
+  - id: pulsar_internal_hop_key
+    type: private
 
+# ssh config for ubuntu to ssh to worker machines
+ssh_config_id_file: "/home/{{ ssh_config_user }}/.ssh/pulsar_internal_hop_key"
+ssh_config_user: ubuntu
+ssh_config_hosts: "{{ groups['pulsar_nci_training_workers'] }}"
 
 # docker
 docker_daemon_options:

diff --git a/hosts b/hosts
@@ -106,13 +106,16 @@ nci-build ansible_ssh_host=130.56.246.156 internal_ip=10.0.0.249
 [pulsar_nci_training_head]
 pulsar-nci-training ansible_ssh_host=130.56.246.233 internal_ip=10.0.1.46
 
+[pulsar_nci_training_workers_unreachable]
+pulsar-nci-training-w5 ansible_ssh_host=10.0.1.218 internal_ip=10.0.1.218
+
 [pulsar_nci_training_workers]
 pulsar-nci-training-w0 ansible_ssh_host=10.0.1.197 internal_ip=10.0.1.197
 pulsar-nci-training-w1 ansible_ssh_host=10.0.0.28 internal_ip=10.0.0.28
 pulsar-nci-training-w2 ansible_ssh_host=10.0.3.167 internal_ip=10.0.3.167
 pulsar-nci-training-w3 ansible_ssh_host=10.0.2.236 internal_ip=10.0.2.236
 pulsar-nci-training-w4 ansible_ssh_host=10.0.0.175 internal_ip=10.0.0.175
-pulsar-nci-training-w5 ansible_ssh_host=10.0.1.218 internal_ip=10.0.1.218
+# pulsar-nci-training-w5 ansible_ssh_host=10.0.1.218 internal_ip=10.0.1.218
 pulsar-nci-training-w6 ansible_ssh_host=10.0.2.125 internal_ip=10.0.2.125
 pulsar-nci-training-w7 ansible_ssh_host=10.0.2.186 internal_ip=10.0.2.186
 pulsar-nci-training-w8 ansible_ssh_host=10.0.1.143 internal_ip=10.0.1.143

diff --git a/jenkins_playbook.yml b/jenkins_playbook.yml
@@ -5,7 +5,7 @@
     - group_vars/all.yml
     - group_vars/jenkins.yml
     - group_vars/VAULT
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
     - host_vars/jenkins.yml
 
   pre_tasks:

diff --git a/pulsar-QLD-nfs_playbook.yml b/pulsar-QLD-nfs_playbook.yml
@@ -7,7 +7,7 @@
       - group_vars/nfs_servers.yml
       - host_vars/pulsar-QLD/pulsar-QLD-nfs.genome.edu.au.yml
       - secret_group_vars/stats_server_vault
-      - secret_group_vars/ubuntu_maintenance_key
+      - secret_group_vars/ssh_keys
   roles:
       - attached-volumes
       - common

diff --git a/pulsar-QLD-queue_playbook.yml b/pulsar-QLD-queue_playbook.yml
@@ -8,7 +8,7 @@
       - host_vars/pulsar-QLD/pulsar-QLD-queue.genome.edu.au.yml
       - group_vars/pulsar_QLD/pulsar-QLD_slurm.yml
       - secret_group_vars/stats_server_vault
-      - secret_group_vars/ubuntu_maintenance_key
+      - secret_group_vars/ssh_keys
   roles:
       - common
       - insspb.hostname

diff --git a/pulsar-QLD_playbook.yml b/pulsar-QLD_playbook.yml
@@ -8,7 +8,7 @@
     - group_vars/VAULT
     - host_vars/pulsar-QLD/pulsar-QLD.genome.edu.au.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   pre_tasks:
     - name: Attach volume to instance
       include_role:

diff --git a/pulsar-high-mem1_playbook.yml b/pulsar-high-mem1_playbook.yml
@@ -6,7 +6,7 @@
     - group_vars/VAULT
     - host_vars/pulsar-high-mem1/pulsar-high-mem1.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   pre_tasks:
     - name: Create pulsar deps path
       file:

diff --git a/pulsar-high-mem2_playbook.yml b/pulsar-high-mem2_playbook.yml
@@ -6,7 +6,7 @@
     - group_vars/VAULT
     - host_vars/pulsar-high-mem2/pulsar-high-mem2.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   pre_tasks:
     - name: Create pulsar deps path
       file:

diff --git a/pulsar-mel2_playbook.yml b/pulsar-mel2_playbook.yml
@@ -8,7 +8,7 @@
     - group_vars/VAULT
     - host_vars/pulsar-mel2/pulsar-mel2.genome.edu.au.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   pre_tasks:
     - name: Attach volume to instance
       include_role:

diff --git a/pulsar-mel3_playbook.yml b/pulsar-mel3_playbook.yml
@@ -8,7 +8,7 @@
     - group_vars/VAULT
     - host_vars/pulsar-mel3/pulsar-mel3.genome.edu.au.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   pre_tasks:
     - name: apt update
       apt:

diff --git a/pulsar-nci-test_playbook.yml b/pulsar-nci-test_playbook.yml
@@ -7,6 +7,7 @@
     - group_vars/pulsar-nci-test/pulsar-nci-test_slurm.yml
     - group_vars/VAULT
     - host_vars/pulsar-nci-test/pulsar-nci-test.usegalaxy.org.au.yml
+    - secret_group_vars/ssh_keys
   roles:
     - common
     - insspb.hostname

diff --git a/pulsar-nci-test_workers_playbook.yml b/pulsar-nci-test_workers_playbook.yml
@@ -8,6 +8,7 @@
     - group_vars/pulsar-nci-test/pulsar-nci-test_workers.yml
     - group_vars/pulsar-nci-test/pulsar-nci-test_slurm.yml
     - group_vars/VAULT
+    - secret_group_vars/ssh_keys
   pre_tasks:
     # - name: Attach volume to instance
     #  include_role:

diff --git a/pulsar-nci-training_playbook.yml b/pulsar-nci-training_playbook.yml
@@ -26,7 +26,7 @@
     - group_vars/VAULT
     - host_vars/pulsar-nci-training/pulsar-nci-training.usegalaxy.org.au.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   roles:
     - common
     - insspb.hostname

diff --git a/pulsar-nci-training_workers_playbook.yml b/pulsar-nci-training_workers_playbook.yml
@@ -9,6 +9,7 @@
     - group_vars/pulsar_nci_training/pulsar-nci-training_slurm.yml
     - group_vars/VAULT
     - secret_group_vars/stats_server_vault
+    - secret_group_vars/ssh_keys
   pre_tasks:
     - name: Remove cloudinit entry in fstab
       # remove line '/dev/vdb	/mnt	auto	defaults,nofail,x-systemd.requires=cloud-init.service,comment=cloudconfig	0	2'

diff --git a/pulsar-qld-blast_playbook.yml b/pulsar-qld-blast_playbook.yml
@@ -6,7 +6,7 @@
     - group_vars/VAULT
     - host_vars/pulsar-qld-blast.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   roles:
     - common
     - insspb.hostname

diff --git a/pulsar-qld-gpu-dev_playbook.yml b/pulsar-qld-gpu-dev_playbook.yml
@@ -7,7 +7,7 @@
     - group_vars/pulsar_qld_gpus.yml
     - group_vars/VAULT
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
     - host_vars/pulsar-qld-gpu-dev.yml
   pre_tasks:
     - name: ensure Nvidia GPU has been pre-configured with one-offs/initial_nvidia_config_and_disable_cgroups_v2_playbook.yml

diff --git a/pulsar-qld-gpu_playbook.yml b/pulsar-qld-gpu_playbook.yml
@@ -6,7 +6,7 @@
     - group_vars/VAULT
     - group_vars/pulsar_qld_gpus.yml
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
   pre_tasks:
     - name: ensure Nvidia GPU has been pre-configured with one-offs/initial_nvidia_config_and_disable_cgroups_v2_playbook.yml
       shell:

diff --git a/pulsar-qld-high-mem_playbook.yml b/pulsar-qld-high-mem_playbook.yml
@@ -5,7 +5,7 @@
     - group_vars/pulsarservers.yml
     - group_vars/VAULT
     - secret_group_vars/stats_server_vault
-    - secret_group_vars/ubuntu_maintenance_key
+    - secret_group_vars/ssh_keys
       #    - host_vars/pulsar-high-mem2/pulsar-high-mem2.yml
   pre_tasks:
     - name: Attach volume to instance
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDZHp4U2WsvJMYyholic8UId8YLNX+D5fYDrqKVlUIz95yeEK/drTGp6Qjng0uHz7976TdLRaVKQfarcMwHiVPu/HufdcR8LBuzgpgrWinTrahAXfmejGEHl3cJ+PC2nTpUY5nMhhtIPHBDUXFGz2tup8HTfyt3JkDLwc+4XF7SyQ== pulsar_internal_hop_key