From 98be3b508104841a363c0f9d912404eaed903e9d Mon Sep 17 00:00:00 2001 From: cat-bro Date: Fri, 22 Dec 2023 13:23:24 +1100 Subject: [PATCH 1/5] Remove pg-post-tasks and add common bash variable and functions --- dev_playbook.yml | 1 - galaxy-handlers_playbook.yml | 1 - galaxy_playbook.yml | 1 - group_vars/all.yml | 6 -- group_vars/galaxyservers.yml | 3 +- roles/common/defaults/main.yml | 8 ++ roles/common/tasks/bashrc.yml | 80 +++++++++++++++++++ roles/common/tasks/machine_users.yml | 62 +++++++------- roles/common/tasks/main.yml | 7 +- .../bashrc/get_jwd_path_galaxy.py.j2 | 26 ++++++ roles/pg-post-tasks/tasks/main.yml | 23 ------ roles/pg-post-tasks/tasks/pgpass_bashrc.yml | 38 --------- staging_playbook.yml | 1 - 13 files changed, 153 insertions(+), 104 deletions(-) create mode 100644 roles/common/tasks/bashrc.yml create mode 100644 roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 delete mode 100644 roles/pg-post-tasks/tasks/main.yml delete mode 100644 roles/pg-post-tasks/tasks/pgpass_bashrc.yml diff --git a/dev_playbook.yml b/dev_playbook.yml index 07560399d..03f35a5ab 100644 --- a/dev_playbook.yml +++ b/dev_playbook.yml @@ -65,7 +65,6 @@ # - galaxyproject.s3fs - galaxyproject.cvmfs - galaxyproject.gxadmin - - pg-post-tasks - remote-pulsar-cron - galaxy-pg-cleanup #- galaxyproject.tiaas2 # broken in galaxy release_23.1 - missing static/style/base.css diff --git a/galaxy-handlers_playbook.yml b/galaxy-handlers_playbook.yml index 40fc568d5..02c736c7a 100644 --- a/galaxy-handlers_playbook.yml +++ b/galaxy-handlers_playbook.yml @@ -63,7 +63,6 @@ - galaxyproject.slurm - galaxyproject.cvmfs - galaxyproject.gxadmin - - pg-post-tasks - geerlingguy.docker - dj-wasabi.telegraf - usegalaxy_eu.flower diff --git a/galaxy_playbook.yml b/galaxy_playbook.yml index 0ca103afd..5d360d4e1 100644 --- a/galaxy_playbook.yml +++ b/galaxy_playbook.yml @@ -56,7 +56,6 @@ - dj-wasabi.telegraf - geerlingguy.nfs - galaxyproject.gxadmin - - pg-post-tasks - postfix-mail-relay - remote-pulsar-cron # - delete-tmp-jwds-cron # commented out until galaxy etca is production galaxy, also needs to be enabled diff --git a/group_vars/all.yml b/group_vars/all.yml index e7463496b..ed24a7516 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -176,12 +176,6 @@ galaxy_cvmfs_server_urls: - "http://cvmfs1-tacc0.galaxyproject.org/cvmfs/@fqrn@" - "http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@" -#sinfo and squeue configs -bashrc_env: | - export SINFO_FORMAT="%24n %.14C %.8t" - export SQUEUE_FORMAT="%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z" - export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" - # miniconda miniconda_version: '4.12.0' miniconda_mamba_version: '0.23.3' diff --git a/group_vars/galaxyservers.yml b/group_vars/galaxyservers.yml index cab5edab8..dbafead4d 100644 --- a/group_vars/galaxyservers.yml +++ b/group_vars/galaxyservers.yml @@ -121,6 +121,7 @@ galaxy_venv_dir: "{{ galaxy_root }}/venv" # set this explicitly as it can fail w galaxy_mutable_config_dir: "{{ galaxy_root }}/var" galaxy_mutable_data_dir: "{{ galaxy_root }}" galaxy_config_dir: "{{ galaxy_root }}/config" +galaxy_job_working_directory: "{{ galaxy_tmp_dir }}/job_working_directory" galaxy_conda_prefix: "{{ galaxy_tools_indices_dir }}/tool_dependencies/_conda" galaxy_conda_exec: mamba @@ -157,7 +158,7 @@ group_galaxy_config: check_migrate_tools: false log_level: TRACE new_file_path: "{{ galaxy_tmp_dir }}" - job_working_directory: "{{ galaxy_tmp_dir }}/job_working_directory" + job_working_directory: "{{ galaxy_job_working_directory }}" allow_user_impersonation: true allow_user_deletion: true allow_path_paste: true diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 5baf8e886..03d654a53 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -8,3 +8,11 @@ htpass_username: "ubuntu" internal_ssh_keys: {} extra_keys: [] + +common_sinfo_format: "%24n %.14C %.8t" +common_squeue_format: "%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z" +common_sacct_format: "jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" + +common_bashrc_dir: /etc/bashrc_common +common_bashrc_vars_file: "{{ common_bashrc_dir }}/common_variables" +common_bashrc_functions_file: "{{ common_bashrc_dir }}/common_functions" diff --git a/roles/common/tasks/bashrc.yml b/roles/common/tasks/bashrc.yml new file mode 100644 index 000000000..a2ac0a4c6 --- /dev/null +++ b/roles/common/tasks/bashrc.yml @@ -0,0 +1,80 @@ +--- +- name: Ensure directory for bashrc common files + file: + state: directory + path: "{{ common_bashrc_dir }}" +- name: Template bashrc_common path + copy: + dest: "{{ common_bashrc_vars_file }}" + mode: 0755 + content: | + # slurm + export SINFO_FORMAT="{{ common_sinfo_format }}" + export SQUEUE_FORMAT="{{ common_squeue_format }}" + export SINFO_FORMAT="{{ common_sacct_format }}" + + {% if is_galaxy_head_node %} + # galaxy + export PGHOST='{{ db_address }}' + export PGDATABASE='galaxy' + export PGPORT='5432' + export GALAXY_CONFIG_DIR={{ galaxy_config_dir }} + export GALAXY_MUTABLE_CONFIG_DIR={{ galaxy_mutable_config_dir }} + export GALAXY_ROOT={{ galaxy_server_dir }} + export GALAXY_CONFIG_FILE={{ galaxy_config_file }} + export GALAXY_JWD_PATH="{{ galaxy_job_working_directory }}" + {% endif %} +- name: Template common_bashrc path + copy: + dest: "{{ common_bashrc_functions_file }}" + mode: 0755 + content: | + {% if is_galaxy_head_node %} + # galaxy + jwd() { # print path of job working directory for job id + python "{{ common_bashrc_dir }}/get_jwd_path_galaxy.py" $1 + } + + jwd-size() { + du -sh $(jwd $1) + } + + go-jwd() { # go to job working directory for job id + cd $(jwd $1) + } + + tail-stderr() { + stderr_filename=$(jwd $1)/outputs/tool_stderr + if [ ! -f $stderr_filename ]; then + echo "No stderr file for job $1"; + else + tail $stderr_filename; + fi + } + {% endif %} + +- name: Template helper scripts + template: + src: "bashrc/{{ item }}.j2" + dest: "{{ common_bashrc_dir }}/{{ item }}" + with_items: + - get_jwd_path_galaxy.py + +- name: Add to .bashrc files + blockinfile: + marker: "# {mark} ANSIBLE MANAGED BLOCK (common role: source common bash env for users)" + dest: "/home/{{ item.name }}/.bashrc" + block: | + source {{ common_bashrc_vars_file }} + source {{ common_bashrc_functions_file }} + + # local to user + {% if item == 'ubuntu' %} + export PGUSER='galaxy' + {% elif 'tiaas_admin' in item.roles|d([]) %} + export PGUSER='tiaasadmin' + {% else %} + export PGUSER='reader' + {% endif %} + + with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" diff --git a/roles/common/tasks/machine_users.yml b/roles/common/tasks/machine_users.yml index c9ef29ec1..eac2ff7de 100644 --- a/roles/common/tasks/machine_users.yml +++ b/roles/common/tasks/machine_users.yml @@ -4,13 +4,6 @@ become_user: root block: - # FIX for deprecated code that added user files in /etc/sudoers.d/ - can be removed in future - - name: delete /etc/sudoers.d/username files for machine users - file: - path: "/etc/sudoers.d/{{ item.name }}" - state: absent - with_items: "{{ remove_machine_users + machine_users }}" - - name: Ensure group "devs" exists group: name: devs @@ -65,15 +58,6 @@ force: yes with_items: "{{ remove_machine_users }}" - - name: Add some env_vars to .bashrcs - blockinfile: - path: "/home/{{ item.name }}/.bashrc" - block: | - {{ bashrc_env }} - marker: "# {mark} MANAGED BY ANSIBLE - DO NOT MODIFY" - with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" - when: bashrc_env is defined - - name: Add SSH keys for users authorized_key: user: "{{ item.name }}" @@ -81,18 +65,34 @@ state: present with_items: "{{ machine_users }}" - - name: Add bash function gxctl for sudo users only - blockinfile: - path: "/home/{{ item.name }}/.bashrc" - block: | - gxctl() { - export gctl_args="$@" - export GRAVITY_STATE_DIR={{ galaxy_gravity_state_dir }} - export GALAXY_CONFIG_FILE={{ galaxy_config_file }} - - sudo -H -Eu {{ galaxy_user.name }} bash -c '. {{ galaxy_venv_dir }}/bin/activate && {{ galaxy_venv_dir }}/bin/galaxyctl $gctl_args' - } - marker: "# {mark} MANAGED BY ANSIBLE - DO NOT MODIFY (gxctl)" - state: absent # TODO: Once this is gone from dev/staging/aarnet, remove this task altogether - with_items: "{{ machine_users | selectattr('roles', 'contains', 'sudo') | list + [{'name': 'ubuntu'}] }}" - when: is_galaxy_head_node|d(false) + # former pg-post-tasks tasks + - name: Pgpass creation and gxadmin user tasks + when: is_galaxy_head_node|d(False) + block: + - name: Create .pgpass files for users + copy: + dest: "/home/{{ item.name }}/.pgpass" + content: "{{ db_address }}:5432:*:{{ db_role }}:{{ db_password }}" + mode: "600" + group: "{{ item.name }}" + owner: "{{ item.name }}" + vars: + db_role: "{{ 'galaxy' if item.name == 'ubuntu' else ('reader' if not 'tiaas_admin' in item.roles|d([]) else 'tiaasadmin') }}" + db_password: "{{ pg_db_password[db_role] }}" + with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" + - name: Ensure gxadmin config directory exists for all users including ubuntu + file: + path: "/home/{{ item }}/.config" + state: directory + group: "{{ item }}" + owner: "{{ item }}" + with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" + - name: Ensure that all users + ubuntu have a copy of gxadmin-local.sh + copy: + src: files/galaxy/gxadmin/gxadmin-local.sh + dest: "/home/{{ item }}/.config/gxadmin-local.sh" + mode: "755" + group: "{{ item }}" + owner: "{{ item }}" + force: yes + with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 78ab65ca1..e574ccf2c 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -83,6 +83,11 @@ - name: Add machine users import_tasks: machine_users.yml + when: not skip_machine_users|d(False) + +- name: Add variables to bashrc files + import_tasks: bashrc.yml + when: not skip_machine_users|d(False) - name: Add extra ssh keys include_tasks: extra_keys.yml @@ -122,4 +127,4 @@ lineinfile: path: /etc/environment create: true - line: export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" + line: export SACCT_FORMAT="{{ common_sacct_format }}" diff --git a/roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 b/roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 new file mode 100644 index 000000000..a60c6ca8d --- /dev/null +++ b/roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 @@ -0,0 +1,26 @@ +import argparse +import os + +def main(): + parser = argparse.ArgumentParser(description='Get job working directory from a job id') + parser.add_argument('job_id', type=int, help='Galaxy job id') + args = parser.parse_args() + + prefix = os.environ.get("GALAXY_JWD_PATH") + if not prefix: + raise ValueError("Please set GALAXY_JWD_PATH environment variable") + + jwd_path = get_jwd_path(args.job_id, prefix) + print(jwd_path) + +def get_jwd_path(job_id, prefix): + if len(str(job_id)) > 6: # on production this is the case + nine_digit_id = '0'*(9-len(str(job_id))) + str(job_id) + return os.path.join(prefix, nine_digit_id[:3], nine_digit_id[3:6], str(job_id)) + elif len(str(job_id)) <= 6: + six_digit_id = '0'*(6-len(str(job_id))) + str(job_id) + return os.path.join(prefix, six_digit_id[:3], str(job_id)) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/roles/pg-post-tasks/tasks/main.yml b/roles/pg-post-tasks/tasks/main.yml deleted file mode 100644 index 7be3756a0..000000000 --- a/roles/pg-post-tasks/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- - - name: Set .pgpass file and variables for gxadmin in .bashrc for all users - include: pgpass_bashrc.yml - loop: "{{ machine_users + [{'name': 'ubuntu'}] }}" - loop_control: - loop_var: machine_user - - name: Ensure gxadmin config directory exists for all users including ubuntu - file: - path: "/home/{{ item }}/.config" - state: directory - group: "{{ item }}" - owner: "{{ item }}" - with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" - - name: Ensure that all users + ubuntu have a copy of gxadmin-local.sh - copy: - src: files/galaxy/gxadmin/gxadmin-local.sh - dest: "/home/{{ item }}/.config/gxadmin-local.sh" - mode: "755" - group: "{{ item }}" - owner: "{{ item }}" - force: yes - with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" - diff --git a/roles/pg-post-tasks/tasks/pgpass_bashrc.yml b/roles/pg-post-tasks/tasks/pgpass_bashrc.yml deleted file mode 100644 index c8e89899d..000000000 --- a/roles/pg-post-tasks/tasks/pgpass_bashrc.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- - - name: Set pg_db_role - set_fact: - db_role: "{{ 'galaxy' if machine_user.name == 'ubuntu' else ('reader' if not 'tiaas_admin' in machine_user.roles|d([]) else 'tiaasadmin') }}" - - name: Set pg_db_password - set_fact: - db_password: "{{ pg_db_password[db_role] }}" - - name: Create .pgpass files for users - copy: - dest: "/home/{{ machine_user.name }}/.pgpass" - content: "{{ db_address }}:5432:*:{{ db_role }}:{{ db_password }}" - mode: "600" - group: "{{ machine_user.name }}" - owner: "{{ machine_user.name }}" - - name: Add env vars for gxadmin to machine_users .bashrc files - lineinfile: - path: "/home/{{ machine_user.name }}/.bashrc" - regexp: "^export {{ obj.var }}=" - line: "export {{ obj.var }}='{{ obj.val }}'" - loop: - - var: PGHOST - val: "{{ db_address }}" - - var: PGUSER - val: "{{ db_role }}" - - var: PGDATABASE - val: "galaxy" - - var: PGPORT - val: "5432" - - var: GALAXY_CONFIG_DIR - val: "{{ galaxy_config_dir }}" - - var: GALAXY_MUTABLE_CONFIG_DIR - val: "{{ galaxy_mutable_config_dir }}" - - var: GALAXY_ROOT - val: "{{ galaxy_server_dir }}" - - var: GALAXY_CONFIG_FILE - val: "{{ galaxy_config_dir }}/galaxy.yml" - loop_control: - loop_var: obj diff --git a/staging_playbook.yml b/staging_playbook.yml index c8f27aac2..9f046bfb0 100644 --- a/staging_playbook.yml +++ b/staging_playbook.yml @@ -45,7 +45,6 @@ - acl-on-startup - galaxyproject.gxadmin - dj-wasabi.telegraf - - pg-post-tasks - postfix-mail-relay post_tasks: - name: Ensure object store paths exist From b96c24526120602857e613e4a5e34b35666ab637 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Mon, 15 Jan 2024 11:59:38 +1100 Subject: [PATCH 2/5] Add helpful variables --- roles/common/tasks/bashrc.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/common/tasks/bashrc.yml b/roles/common/tasks/bashrc.yml index a2ac0a4c6..d94531bbe 100644 --- a/roles/common/tasks/bashrc.yml +++ b/roles/common/tasks/bashrc.yml @@ -13,6 +13,11 @@ export SQUEUE_FORMAT="{{ common_squeue_format }}" export SINFO_FORMAT="{{ common_sacct_format }}" + # helpful settings + export EDITOR=vim + HISTSIZE= + HISTFILESIZE= + {% if is_galaxy_head_node %} # galaxy export PGHOST='{{ db_address }}' From 3467a6e761e1853ee5a3cb0b71692a54b7b9cc58 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Tue, 16 Jan 2024 16:33:45 +1100 Subject: [PATCH 3/5] More host groups --- hosts | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/hosts b/hosts index fa3dbd2c3..fd99043b1 100644 --- a/hosts +++ b/hosts @@ -149,15 +149,30 @@ pulsar_qld_blast pulsar_nci_training_head pulsar_azure_0_head +# All pulsar head nodes +[pulsar_head_nodes:children] +production_pulsars +dev_pulsar_head +staging_pulsar_head +pulsar_nci_test_head + +# All pulsar workers +[pulsar_workers:children] +pulsar_paw_workers +pulsar_mel3_workers +pulsar_mel2_workers +pulsar_QLD_workers +pulsar_nci_training_workers +pulsar_nci_test_workers + # Dev cluster (GenomicsVL_Devt) -[dev_group] -dev-pulsar ansible_ssh_host=45.113.233.82 [dev_group:children] dev_db_server dev_galaxy_server dev_handlers_server dev_slurm_head dev_workers +dev_pulsar_head [dev_db_server] dev-db ansible_ssh_host=45.113.232.252 @@ -175,14 +190,16 @@ dev-queue ansible_ssh_host=45.113.232.149 dev-w1 ansible_ssh_host=45.113.233.251 dev-w2 ansible_ssh_host=45.113.233.7 +[dev_pulsar_head] +dev-pulsar ansible_ssh_host=45.113.233.82 + # Staging cluster (GenomicsVL) -[staging_group] -staging-pulsar ansible_ssh_host=115.146.87.193 [staging_group:children] staging_db_server staging_galaxy_server staging_slurm_head staging_workers +staging_pulsar_head [staging_db_server] staging-db ansible_ssh_host=115.146.87.93 @@ -196,6 +213,9 @@ staging-queue ansible_ssh_host=115.146.84.121 [staging_workers] staging-w1 ansible_ssh_host=115.146.87.130 +[staging_pulsar_head] +staging-pulsar ansible_ssh_host=115.146.87.193 + # CVMFS [cvmfsstratum1servers] cvmfs1-mel0.gvl.org.au ansible_user=ec2-user ansible_ssh_host=115.146.85.207 From e48fcf36bdb269acd6b699d777e05cfdb2928143 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Tue, 16 Jan 2024 16:34:06 +1100 Subject: [PATCH 4/5] add some pulsar functions --- roles/common/tasks/bashrc.yml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/roles/common/tasks/bashrc.yml b/roles/common/tasks/bashrc.yml index d94531bbe..56f6c411c 100644 --- a/roles/common/tasks/bashrc.yml +++ b/roles/common/tasks/bashrc.yml @@ -58,6 +58,30 @@ } {% endif %} + {% if ansible_hostname in groups['pulsar_head_nodes'] %} + # pulsar + jwd() { # print path of job working directory for job id + echo {{ pulsar_staging_dir }}/${1} + } + + jwd-size() { + du -sh $(jwd $1) + } + + go-jwd() { # go to job working directory for job id + cd $(jwd $1) + } + + tail-stderr() { + stderr_filename=$(jwd $1)/metadata/tool_stderr + if [ ! -f $stderr_filename ]; then + echo "No stderr file for job $1"; + else + tail $stderr_filename; + fi + } + {% endif %} + - name: Template helper scripts template: src: "bashrc/{{ item }}.j2" @@ -73,7 +97,8 @@ source {{ common_bashrc_vars_file }} source {{ common_bashrc_functions_file }} - # local to user + {% if is_galaxy_head_node %} + # galaxy (per user) {% if item == 'ubuntu' %} export PGUSER='galaxy' {% elif 'tiaas_admin' in item.roles|d([]) %} @@ -81,5 +106,6 @@ {% else %} export PGUSER='reader' {% endif %} + {% endif %} with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" From 6c7f2110719a675571dd1a99d1d1c67f3b581e69 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Tue, 16 Jan 2024 18:11:08 +1100 Subject: [PATCH 5/5] remove more references to pg-post-tasks --- README.md | 1 - dev-handler_playbook.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/README.md b/README.md index 0d6c74b0b..008aaf7fb 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,6 @@ There are a few custom roles associated with these playbooks. They are: | common | Performs common tasks on our virtual machines including setting up users, installing some required packages amongst other tasks | | mariadb | Sets up MariaDB for Slurm accounting on various clusters | | mounts | Sets up NFS mounts for various clusters | -| pg-post-tasks | Adds some users and extra permissions to the Galaxy Database | | slg.db-backup | Role to setup database backups for Galaxy Australia | | slg.galaxy_stats | Sets up collection of statistics from Galaxy for Grafana | diff --git a/dev-handler_playbook.yml b/dev-handler_playbook.yml index 7692fb5da..e38649d02 100644 --- a/dev-handler_playbook.yml +++ b/dev-handler_playbook.yml @@ -50,7 +50,6 @@ # - galaxyproject.s3fs - galaxyproject.cvmfs - galaxyproject.gxadmin - - pg-post-tasks - geerlingguy.docker - usegalaxy_eu.gie_proxy - dj-wasabi.telegraf