diff --git a/dev_playbook.yml b/dev_playbook.yml index 07560399d..03f35a5ab 100644 --- a/dev_playbook.yml +++ b/dev_playbook.yml @@ -65,7 +65,6 @@ # - galaxyproject.s3fs - galaxyproject.cvmfs - galaxyproject.gxadmin - - pg-post-tasks - remote-pulsar-cron - galaxy-pg-cleanup #- galaxyproject.tiaas2 # broken in galaxy release_23.1 - missing static/style/base.css diff --git a/galaxy-handlers_playbook.yml b/galaxy-handlers_playbook.yml index 40fc568d5..02c736c7a 100644 --- a/galaxy-handlers_playbook.yml +++ b/galaxy-handlers_playbook.yml @@ -63,7 +63,6 @@ - galaxyproject.slurm - galaxyproject.cvmfs - galaxyproject.gxadmin - - pg-post-tasks - geerlingguy.docker - dj-wasabi.telegraf - usegalaxy_eu.flower diff --git a/galaxy_playbook.yml b/galaxy_playbook.yml index 0ca103afd..5d360d4e1 100644 --- a/galaxy_playbook.yml +++ b/galaxy_playbook.yml @@ -56,7 +56,6 @@ - dj-wasabi.telegraf - geerlingguy.nfs - galaxyproject.gxadmin - - pg-post-tasks - postfix-mail-relay - remote-pulsar-cron # - delete-tmp-jwds-cron # commented out until galaxy etca is production galaxy, also needs to be enabled diff --git a/group_vars/all.yml b/group_vars/all.yml index e7463496b..eb6665dbb 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -176,11 +176,11 @@ galaxy_cvmfs_server_urls: - "http://cvmfs1-tacc0.galaxyproject.org/cvmfs/@fqrn@" - "http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@" -#sinfo and squeue configs -bashrc_env: | - export SINFO_FORMAT="%24n %.14C %.8t" - export SQUEUE_FORMAT="%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z" - export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" +# #sinfo and squeue configs +# bashrc_env: | +# export SINFO_FORMAT="%24n %.14C %.8t" +# export SQUEUE_FORMAT="%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z" +# export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" # miniconda miniconda_version: '4.12.0' diff --git a/group_vars/galaxyservers.yml b/group_vars/galaxyservers.yml index cab5edab8..dbafead4d 100644 --- a/group_vars/galaxyservers.yml +++ b/group_vars/galaxyservers.yml @@ -121,6 +121,7 @@ galaxy_venv_dir: "{{ galaxy_root }}/venv" # set this explicitly as it can fail w galaxy_mutable_config_dir: "{{ galaxy_root }}/var" galaxy_mutable_data_dir: "{{ galaxy_root }}" galaxy_config_dir: "{{ galaxy_root }}/config" +galaxy_job_working_directory: "{{ galaxy_tmp_dir }}/job_working_directory" galaxy_conda_prefix: "{{ galaxy_tools_indices_dir }}/tool_dependencies/_conda" galaxy_conda_exec: mamba @@ -157,7 +158,7 @@ group_galaxy_config: check_migrate_tools: false log_level: TRACE new_file_path: "{{ galaxy_tmp_dir }}" - job_working_directory: "{{ galaxy_tmp_dir }}/job_working_directory" + job_working_directory: "{{ galaxy_job_working_directory }}" allow_user_impersonation: true allow_user_deletion: true allow_path_paste: true diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 5baf8e886..03d654a53 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -8,3 +8,11 @@ htpass_username: "ubuntu" internal_ssh_keys: {} extra_keys: [] + +common_sinfo_format: "%24n %.14C %.8t" +common_squeue_format: "%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z" +common_sacct_format: "jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" + +common_bashrc_dir: /etc/bashrc_common +common_bashrc_vars_file: "{{ common_bashrc_dir }}/common_variables" +common_bashrc_functions_file: "{{ common_bashrc_dir }}/common_functions" diff --git a/roles/common/tasks/bashrc.yml b/roles/common/tasks/bashrc.yml new file mode 100644 index 000000000..964392d18 --- /dev/null +++ b/roles/common/tasks/bashrc.yml @@ -0,0 +1,81 @@ +--- +- name: Ensure directory for bashrc common files + file: + state: directory + path: "{{ common_bashrc_dir }}" +- name: Template bashrc_common path + copy: + dest: "{{ common_bashrc_vars_file }}" + mode: 0755 + content: | + # slurm + export SINFO_FORMAT="{{ common_sinfo_format }}" + export SQUEUE_FORMAT="{{ common_squeue_format }}" + export SINFO_FORMAT="{{ common_sacct_format }}" + + {% if is_galaxy_head_node %} + # galaxy + export PGHOST='{{ db_address }}' + ### export PGUSER='galaxy' this goes in local file + export PGDATABASE='galaxy' + export PGPORT='5432' + export GALAXY_CONFIG_DIR={{ galaxy_config_dir }} + export GALAXY_MUTABLE_CONFIG_DIR={{ galaxy_mutable_config_dir }} + export GALAXY_ROOT={{ galaxy_server_dir }} + export GALAXY_CONFIG_FILE={{ galaxy_config_file }} + export GALAXY_JWD_PATH="{{ galaxy_job_working_directory }}" + {% endif %} +- name: Template common_bashrc path + copy: + dest: "{{ common_bashrc_functions_file }}" + mode: 0755 + content: | + {% if is_galaxy_head_node %} + # galaxy + jwd() { # print path of job working directory for job id + python "{{ common_bashrc_dir }}/get_jwd_path_galaxy.py" $1 + } + + jwd-size() { + du -sh $(jwd $1) + } + + go-jwd() { # go to job working directory for job id + cd $(jwd $1) + } + + tail-stderr() { + stderr_filename=$(jwd $1)/outputs/tool_stderr + if [ ! -f $stderr_filename ]; then + echo "No stderr file for job $1"; + else + tail $stderr_filename; + fi + } + {% endif %} + +- name: Template helper scripts + template: + src: "bashrc/{{ item }}.j2" + dest: "{{ common_bashrc_dir }}/{{ item }}" + with_items: + - get_jwd_path_galaxy.py + +- name: Add to .bashrc files + blockinfile: + marker: "# {mark} ANSIBLE MANAGED BLOCK (common role: source common bash env for users)" + dest: "/home/{{ item.name }}/.bashrc" + block: | + source {{ common_bashrc_vars_file }} + source {{ common_bashrc_functions_file }} + + # local to user + {% if item == 'ubuntu' %} + export PGUSER='galaxy' + {% elif 'tiaas_admin' in item.roles|d([]) %} + export PGUSER='tiaasadmin' + {% else %} + export PGUSER='reader' + {% endif %} + + with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" diff --git a/roles/common/tasks/machine_users.yml b/roles/common/tasks/machine_users.yml index c9ef29ec1..eac2ff7de 100644 --- a/roles/common/tasks/machine_users.yml +++ b/roles/common/tasks/machine_users.yml @@ -4,13 +4,6 @@ become_user: root block: - # FIX for deprecated code that added user files in /etc/sudoers.d/ - can be removed in future - - name: delete /etc/sudoers.d/username files for machine users - file: - path: "/etc/sudoers.d/{{ item.name }}" - state: absent - with_items: "{{ remove_machine_users + machine_users }}" - - name: Ensure group "devs" exists group: name: devs @@ -65,15 +58,6 @@ force: yes with_items: "{{ remove_machine_users }}" - - name: Add some env_vars to .bashrcs - blockinfile: - path: "/home/{{ item.name }}/.bashrc" - block: | - {{ bashrc_env }} - marker: "# {mark} MANAGED BY ANSIBLE - DO NOT MODIFY" - with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" - when: bashrc_env is defined - - name: Add SSH keys for users authorized_key: user: "{{ item.name }}" @@ -81,18 +65,34 @@ state: present with_items: "{{ machine_users }}" - - name: Add bash function gxctl for sudo users only - blockinfile: - path: "/home/{{ item.name }}/.bashrc" - block: | - gxctl() { - export gctl_args="$@" - export GRAVITY_STATE_DIR={{ galaxy_gravity_state_dir }} - export GALAXY_CONFIG_FILE={{ galaxy_config_file }} - - sudo -H -Eu {{ galaxy_user.name }} bash -c '. {{ galaxy_venv_dir }}/bin/activate && {{ galaxy_venv_dir }}/bin/galaxyctl $gctl_args' - } - marker: "# {mark} MANAGED BY ANSIBLE - DO NOT MODIFY (gxctl)" - state: absent # TODO: Once this is gone from dev/staging/aarnet, remove this task altogether - with_items: "{{ machine_users | selectattr('roles', 'contains', 'sudo') | list + [{'name': 'ubuntu'}] }}" - when: is_galaxy_head_node|d(false) + # former pg-post-tasks tasks + - name: Pgpass creation and gxadmin user tasks + when: is_galaxy_head_node|d(False) + block: + - name: Create .pgpass files for users + copy: + dest: "/home/{{ item.name }}/.pgpass" + content: "{{ db_address }}:5432:*:{{ db_role }}:{{ db_password }}" + mode: "600" + group: "{{ item.name }}" + owner: "{{ item.name }}" + vars: + db_role: "{{ 'galaxy' if item.name == 'ubuntu' else ('reader' if not 'tiaas_admin' in item.roles|d([]) else 'tiaasadmin') }}" + db_password: "{{ pg_db_password[db_role] }}" + with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" + - name: Ensure gxadmin config directory exists for all users including ubuntu + file: + path: "/home/{{ item }}/.config" + state: directory + group: "{{ item }}" + owner: "{{ item }}" + with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" + - name: Ensure that all users + ubuntu have a copy of gxadmin-local.sh + copy: + src: files/galaxy/gxadmin/gxadmin-local.sh + dest: "/home/{{ item }}/.config/gxadmin-local.sh" + mode: "755" + group: "{{ item }}" + owner: "{{ item }}" + force: yes + with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 78ab65ca1..e574ccf2c 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -83,6 +83,11 @@ - name: Add machine users import_tasks: machine_users.yml + when: not skip_machine_users|d(False) + +- name: Add variables to bashrc files + import_tasks: bashrc.yml + when: not skip_machine_users|d(False) - name: Add extra ssh keys include_tasks: extra_keys.yml @@ -122,4 +127,4 @@ lineinfile: path: /etc/environment create: true - line: export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" + line: export SACCT_FORMAT="{{ common_sacct_format }}" diff --git a/roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 b/roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 new file mode 100644 index 000000000..a60c6ca8d --- /dev/null +++ b/roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 @@ -0,0 +1,26 @@ +import argparse +import os + +def main(): + parser = argparse.ArgumentParser(description='Get job working directory from a job id') + parser.add_argument('job_id', type=int, help='Galaxy job id') + args = parser.parse_args() + + prefix = os.environ.get("GALAXY_JWD_PATH") + if not prefix: + raise ValueError("Please set GALAXY_JWD_PATH environment variable") + + jwd_path = get_jwd_path(args.job_id, prefix) + print(jwd_path) + +def get_jwd_path(job_id, prefix): + if len(str(job_id)) > 6: # on production this is the case + nine_digit_id = '0'*(9-len(str(job_id))) + str(job_id) + return os.path.join(prefix, nine_digit_id[:3], nine_digit_id[3:6], str(job_id)) + elif len(str(job_id)) <= 6: + six_digit_id = '0'*(6-len(str(job_id))) + str(job_id) + return os.path.join(prefix, six_digit_id[:3], str(job_id)) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/roles/pg-post-tasks/tasks/main.yml b/roles/pg-post-tasks/tasks/main.yml deleted file mode 100644 index 7be3756a0..000000000 --- a/roles/pg-post-tasks/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- - - name: Set .pgpass file and variables for gxadmin in .bashrc for all users - include: pgpass_bashrc.yml - loop: "{{ machine_users + [{'name': 'ubuntu'}] }}" - loop_control: - loop_var: machine_user - - name: Ensure gxadmin config directory exists for all users including ubuntu - file: - path: "/home/{{ item }}/.config" - state: directory - group: "{{ item }}" - owner: "{{ item }}" - with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" - - name: Ensure that all users + ubuntu have a copy of gxadmin-local.sh - copy: - src: files/galaxy/gxadmin/gxadmin-local.sh - dest: "/home/{{ item }}/.config/gxadmin-local.sh" - mode: "755" - group: "{{ item }}" - owner: "{{ item }}" - force: yes - with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" - diff --git a/roles/pg-post-tasks/tasks/pgpass_bashrc.yml b/roles/pg-post-tasks/tasks/pgpass_bashrc.yml deleted file mode 100644 index c8e89899d..000000000 --- a/roles/pg-post-tasks/tasks/pgpass_bashrc.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- - - name: Set pg_db_role - set_fact: - db_role: "{{ 'galaxy' if machine_user.name == 'ubuntu' else ('reader' if not 'tiaas_admin' in machine_user.roles|d([]) else 'tiaasadmin') }}" - - name: Set pg_db_password - set_fact: - db_password: "{{ pg_db_password[db_role] }}" - - name: Create .pgpass files for users - copy: - dest: "/home/{{ machine_user.name }}/.pgpass" - content: "{{ db_address }}:5432:*:{{ db_role }}:{{ db_password }}" - mode: "600" - group: "{{ machine_user.name }}" - owner: "{{ machine_user.name }}" - - name: Add env vars for gxadmin to machine_users .bashrc files - lineinfile: - path: "/home/{{ machine_user.name }}/.bashrc" - regexp: "^export {{ obj.var }}=" - line: "export {{ obj.var }}='{{ obj.val }}'" - loop: - - var: PGHOST - val: "{{ db_address }}" - - var: PGUSER - val: "{{ db_role }}" - - var: PGDATABASE - val: "galaxy" - - var: PGPORT - val: "5432" - - var: GALAXY_CONFIG_DIR - val: "{{ galaxy_config_dir }}" - - var: GALAXY_MUTABLE_CONFIG_DIR - val: "{{ galaxy_mutable_config_dir }}" - - var: GALAXY_ROOT - val: "{{ galaxy_server_dir }}" - - var: GALAXY_CONFIG_FILE - val: "{{ galaxy_config_dir }}/galaxy.yml" - loop_control: - loop_var: obj diff --git a/staging_playbook.yml b/staging_playbook.yml index c8f27aac2..9f046bfb0 100644 --- a/staging_playbook.yml +++ b/staging_playbook.yml @@ -45,7 +45,6 @@ - acl-on-startup - galaxyproject.gxadmin - dj-wasabi.telegraf - - pg-post-tasks - postfix-mail-relay post_tasks: - name: Ensure object store paths exist