Skip to content

Commit

Permalink
Remove pg-post-tasks and add common bash variable and functions
Browse files Browse the repository at this point in the history
  • Loading branch information
cat-bro committed Dec 22, 2023
1 parent c8a545c commit 98be3b5
Show file tree
Hide file tree
Showing 13 changed files with 153 additions and 104 deletions.
1 change: 0 additions & 1 deletion dev_playbook.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@
# - galaxyproject.s3fs
- galaxyproject.cvmfs
- galaxyproject.gxadmin
- pg-post-tasks
- remote-pulsar-cron
- galaxy-pg-cleanup
#- galaxyproject.tiaas2 # broken in galaxy release_23.1 - missing static/style/base.css
Expand Down
1 change: 0 additions & 1 deletion galaxy-handlers_playbook.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@
- galaxyproject.slurm
- galaxyproject.cvmfs
- galaxyproject.gxadmin
- pg-post-tasks
- geerlingguy.docker
- dj-wasabi.telegraf
- usegalaxy_eu.flower
Expand Down
1 change: 0 additions & 1 deletion galaxy_playbook.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@
- dj-wasabi.telegraf
- geerlingguy.nfs
- galaxyproject.gxadmin
- pg-post-tasks
- postfix-mail-relay
- remote-pulsar-cron
# - delete-tmp-jwds-cron # commented out until galaxy etca is production galaxy, also needs to be enabled
Expand Down
6 changes: 0 additions & 6 deletions group_vars/all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,12 +176,6 @@ galaxy_cvmfs_server_urls:
- "http://cvmfs1-tacc0.galaxyproject.org/cvmfs/@fqrn@"
- "http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@"

#sinfo and squeue configs
bashrc_env: |
export SINFO_FORMAT="%24n %.14C %.8t"
export SQUEUE_FORMAT="%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z"
export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24"
# miniconda
miniconda_version: '4.12.0'
miniconda_mamba_version: '0.23.3'
Expand Down
3 changes: 2 additions & 1 deletion group_vars/galaxyservers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ galaxy_venv_dir: "{{ galaxy_root }}/venv" # set this explicitly as it can fail w
galaxy_mutable_config_dir: "{{ galaxy_root }}/var"
galaxy_mutable_data_dir: "{{ galaxy_root }}"
galaxy_config_dir: "{{ galaxy_root }}/config"
galaxy_job_working_directory: "{{ galaxy_tmp_dir }}/job_working_directory"
galaxy_conda_prefix: "{{ galaxy_tools_indices_dir }}/tool_dependencies/_conda"
galaxy_conda_exec: mamba

Expand Down Expand Up @@ -157,7 +158,7 @@ group_galaxy_config:
check_migrate_tools: false
log_level: TRACE
new_file_path: "{{ galaxy_tmp_dir }}"
job_working_directory: "{{ galaxy_tmp_dir }}/job_working_directory"
job_working_directory: "{{ galaxy_job_working_directory }}"
allow_user_impersonation: true
allow_user_deletion: true
allow_path_paste: true
Expand Down
8 changes: 8 additions & 0 deletions roles/common/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,11 @@ htpass_username: "ubuntu"
internal_ssh_keys: {}

extra_keys: []

common_sinfo_format: "%24n %.14C %.8t"
common_squeue_format: "%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z"
common_sacct_format: "jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24"

common_bashrc_dir: /etc/bashrc_common
common_bashrc_vars_file: "{{ common_bashrc_dir }}/common_variables"
common_bashrc_functions_file: "{{ common_bashrc_dir }}/common_functions"
80 changes: 80 additions & 0 deletions roles/common/tasks/bashrc.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
---
- name: Ensure directory for bashrc common files
file:
state: directory
path: "{{ common_bashrc_dir }}"
- name: Template bashrc_common path
copy:
dest: "{{ common_bashrc_vars_file }}"
mode: 0755
content: |
# slurm
export SINFO_FORMAT="{{ common_sinfo_format }}"
export SQUEUE_FORMAT="{{ common_squeue_format }}"
export SINFO_FORMAT="{{ common_sacct_format }}"
{% if is_galaxy_head_node %}
# galaxy
export PGHOST='{{ db_address }}'
export PGDATABASE='galaxy'
export PGPORT='5432'
export GALAXY_CONFIG_DIR={{ galaxy_config_dir }}
export GALAXY_MUTABLE_CONFIG_DIR={{ galaxy_mutable_config_dir }}
export GALAXY_ROOT={{ galaxy_server_dir }}
export GALAXY_CONFIG_FILE={{ galaxy_config_file }}
export GALAXY_JWD_PATH="{{ galaxy_job_working_directory }}"
{% endif %}
- name: Template common_bashrc path
copy:
dest: "{{ common_bashrc_functions_file }}"
mode: 0755
content: |
{% if is_galaxy_head_node %}
# galaxy
jwd() { # print path of job working directory for job id
python "{{ common_bashrc_dir }}/get_jwd_path_galaxy.py" $1
}
jwd-size() {
du -sh $(jwd $1)
}
go-jwd() { # go to job working directory for job id
cd $(jwd $1)
}
tail-stderr() {
stderr_filename=$(jwd $1)/outputs/tool_stderr
if [ ! -f $stderr_filename ]; then
echo "No stderr file for job $1";
else
tail $stderr_filename;
fi
}
{% endif %}
- name: Template helper scripts
template:
src: "bashrc/{{ item }}.j2"
dest: "{{ common_bashrc_dir }}/{{ item }}"
with_items:
- get_jwd_path_galaxy.py

- name: Add to .bashrc files
blockinfile:
marker: "# {mark} ANSIBLE MANAGED BLOCK (common role: source common bash env for users)"
dest: "/home/{{ item.name }}/.bashrc"
block: |
source {{ common_bashrc_vars_file }}
source {{ common_bashrc_functions_file }}
# local to user
{% if item == 'ubuntu' %}
export PGUSER='galaxy'
{% elif 'tiaas_admin' in item.roles|d([]) %}
export PGUSER='tiaasadmin'
{% else %}
export PGUSER='reader'
{% endif %}
with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}"
62 changes: 31 additions & 31 deletions roles/common/tasks/machine_users.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,6 @@
become_user: root
block:

# FIX for deprecated code that added user files in /etc/sudoers.d/ - can be removed in future
- name: delete /etc/sudoers.d/username files for machine users
file:
path: "/etc/sudoers.d/{{ item.name }}"
state: absent
with_items: "{{ remove_machine_users + machine_users }}"

- name: Ensure group "devs" exists
group:
name: devs
Expand Down Expand Up @@ -65,34 +58,41 @@
force: yes
with_items: "{{ remove_machine_users }}"

- name: Add some env_vars to .bashrcs
blockinfile:
path: "/home/{{ item.name }}/.bashrc"
block: |
{{ bashrc_env }}
marker: "# {mark} MANAGED BY ANSIBLE - DO NOT MODIFY"
with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}"
when: bashrc_env is defined

- name: Add SSH keys for users
authorized_key:
user: "{{ item.name }}"
key: "{{ lookup('file', item.key) }}"
state: present
with_items: "{{ machine_users }}"

- name: Add bash function gxctl for sudo users only
blockinfile:
path: "/home/{{ item.name }}/.bashrc"
block: |
gxctl() {
export gctl_args="$@"
export GRAVITY_STATE_DIR={{ galaxy_gravity_state_dir }}
export GALAXY_CONFIG_FILE={{ galaxy_config_file }}
sudo -H -Eu {{ galaxy_user.name }} bash -c '. {{ galaxy_venv_dir }}/bin/activate && {{ galaxy_venv_dir }}/bin/galaxyctl $gctl_args'
}
marker: "# {mark} MANAGED BY ANSIBLE - DO NOT MODIFY (gxctl)"
state: absent # TODO: Once this is gone from dev/staging/aarnet, remove this task altogether
with_items: "{{ machine_users | selectattr('roles', 'contains', 'sudo') | list + [{'name': 'ubuntu'}] }}"
when: is_galaxy_head_node|d(false)
# former pg-post-tasks tasks
- name: Pgpass creation and gxadmin user tasks
when: is_galaxy_head_node|d(False)
block:
- name: Create .pgpass files for users
copy:
dest: "/home/{{ item.name }}/.pgpass"
content: "{{ db_address }}:5432:*:{{ db_role }}:{{ db_password }}"
mode: "600"
group: "{{ item.name }}"
owner: "{{ item.name }}"
vars:
db_role: "{{ 'galaxy' if item.name == 'ubuntu' else ('reader' if not 'tiaas_admin' in item.roles|d([]) else 'tiaasadmin') }}"
db_password: "{{ pg_db_password[db_role] }}"
with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}"
- name: Ensure gxadmin config directory exists for all users including ubuntu
file:
path: "/home/{{ item }}/.config"
state: directory
group: "{{ item }}"
owner: "{{ item }}"
with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}"
- name: Ensure that all users + ubuntu have a copy of gxadmin-local.sh
copy:
src: files/galaxy/gxadmin/gxadmin-local.sh
dest: "/home/{{ item }}/.config/gxadmin-local.sh"
mode: "755"
group: "{{ item }}"
owner: "{{ item }}"
force: yes
with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}"
7 changes: 6 additions & 1 deletion roles/common/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@

- name: Add machine users
import_tasks: machine_users.yml
when: not skip_machine_users|d(False)

- name: Add variables to bashrc files
import_tasks: bashrc.yml
when: not skip_machine_users|d(False)

- name: Add extra ssh keys
include_tasks: extra_keys.yml
Expand Down Expand Up @@ -122,4 +127,4 @@
lineinfile:
path: /etc/environment
create: true
line: export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24"
line: export SACCT_FORMAT="{{ common_sacct_format }}"
26 changes: 26 additions & 0 deletions roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import argparse
import os

def main():
parser = argparse.ArgumentParser(description='Get job working directory from a job id')
parser.add_argument('job_id', type=int, help='Galaxy job id')
args = parser.parse_args()

prefix = os.environ.get("GALAXY_JWD_PATH")
if not prefix:
raise ValueError("Please set GALAXY_JWD_PATH environment variable")

jwd_path = get_jwd_path(args.job_id, prefix)
print(jwd_path)

def get_jwd_path(job_id, prefix):
if len(str(job_id)) > 6: # on production this is the case
nine_digit_id = '0'*(9-len(str(job_id))) + str(job_id)
return os.path.join(prefix, nine_digit_id[:3], nine_digit_id[3:6], str(job_id))
elif len(str(job_id)) <= 6:
six_digit_id = '0'*(6-len(str(job_id))) + str(job_id)
return os.path.join(prefix, six_digit_id[:3], str(job_id))


if __name__ == '__main__':
main()
23 changes: 0 additions & 23 deletions roles/pg-post-tasks/tasks/main.yml

This file was deleted.

38 changes: 0 additions & 38 deletions roles/pg-post-tasks/tasks/pgpass_bashrc.yml

This file was deleted.

1 change: 0 additions & 1 deletion staging_playbook.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
- acl-on-startup
- galaxyproject.gxadmin
- dj-wasabi.telegraf
- pg-post-tasks
- postfix-mail-relay
post_tasks:
- name: Ensure object store paths exist
Expand Down

0 comments on commit 98be3b5

Please sign in to comment.