Merge pull request #153 from galaxyproject/gravity

Support for running Galaxy from systemd using Gravity (galaxyctl)

.github/workflows/ci.yml

@@ -5,6 +5,11 @@ on: [pull_request]
 
 jobs:
   setup:
+    strategy:
+      matrix:
+        galaxy-version:
+          - 21.01
+          - 22.01
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
@@ -14,13 +19,13 @@ jobs:
       - name: Install dependencies
         run: |
             sudo apt-get install git python3 python3-virtualenv
-            pip3 install 'ansible<2.10'
+            pip3 install 'ansible<2.11'
             printf '[defaults]\nroles_path = ../\ninterpreter_python = /usr/bin/python3' > ansible.cfg
 
       - name: Run Test
         run: |
             export ANSIBLE_STDOUT_CALLBACK=yaml
-            ansible-playbook -i "localhost, " -c local tests/test_playbook.yml
+            ansible-playbook -i "localhost, " -c local tests/test_playbook.yml -e "galaxy_commit_id=release_${{ matrix.galaxy-version }}"
             systemctl status galaxy || true
             journalctl -u galaxy
             sleep 10

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
 __pycache__
+*.swp

README.md

@@ -95,6 +95,16 @@ Options below that control individual file or subdirectory placement can still o
 [custom]: vars/layout-custom.yml
 [fhs]: http://www.pathname.com/fhs/
 
+**New options for Galaxy 22.01 and later**
+
+The role can now manage the Galaxy service using [gravity][gravity]. This is the default for Galaxy 22.05 and later.
+Additionally, support for the `galaxy_restart_handler_name` variable has been removed. If you need to enable your own
+custom restart handler, you can use the "`listen`" option to the handler as explained in the
+[handler documentation](https://docs.ansible.com/ansible/latest/user_guide/playbooks_handlers.html#using-variables-with-handlers).
+The handler should "listen" to the topic `"restart galaxy"`.
+
+[gravity]: https://github.com/galaxyproject/gravity
+
 **New options for Galaxy 18.01 and later**
 
 - `galaxy_config_style` (default: `yaml`): The type of Galaxy configuration file to write, `yaml` for the YAML format supported by uWSGI or `ini-paste` for the traditional PasteDeploy-style INI file
@@ -223,10 +233,13 @@ Several variables control which functions this role will perform (all default to
 - `galaxy_build_client`: Build the Galaxy client application (web UI).
 - `galaxy_client_make_target` (default: `client-production-maps`): Set the client build type. Options include: `client`,
   `client-production` and `client-production-maps`. See [Galaxy client readme][client-build] for details.
+- `galaxy_manage_systemd` (default: `no`): Install a [systemd][systemd] service unit to start and stop Galaxy with the
+  system (and using the `systemctl` command).
 - `galaxy_manage_errordocs` (default: `no`): Install Galaxy-styled 413 and 502 HTTP error documents for nginx. Requires
   write privileges for the nginx error document directory.
 
 [client-build]: https://github.com/galaxyproject/galaxy/blob/dev/client/README.md#complete-client-build
+[systemd]: https://www.freedesktop.org/wiki/Software/systemd/
 
 **Galaxy code and configuration**
 
@@ -323,6 +336,23 @@ The role needs to perform tasks as different users depending on which features y
 connecting to the target host. By default, the role will use `become` (i.e. sudo) to perform tasks as the appropriate
 user if deemed necessary. Overriding this behavior is discussed in the [defaults file][defaults].
 
+**systemd**
+
+[systemd][systemd] is the standard system init daemon on most modern Linux flavors (and all of the ones supported by
+this role). If `galaxy_manage_systemd` is enabled, a `galaxy` service will be configured in systemd to run Galaxy. This
+service will be automatically started and configured to start when your system boots. You can control the Galaxy
+service with the `systemctl` utility as the `root` user or with `sudo`:
+
+```console
+# systemctl start galaxy     # start galaxy
+# systemctl reload galaxy    # attempt a "graceful" reload
+# systemctl restart galaxy   # perform a hard restart
+# systemctl stop galaxy      # stop galaxy
+```
+
+You can use systemd user mode if you do not have root privileges on your system by setting `galaxy_systemd_root` to
+`false`. Add `--user` to the `systemctl` commands above to interact with systemd in user mode:
+
 **Error documents**
 
 - `galaxy_errordocs_dir`: Install Galaxy-styled HTTP 413 and 502 error documents under this directory. The 502 message
@@ -334,9 +364,6 @@ user if deemed necessary. Overriding this behavior is discussed in the [defaults
 
 **Miscellaneous options**
 
-- `galaxy_restart_handler_name`: The role doesn't restart Galaxy since it doesn't control how Galaxy is started and
-  stopped. Because of this, you can write your own restart handler and inform the role of your handler's name with this
-  variable. See the examples
 - `galaxy_admin_email_to`: If set, email this address when Galaxy has been updated. Assumes mail is properly configured
   on the managed host.
 - `galaxy_admin_email_from`: Address to send the aforementioned email from.
@@ -398,7 +425,6 @@ Install Galaxy as per the current production server best practices:
     galaxy_user: galaxy
     galaxy_privsep_user: gxpriv
     galaxy_group: galaxy
-    galaxy_restart_handler_name: Restart Galaxy
     postgresql_objects_users:
       - name: galaxy
         password: null
@@ -453,6 +479,7 @@ Install Galaxy as per the current production server best practices:
       supervisorctl:
         name: galaxy
         state: restarted
+      listen: restart galaxy
 ```
 
 License

defaults/main.yml

@@ -19,6 +19,7 @@ galaxy_fetch_dependencies: yes
 galaxy_build_client: yes
 galaxy_manage_errordocs: no
 galaxy_backup_configfiles: yes
+galaxy_manage_gravity: "{{ false if __galaxy_major_version is version('22.05', '<') else true }}"
 galaxy_manage_systemd: no # For Galaxy
 galaxy_manage_systemd_reports: no # For Reports
 
@@ -90,6 +91,9 @@ __galaxy_privsep_user_name: "{{ galaxy_privsep_user.name | default(galaxy_privse
 # If `galaxy_create_user` is enabled, the privsep user will also be created, but you can override this behavior
 galaxy_create_privsep_user: "{{ galaxy_create_user if __galaxy_privsep_user_name != 'root' else false }}"
 
+# Where to write Galaxy logs to
+galaxy_log_dir: "{{ galaxy_mutable_data_dir }}/log"
+
 # Directories to create as the Galaxy user if galaxy_manage_paths is enabled
 galaxy_dirs:
   - "{{ galaxy_mutable_data_dir }}"
@@ -100,6 +104,7 @@ galaxy_dirs:
   - "{{ galaxy_file_path }}"
   - "{{ galaxy_job_working_directory }}"
   - "{{ galaxy_tool_data_path }}"
+  - "{{ galaxy_log_dir }}"
 
 # Directories to create as the privilege separated user if galaxy_manage_paths is enabled
 galaxy_privsep_dirs:
@@ -242,6 +247,16 @@ galaxy_errordocs_prefix: /error
 galaxy_config_files: []
 galaxy_config_templates: []
 
+# Default Gravity configuration
+galaxy_gravity_state_dir: "{{ (galaxy_mutable_data_dir, 'gravity') | path_join }}"
+galaxy_gravity_config_default:
+  galaxy_root: "{{ galaxy_server_dir }}"
+  log_dir: "{{ galaxy_log_dir }}"
+  virtualenv: "{{ galaxy_venv_dir }}"
+  app_server: gunicorn
+  gunicorn:
+    bind: localhost:8080
+
 # Default uWSGI configuration
 galaxy_uwsgi_yaml_parser: internal
 galaxy_uwsgi_config_default:
@@ -287,15 +302,24 @@ galaxy_client_build_steps:
     - stageLibs
     - plugins
 
-# Restart handler is already set if using the systemd options in this role, otherwise you need to define one yourself
-galaxy_restart_handler_name: "{{ 'galaxy mule restart' if (galaxy_systemd_mode == 'mule' and galaxy_manage_systemd) else 'default restart galaxy handler' }}"
-
 #
 # systemd options
 #
 
-# Currently `mule` is the only systemd mode
-galaxy_systemd_mode: mule
+# Currently `mule` (aka uWSGI) and `gravity` (runs Galaxy 22.01+ under gunicorn) are supported
+galaxy_systemd_mode: "{{ 'mule' if __galaxy_major_version is version('22.05', '<') else 'gravity' }}"
+
+# Control whether the systemd service unit is installed/controlled as root or the Galaxy user
+galaxy_systemd_root: true
+
+# The command that systemd will run to start Galaxy, depends on galaxy_systemd_mode or override
+galaxy_systemd_command: "{{ __galaxy_systemd_command[galaxy_systemd_mode] }}"
+__galaxy_systemd_command:
+  mule: "{{ galaxy_venv_dir }}/bin/uwsgi {{ '--yaml' if galaxy_config_style in ('yaml', 'yml') else '--ini' }} {{ galaxy_config_file }}"
+  gravity: "{{ galaxy_venv_dir }}/bin/galaxyctl start --foreground --quiet"
+
+# TimeoutStartSec is unfortunately also the timeout used for reload operations, which take longer
+galaxy_systemd_timeout_start_sec: "{{ (galaxy_systemd_mode == 'gravity') | ternary(60, 10) }}"
 
 # Systemd will create cgroups to enforce these limits (sizes in GB), override individual limits by setting
 # `galaxy_systemd_memory_limit.<limit>` in your vars.

handlers/main.yml

@@ -1,29 +1,36 @@
 ---
 # defaults file for galaxyproject.galaxy
 
-# TODO: Magical restarters depending on how you're running Galaxy.
-- name: default restart galaxy handler
-  debug:
-    msg: "RESTARTER NOT IMPLEMENTED - Please restart Galaxy manually. You can define your own handler and enable it with `galaxy_restart_handler_name`"
-
-- name: email administrator with commit id
-  mail:
-    from: "{{ galaxy_admin_email_from | default( 'root' ) }}"
-    to: "{{ galaxy_admin_email_to }}"
-    subject: "Galaxy instance {{ galaxy_instance_hostname | default( inventory_hostname ) }} is at commit {{ galaxy_commit_id }}"
-    body: " "
-  when: galaxy_admin_email_to is defined
-
-- name: daemon reload
+- name: systemd daemon reload
   systemd:
     daemon_reload: yes
+    scope: "{{ galaxy_systemd_root | ternary(omit, 'user') }}"
 
-- name: galaxy mule start
+- name: galaxy systemd start
   systemd:
-      name: galaxy.service
-      state: started
+    name: galaxy.service
+    state: started
+    scope: "{{ galaxy_systemd_root | ternary(omit, 'user') }}"
+  when: "galaxy_systemd_mode == 'gravity'"
+  listen: "start galaxy"
 
 - name: galaxy mule restart
   systemd:
-      name: galaxy.service
-      state: restarted
+    name: galaxy.service
+    state: restarted
+    scope: "{{ galaxy_systemd_root | ternary(omit, 'user') }}"
+  when: "galaxy_systemd_mode == 'mule'"
+  listen: "restart galaxy"
+
+- name: galaxy gravity restart
+  command: "{{ galaxy_venv_dir }}/bin/galaxyctl graceful"
+  environment:
+    GRAVITY_STATE_DIR: "{{ galaxy_gravity_state_dir }}"
+  when: "galaxy_systemd_mode == 'gravity'"
+  listen: "restart galaxy"
+
+- name: galaxyctl update
+  command: "{{ galaxy_venv_dir }}/bin/galaxyctl update"
+  environment:
+    GRAVITY_STATE_DIR: "{{ galaxy_gravity_state_dir }}"
+  when: "galaxy_systemd_mode == 'gravity'"

meta/main.yml

@@ -6,7 +6,7 @@ galaxy_info:
   description: Install and manage a Galaxy (http://galaxyproject.org/) server.
   company: The Galaxy Project
   license: AFL v3.0
-  min_ansible_version: 2.7
+  min_ansible_version: 2.10
   github_branch: main
   platforms:
     - name: EL

tasks/clone.yml

@@ -15,8 +15,7 @@
       diff: "{{ galaxy_diff_mode_verbose }}"
       register: __galaxy_git_update_result
       notify:
-        - "{{ galaxy_restart_handler_name }}"
-        - email administrator with commit id
+        - restart galaxy
 
     - name: Report Galaxy version change
       debug:

tasks/dependencies.yml

@@ -46,6 +46,15 @@
         VIRTUAL_ENV: "{{ galaxy_venv_dir }}"
       when: galaxy_additional_venv_packages
 
+    - name: Configure $GRAVITY_STATE_DIR in virtualenv activate
+      blockinfile:
+        path: "{{ galaxy_venv_dir }}/bin/activate"
+        marker: "# {mark} GALAXYPROJECT.GALAXY ROLE ANSIBLE MANAGED BLOCK"
+        block: |
+          # Galaxy Gravity per-instance state directory configured by galaxyproject.galaxy Ansible role
+          GRAVITY_STATE_DIR={{ galaxy_gravity_state_dir | quote }}
+          export GRAVITY_STATE_DIR
+
   remote_user: "{{ galaxy_remote_users.privsep | default(__galaxy_remote_user) }}"
   become: "{{ true if galaxy_become_users.privsep is defined else __galaxy_become }}"
   become_user: "{{ galaxy_become_users.privsep | default(__galaxy_become_user) }}"

tasks/gravity.yml

@@ -0,0 +1,21 @@
+---
+# Configure Gravity
+
+- name: Gravity setup
+  block:
+
+    - name: Register Galaxy config with Gravity
+      command: "{{ galaxy_venv_dir }}/bin/galaxyctl register {{ galaxy_config_file }}"
+      args:
+        creates: "{{ galaxy_gravity_state_dir }}/configstate.yaml"
+
+    - name: Update Gravity process management files
+      command: "{{ galaxy_venv_dir }}/bin/galaxyctl update"
+      args:
+        creates: "{{ galaxy_gravity_state_dir }}/supervisor/supervisord.conf.d"
+
+  environment:
+    GRAVITY_STATE_DIR: "{{ galaxy_gravity_state_dir }}"
+  remote_user: "{{ galaxy_remote_users.galaxy | default(__galaxy_remote_user) }}"
+  become: "{{ true if galaxy_become_users.galaxy is defined else __galaxy_become }}"
+  become_user: "{{ galaxy_become_users.galaxy | default(__galaxy_become_user) }}"

tasks/main.yml

@@ -35,6 +35,10 @@
   tags:
     - galaxy_create_user
 
+- name: Collect Galaxy user facts
+  import_tasks: user-facts.yml
+  tags: always
+
 - name: Include path management tasks
   include_tasks:
     file: paths.yml
@@ -121,12 +125,21 @@
   tags:
     - galaxy_manage_errordocs
 
-- name: Include systemd unit setup tasks (Mules)
+- name: Include Gravity setup tasks
+  include_tasks:
+    file: gravity.yml
+    apply:
+      tags: galaxy_manage_gravity
+  when: galaxy_manage_gravity
+  tags:
+    - galaxy_manage_gravity
+
+- name: Include systemd unit setup tasks (Galaxy)
   include_tasks:
-    file: systemd-mule.yml
+    file: systemd-galaxy.yml
     apply:
       tags: galaxy_manage_systemd
-  when: galaxy_manage_systemd and galaxy_systemd_mode == "mule"
+  when: galaxy_manage_systemd and galaxy_systemd_mode in ["mule", "gravity"]
   tags:
     - galaxy_manage_systemd
 

tasks/paths.yml

@@ -3,35 +3,6 @@
 - name: Manage Paths
   block:
 
-    - name: Get group IDs for Galaxy users
-      getent:
-        database: passwd
-        key: "{{ item }}"
-      with_items:
-        - "{{ __galaxy_user_name }}"
-        - "{{ __galaxy_privsep_user_name }}"
-      when: galaxy_group is not defined
-      register: __galaxy_passwd_result
-
-    - name: Get group names for Galaxy users
-      getent:
-        database: group
-        key: "{{ item.ansible_facts.getent_passwd[item.invocation.module_args.key][2] }}"
-      with_items: "{{ __galaxy_passwd_result.results }}"
-      loop_control:
-        label: "{{ item.item }}"
-      when: galaxy_group is not defined
-      register: __galaxy_group_result
-
-    - name: Set Galaxy user facts
-      set_fact:
-        __galaxy_user_group: "{{ ((galaxy_group | default({})).name | default(galaxy_group)) if galaxy_group is defined else (__galaxy_group_result.results[0].ansible_facts.getent_group.keys() | first) }}"
-        __galaxy_privsep_user_group: "{{ ((galaxy_group | default({})).name | default(galaxy_group)) if galaxy_group is defined else (__galaxy_group_result.results[1].ansible_facts.getent_group.keys() | first) }}"
-
-    - name: Determine whether to restrict to group permissions
-      set_fact:
-        __galaxy_dir_perms: "{{ '0750' if __galaxy_user_group == __galaxy_privsep_user_group else '0755' }}"
-
     # try to become root. if you need something fancier, don't use the role and write your own
     - name: Create galaxy_root
       file:

tasks/static_setup.yml

@@ -16,7 +16,7 @@
         mode: "{{ galaxy_config_perms }}"
       with_items: "{{ galaxy_config_files }}"
       notify:
-        - "{{ galaxy_restart_handler_name }}"
+        - restart galaxy
 
     - name: Install additional Galaxy config files (template)
       template:
@@ -26,7 +26,7 @@
         mode: "{{ galaxy_config_perms }}"
       with_items: "{{ galaxy_config_templates }}"
       notify:
-        - "{{ galaxy_restart_handler_name }}"
+        - restart galaxy
 
     - name: Install local tools
       copy:
@@ -127,7 +127,8 @@
         backup: "{{ galaxy_backup_configfiles }}"
         mode: "{{ galaxy_config_perms }}"
       notify:
-        - "{{ galaxy_restart_handler_name }}"
+        - galaxyctl update
+        - restart galaxy
 
   remote_user: "{{ galaxy_remote_users.privsep | default(__galaxy_remote_user) }}"
   become: "{{ true if galaxy_become_users.privsep is defined else __galaxy_become }}"