Lab generator and YAML schema docs

.github/workflows/deploy.yml

@@ -36,10 +36,10 @@ jobs:
         run: |
           if git diff --name-only "${{ github.event.workflow_run.head_branch }}" | grep -q -e 'Dockerfile' -e 'requirements.txt'; then
             echo "Dockerfile or requirements.txt changed"
-            echo "::set-output name=updated::true"
+            echo echo "updated=true" >> $GITHUB_OUTPUT
           else
             echo "Dockerfile or requirements.txt not changed"
-            echo "::set-output name=updated::false"
+            echo echo "updated=false" >> $GITHUB_OUTPUT
           fi
 
       - name: Set up Docker Buildx

.gitignore

@@ -2,6 +2,7 @@
 *.sqlite3
 app/app/static/
 app/app/media/
+temp/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

ansible/group_vars/webservers.yml

@@ -40,6 +40,7 @@ create_directories:
   - "{{ config_root }}"
   - "{{ django_root }}/app/media"
   - "{{ django_root }}/app/logs"
+  - "{{ temp_dir }}"
 
 # Admin user login for the web admin
 admin_user:

ansible/roles/galaxy_labs_engine/defaults/main.yml

@@ -11,6 +11,7 @@ nginx_limit_requests_per_minute: 10
 project_root: /home/ubuntu/labs-engine
 config_root: /home/ubuntu/config
 django_root: "{{ project_root }}/app"
+temp_dir: /tmp/labs_engine
 
 labs_engine:
   templates:

ansible/roles/galaxy_labs_engine/tasks/main.yml

@@ -8,6 +8,8 @@
   ansible.builtin.docker_image:
     name: "{{ labs_engine_docker_image }}"
     source: pull
+    force_source: yes  # Ensures it re-checks the remote registry
+    state: present
   tags: update
 
 - name: clone git repository for galaxy-labs-engine

ansible/roles/galaxy_labs_engine/templates/.env.j2

@@ -21,6 +21,8 @@ GITHUB_API_TOKEN={{ github_api_token }}
 LOG_LEVEL_CONSOLE={{ django_log_levels.console|upper }}
 LOG_LEVEL_CACHE={{ django_log_levels.cache|upper }}
 
+TMP_DIR={{ temp_dir }}
+
 {% if django_sentry_dns %}
 SENTRY_DNS={{ django_sentry_dns }}
 {% endif %}

app/app/settings/base.py

@@ -42,6 +42,7 @@
 STATIC_ROOT = BASE_DIR / 'app/static'
 MEDIA_ROOT = BASE_DIR / 'app/media'
 LOG_ROOT = ensure_dir(BASE_DIR / 'app/logs')
+TEMP_DIR = ensure_dir(os.getenv('TMP_DIR', '/tmp/labs_engine/'))
 DEFAULT_EXPORTED_LAB_CONTENT_ROOT = (
     f'http://{HOSTNAME}/static/labs/content/docs/base.yml')
 
@@ -97,8 +98,13 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'labs',
+    'crispy_forms',
+    "crispy_bootstrap5",
 ]
 
+CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap5"
+CRISPY_TEMPLATE_PACK = "bootstrap5"
+
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',

app/app/settings/prod.py

@@ -28,8 +28,15 @@
     ]
 
 # Use manifest to manage static file versions for cache busting:
-STATICFILES_STORAGE = ('django.contrib.staticfiles.storage'
-                       '.ManifestStaticFilesStorage')
+STORAGES = {
+    "default": {
+        "BACKEND": "django.core.files.storage.FileSystemStorage",
+    },
+    "staticfiles": {
+        "BACKEND": ('django.contrib.staticfiles.storage'
+                    '.ManifestStaticFilesStorage'),
+    },
+}
 
 SENTRY_DNS = os.getenv('SENTRY_DNS')
 if SENTRY_DNS:

app/labs/bootstrap.py

@@ -0,0 +1,116 @@
+"""Render a new lab from form data."""
+
+import random
+import shutil
+import string
+import time
+import zipfile
+from django.conf import settings
+from django.template.loader import render_to_string
+from django.utils.text import slugify
+from pathlib import Path
+
+HOURS_1 = 60 * 60
+ALPHANUMERIC = string.ascii_letters + string.digits
+TEMPLATE_DIR = Path('labs/bootstrap')
+TEMPLATES_TO_RENDER = [
+    'base.yml',
+    'intro.md',
+    'conclusion.md',
+    'footer.md',
+    'section-1.yml',
+    'README.md',
+    'custom.css',
+    'CONTRIBUTORS',
+]
+GALAXY_SERVERS = {
+    '': 'usegalaxy.org',
+    'Europe': 'usegalaxy.eu',
+    'Australia': 'usegalaxy.org.au',
+}
+
+
+def random_string(length):
+    return ''.join(random.choices(ALPHANUMERIC, k=length))
+
+
+def lab(form_data):
+    """Render a new lab from form data."""
+    clean_dir(settings.TEMP_DIR)
+    output_dir = settings.TEMP_DIR / random_string(6)
+    form_data['logo_filename'] = create_logo(form_data, output_dir)
+    render_templates(form_data, output_dir)
+    render_server_yml(form_data, output_dir)
+    zipfile_path = output_dir.with_suffix('.zip')
+    root_dir = Path(slugify(form_data['lab_name']))
+    with zipfile.ZipFile(zipfile_path, 'w') as zf:
+        for path in output_dir.rglob('*'):
+            zf.write(path, root_dir / path.relative_to(output_dir))
+    return zipfile_path
+
+
+def render_templates(data, output_dir):
+    for template in TEMPLATES_TO_RENDER:
+        subdir = None
+        if template.endswith('.md') and 'README' not in template:
+            subdir = 'templates'
+        elif template.endswith('css'):
+            subdir = 'static'
+        elif template == 'section-1.yml':
+            subdir = 'sections'
+        render_file(
+            data,
+            template,
+            output_dir,
+            subdir=subdir,
+        )
+
+
+def render_file(data, template, output_dir, subdir=None, filename=None):
+    outfile_relpath = filename or template
+    if subdir:
+        outfile_relpath = Path(subdir) / outfile_relpath
+    path = output_dir / outfile_relpath
+    path.parent.mkdir(parents=True, exist_ok=True)
+    content = render_to_string(TEMPLATE_DIR / template, data)
+    path.write_text(content)
+    return path
+
+
+def render_server_yml(data, output_dir):
+    """Create a YAML file for each Galaxy server."""
+    for site_name, root_domain in GALAXY_SERVERS.items():
+        data['site_name'] = site_name
+        data['galaxy_base_url'] = (
+            'https://'
+            + data['subdomain']
+            + '.'
+            + root_domain
+        )
+        data['root_domain'] = root_domain
+        render_file(
+            data,
+            'server.yml',
+            output_dir,
+            filename=f'{root_domain}.yml',
+        )
+
+
+def create_logo(data, output_dir):
+    """Copy the uploaded logo to the output directory."""
+    logo_file = data.get(
+        'logo'
+    ) or settings.BASE_DIR / 'labs/example_labs/docs/static/flask.svg'
+    logo_dest_path = output_dir / 'static' / logo_file.name
+    logo_dest_path.parent.mkdir(parents=True, exist_ok=True)
+    with logo_file.open('rb') as src, logo_dest_path.open('wb') as dest:
+        shutil.copyfileobj(src, dest)
+    return logo_dest_path.name
+
+
+def clean_dir(directory):
+    """Delete directories that were created more than 7 days ago."""
+    for path in directory.iterdir():
+        if path.is_dir() and path.stat().st_ctime < time.time() - HOURS_1:
+            shutil.rmtree(path)
+    return directory

app/labs/forms.py

@@ -1,14 +1,30 @@
 """User facing forms for making support requests (help/tools/data)."""
 
 import logging
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import Layout, HTML, Submit
 from django import forms
 from django.conf import settings
 from django.core.mail import EmailMultiAlternatives
+from django.core.validators import FileExtensionValidator
 from utils.mail import retry_send_mail
+from utils.webforms import SpamFilterFormMixin
+
+from . import bootstrap, validators
 
 logger = logging.getLogger('django')
 
 MAIL_APPEND_TEXT = f"Sent from {settings.HOSTNAME}"
+IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg']
+INTRO_MD = 'Welcome to the Galaxy {{ site_name }} {{ lab_name }}!'
+CONCLUSION_MD = ('Thanks for checking out the Galaxy {{ site_name }}'
+                 ' {{ lab_name }}!')
+FOOTER_MD = f"""
+<footer class="text-center">
+    This page was generated by the
+    <a href="{settings.HOSTNAME}/bootstrap">Galaxy Labs Engine</a>.
+</footer>
+"""
 
 
 def dispatch_form_mail(
@@ -80,3 +96,99 @@ def dispatch(self, subject=None):
                 + data['message']
             )
         )
+
+
+class LabBootstrapForm(SpamFilterFormMixin, forms.Form):
+    """Form to bootstrap a new lab."""
+
+    lab_name = forms.CharField(
+        label="Lab name",
+        widget=forms.TextInput(attrs={
+            'placeholder': "e.g. Genome Lab",
+            'autocomplete': 'off',
+        }),
+    )
+    subdomain = forms.CharField(
+        label="Galaxy Lab Subdomain",
+        widget=forms.TextInput(attrs={
+            'placeholder': "e.g. genome",
+            'autocomplete': 'off',
+        }),
+        help_text=(
+            "The subdomain that the lab will be served under. i.e."
+            " &lt;subdomain&gt;.usegalaxy.org"
+        ),
+    )
+    github_username = forms.CharField(
+        label="GitHub Username",
+        widget=forms.TextInput(attrs={
+            'autocomplete': 'off',
+        }),
+        help_text=(
+            '(Optional) Your GitHub username to add to the list of'
+            ' contributors.'
+        ),
+        validators=[validators.validate_github_username],
+        required=False,
+    )
+    logo = forms.FileField(
+        label="Lab logo",
+        help_text=("(Optional) Upload a custom logo to be displayed in the Lab"
+                   " header. Try to make it square and less than 100KB"
+                   " - SVG format is highly recommended."),
+        required=False,
+        allow_empty_file=False,
+        validators=[
+            FileExtensionValidator(
+                allowed_extensions=IMAGE_EXTENSIONS,
+            ),
+        ],
+        widget=forms.FileInput(attrs={
+            'accept': ','.join(f"image/{ext}" for ext in IMAGE_EXTENSIONS),
+        }),
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.helper = FormHelper()
+        self.helper.layout = Layout(
+            'lab_name',
+            'subdomain',
+            'github_username',
+            'logo',
+            HTML(self.antispam_html),
+            Submit('submit', 'Build'),
+        )
+
+    def clean_subdomain(self):
+        """Validate the subdomain."""
+        subdomain = self.cleaned_data['subdomain'].lower().strip()
+        if not subdomain.isalnum():
+            raise forms.ValidationError("Subdomain must be alphanumeric.")
+        return subdomain
+
+    def clean_lab_name(self):
+        """Validate the lab name."""
+        lab_name = self.cleaned_data['lab_name'].title().strip()
+        if not lab_name.replace(' ', '').isalnum():
+            raise forms.ValidationError("Lab name cannot be empty.")
+        return lab_name
+
+    def clean_github_username(self):
+        username = self.cleaned_data.get('github_username')
+        return username.strip() if username else None
+
+    def bootstrap_lab(self):
+        data = self.cleaned_data
+        data.update({
+            'intro_md': INTRO_MD,  # TODO: Render from user-input
+            'conclusion_md': CONCLUSION_MD,
+            'footer_md': FOOTER_MD,
+            'root_domain': 'usegalaxy.org',
+            'galaxy_base_url': (
+                f"https://{data['subdomain']}.usegalaxy.org"),
+            'section_paths': [
+                'sections/section-1.yml',  # TODO: Auto-render from user input
+            ],
+        })
+        return bootstrap.lab(data)

app/labs/static/labs/content/docs/section_1.yml

@@ -43,11 +43,11 @@ tabs:
 
   - id: workflows
     title: Workflows
-    heading_md: |
+    heading_md: >
       A workflow is a series of Galaxy tools that have been linked together
       to perform a specific analysis. You can use and customize the example workflows
       below.
-      [Learn more](https://galaxyproject.org/learn/advanced-workflow/).
+      <a href="https://galaxyproject.org/learn/advanced-workflow">Learn more</a>.
     content:
       - button_link: "{{ galaxy_base_url }}/workflows/trs_import?trs_server=workflowhub.eu&run_form=true&trs_id=222"
         button_tip: Import to Galaxy AU