preparation to run jobs

README.md

@@ -61,9 +61,9 @@ There are multiple Ansible WebUI products - how do they compare to this product?
 
    It is a single binary and built from Golang (backend) and Node.js/Vue.js (frontend).
 
-   Ansible job [execution is done using shell](https://github.com/ansible-semaphore/semaphore/blob/develop/db_lib/AnsiblePlaybook.go#L57).
+   Ansible job execution is done using [custom implementation](https://github.com/ansible-semaphore/semaphore/blob/develop/db_lib/AnsiblePlaybook.go).
 
-   The project is [managed by a single maintainer and has some issues](https://github.com/ansible-semaphore/semaphore/discussions/1111).
+   The project is [managed by a single maintainer and has some issues](https://github.com/ansible-semaphore/semaphore/discussions/1111). It seems to develop in the direction of large-scale containerized deployments.
 
    The 'Ansible-WebUI' project was inspired by Semaphore.
 
@@ -76,7 +76,7 @@ There are multiple Ansible WebUI products - how do they compare to this product?
 
    The backend stack is built of [gunicorn](https://gunicorn.org/)/[Django](https://www.djangoproject.com/) and the frontend consists of Django templates and JS.
 
-   Ansible job execution is done using the native [Python API](https://ansible.readthedocs.io/projects/runner/en/latest/python_interface/)!
+   Ansible job execution is done using the official [ansible-runner](https://ansible.readthedocs.io/projects/runner/en/latest/python_interface/) library!
 
    Target users are small to medium businesses and Ansible users which just want a UI to run their playbooks.
 

docs/source/usage/2_config.rst

@@ -19,7 +19,7 @@ Environmental variables
 
 * **AW_DB**
 
-   To define the path where the SQLite3 database is placed.
+   Define the path where the SQLite3 database is placed. Default: :code:`${HOME}/.config/ansible-webui/aw.db`
 
 
 * **AW_SECRET**
@@ -28,6 +28,21 @@ Environmental variables
    By default it will be re-generated at service restart.
 
 
+* **AW_PATH_LOG**
+
+   Define the path where full job-logs are saved. Default: :code:`${HOME}/.local/share/ansible-webui/`
+
+
+* **AW_PATH_RUN**
+
+   Base directory for Ansible-Runner runtime files. Default: :code:`/tmp/ansible-webui/`
+
+
+* **AW_PATH_PLAY**
+
+   Path to the [Ansible base/playbook directory](https://docs.ansible.com/ansible/2.8/user_guide/playbooks_best_practices.html#directory-layout). Default: current working directory (*when executing ansible-webui*)
+
+
 * **AW_TIMEZONE**
 
    Override the timezone used.
@@ -59,14 +74,6 @@ Environmental variables
 
    Define the password for the initial admin user.
 
-* **AW_PATH_RUN**
-
-   Base directory for Ansible-Runner runtime files. Default: :code:`/tmp/ansible-webui`
-
-* **AW_PATH_PLAY**
-
-   Path to the [Ansible base/playbook directory](https://docs.ansible.com/ansible/2.8/user_guide/playbooks_best_practices.html#directory-layout). Default: current working directory (*when executing ansible-webui*)
-
 * **AW_RUN_TIMEOUT**
 
    Timeout for the execution of a playbook in seconds. Default: 3600 (1h)

scripts/run_shared.sh

@@ -33,22 +33,22 @@ export DJANGO_SUPERUSER_USERNAME='ansible'
 export DJANGO_SUPERUSER_PASSWORD='automateMe'
 export DJANGO_SUPERUSER_EMAIL='ansible@localhost'
 
-if [[ "$TEST_QUIET" != "1" ]]
-then
-  log 'INSTALLING REQUIREMENTS'
-  python3 -m pip install --upgrade -r ./requirements.txt >/dev/null
-fi
-
 log 'SETTING VERSION'
 bash ./scripts/update_version.sh
 version="$(cat './VERSION')"
 export AW_VERSION="$version"
 
-log 'INITIALIZING DATABASE SCHEMA'
-bash ./scripts/migrate_db.sh "$TEST_MIGRATE"
+if [[ "$TEST_QUIET" != "1" ]]
+then
+  log 'INSTALLING REQUIREMENTS'
+  python3 -m pip install --upgrade -r ./requirements.txt >/dev/null
+
+  log 'INITIALIZING DATABASE SCHEMA'
+  bash ./scripts/migrate_db.sh "$TEST_MIGRATE"
 
-log 'CREATING USERS'
-python3 ./src/ansible-webui/manage.py createsuperuser --noinput || true
+  log 'CREATING USERS'
+  python3 ./src/ansible-webui/manage.py createsuperuser --noinput || true
+fi
 
 log 'STARTING APP'
 python3 ./src/ansible-webui

src/ansible-webui/aw/admin.py

@@ -1,6 +1,9 @@
 from django.contrib import admin
 
-from aw.models import Job, JobPermission
+from aw.models import Job, JobExecution, JobPermission, JobPermissionMemberUser, JobPermissionMemberGroup
 
 admin.site.register(Job)
+admin.site.register(JobExecution)
 admin.site.register(JobPermission)
+admin.site.register(JobPermissionMemberUser)
+admin.site.register(JobPermissionMemberGroup)

src/ansible-webui/aw/config/environment.py

@@ -26,8 +26,10 @@
     'run_isolate_process_path_show': ['AW_RUN_ISOLATE_PS_PATH_SHOW'],
     'run_isolate_process_path_ro': ['AW_RUN_ISOLATE_PS_PATH_RO'],
     'ansible_config': ['ANSIBLE_CONFIG'],
+    'path_log': ['AW_PATH_LOG'],
 }
 
+# todo: move typing to config-init
 AW_ENV_VARS_TYPING = {
     'csv': [
         'run_isolate_process_path_hide', 'run_isolate_process_path_show', 'run_isolate_process_path_ro',
@@ -50,10 +52,13 @@ def _get_existing_ansible_config_file() -> (str, None):
     return None
 
 
+# todo: move static defaults to config-model
 AW_ENV_VARS_DEFAULTS = {
     'run_timeout': 3600,
     'path_run': '/tmp/ansible-webui',
     'path_play': getcwd(),
+    'path_log': f"{environ['HOME']}/.local/share/ansible-webui",
+    'db': f"{environ['HOME']}/.config/ansible-webui",
     'timezone': datetime.now().astimezone().tzname(),
     '_secret': ''.join(random_choice(ascii_letters + digits + punctuation) for _ in range(50)),
     'ansible_config': _get_existing_ansible_config_file(),
@@ -82,6 +87,7 @@ def check_aw_env_var_is_set(var: str) -> bool:
     return get_aw_env_var(var) is not None
 
 
+# only use on edge-cases; as.config.main.check_config_is_true is preferred
 def check_aw_env_var_true(var: str, fallback: bool = False) -> bool:
     val = get_aw_env_var(var)
     if val is None:

src/ansible-webui/aw/config/hardcoded.py

@@ -4,4 +4,5 @@
 LOGIN_PATH = '/a/login/'
 LOGOUT_PATH = '/o/'
 LOG_TIME_FORMAT = '%Y-%m-%d %H:%M:%S %z'
-RUNNER_TMP_DIR_TIME_FORMAT = '%Y-%m-%d_%H-%M-%S'
+SHORT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
+FILE_TIME_FORMAT = '%Y-%m-%d_%H-%M-%S'

src/ansible-webui/aw/config/main.py

@@ -39,6 +39,16 @@ def init_globals():
     if config['timezone'] not in all_timezones:
         config['timezone'] = 'GMT'
 
+    # todo: merge config from webUI
+    # todo: grey-out settings that are provided via env-var in webUI form and show value
+
     environ.setdefault('TZ', config['timezone'])
     config['timezone'] = timezone(config['timezone'])
 
+
+def check_config_is_true(var: str, fallback: bool = False) -> bool:
+    val = config[var]
+    if val is None:
+        return fallback
+
+    return str(val).lower() in ['1', 'true', 'y', 'yes']

src/ansible-webui/aw/execute/play.py

@@ -2,14 +2,22 @@
 from ansible_runner import run as ansible_run
 
 from aw.model.job import Job, JobExecution
-from aw.execute.util import runner_cleanup, runner_prep, parse_run_result
+from aw.execute.util import runner_cleanup, runner_prep, parse_run_result, job_logs
 
 
 def ansible_playbook(job: Job, execution: (JobExecution, None)):
     time_start = datetime.now()
     opts = runner_prep(job=job, execution=execution)
+    logs = job_logs(job=job, execution=execution)
 
-    result = ansible_run(**opts)
+    with (open(logs['stdout'], 'w', encoding='utf-8') as stdout,
+          open(logs['stderr'], 'w', encoding='utf-8') as stderr):
+        result = ansible_run(
+            **opts,
+            runner_mode='subprocess',
+            output_fd=stdout,
+            error_fd=stderr,
+        )
 
     parse_run_result(
         time_start=time_start,

src/ansible-webui/aw/execute/util.py

@@ -1,21 +1,20 @@
-from os import chmod
 from pathlib import Path
 from shutil import rmtree
 from random import choice as random_choice
 from string import digits
 from datetime import datetime
+from re import sub as regex_replace
 
 from ansible_runner import Runner as AnsibleRunner
 
-from aw.config.main import config
-from aw.config.hardcoded import RUNNER_TMP_DIR_TIME_FORMAT
-from aw.config.environment import check_aw_env_var_true, get_aw_env_var, check_aw_env_var_is_set
+from aw.config.main import config, check_config_is_true
+from aw.config.hardcoded import FILE_TIME_FORMAT
 from aw.utils.util import get_choice_key_by_value
 from aw.utils.handlers import AnsibleConfigError
 from aw.model.job import Job, JobExecution, JobExecutionResult, JobExecutionResultHost, CHOICES_JOB_EXEC_STATUS
 
 
-def _decode_env_vars(env_vars_csv: str, src: str) -> dict:
+def _decode_job_env_vars(env_vars_csv: str, src: str) -> dict:
     try:
         env_vars = {}
         for kv in env_vars_csv.split(','):
@@ -48,47 +47,58 @@ def _runner_options(job: Job, execution: JobExecution) -> dict:
     if not path_run.endswith('/'):
         path_run += '/'
 
-    path_run += datetime.now().strftime(RUNNER_TMP_DIR_TIME_FORMAT)
+    path_run += datetime.now().strftime(FILE_TIME_FORMAT)
     path_run += ''.join(random_choice(digits) for _ in range(5))
 
     # merge job + execution env-vars
     env_vars = {}
     if job.environment_vars is not None:
         env_vars = {
             **env_vars,
-            **_decode_env_vars(env_vars_csv=job.environment_vars, src='Job')
+            **_decode_job_env_vars(env_vars_csv=job.environment_vars, src='Job')
         }
 
     if execution.environment_vars is not None:
         env_vars = {
             **env_vars,
-            **_decode_env_vars(env_vars_csv=execution.environment_vars, src='Execution')
+            **_decode_job_env_vars(env_vars_csv=execution.environment_vars, src='Execution')
         }
 
     opts = {
-        'runner_mode': 'pexpect',
         'private_data_dir': path_run,
         'project_dir': config['path_play'],
         'quiet': True,
         'limit': execution.limit if execution.limit is not None else job.limit,
         'envvars': env_vars,
+        'timeout': config['run_timeout'],
     }
 
-    if check_aw_env_var_is_set('run_timeout'):
-        opts['timeout'] = get_aw_env_var('run_timeout')
-
-    if check_aw_env_var_true('run_isolate_dir'):
+    if check_config_is_true('run_isolate_dir'):
         opts['directory_isolation_base_path'] = path_run / 'play_base'
 
-    if check_aw_env_var_true('run_isolate_process'):
+    if check_config_is_true('run_isolate_process'):
         opts['process_isolation'] = True
-        opts['process_isolation_hide_paths'] = get_aw_env_var('run_isolate_process_path_hide')
-        opts['process_isolation_show_paths'] = get_aw_env_var('run_isolate_process_path_show')
-        opts['process_isolation_ro_paths'] = get_aw_env_var('run_isolate_process_path_ro')
+        opts['process_isolation_hide_paths'] = config['run_isolate_process_path_hide']
+        opts['process_isolation_show_paths'] = config['run_isolate_process_path_show']
+        opts['process_isolation_ro_paths'] = config['run_isolate_process_path_ro']
 
     return opts
 
 
+def _create_dirs(path: str, desc: str):
+    try:
+        path = Path(path)
+
+        if not path.is_dir():
+            if not path.parent.is_dir():
+                path.parent.mkdir(mode=0o775)
+
+            path.mkdir(mode=0o750)
+
+    except (OSError, FileNotFoundError):
+        raise OSError(f"Unable to created {desc} directory: '{path}'")
+
+
 def runner_prep(job: Job, execution: (JobExecution, None)) -> dict:
     if execution is None:
         execution = JobExecution(user=None, job=job, comment='Scheduled')
@@ -114,10 +124,8 @@ def runner_prep(job: Job, execution: (JobExecution, None)) -> dict:
         if not Path(pi).exists():
             raise AnsibleConfigError(f"Configured inventory not found: '{pi}'")
 
-    pdd = Path(opts['private_data_dir'])
-    if not pdd.is_dir():
-        pdd.mkdir()
-        chmod(path=pdd, mode=0o750)
+    _create_dirs(path=opts['private_data_dir'], desc='run')
+    _create_dirs(path=config['path_log'], desc='log')
 
     _update_execution_status(execution, status='Running')
 
@@ -128,7 +136,26 @@ def runner_cleanup(opts: dict):
     rmtree(opts['private_data_dir'])
 
 
+def job_logs(job: Job, execution: JobExecution) -> dict:
+    safe_job_name = regex_replace(pattern='[^0-9a-zA-Z-_]+', repl='', string=job.name)
+    if execution.user is None:
+        safe_user_name = 'scheduled'
+    else:
+        safe_user_name = execution.user.replace('.', '_')
+        safe_user_name = regex_replace(pattern='[^0-9a-zA-Z-_]+', repl='', string=safe_user_name)
+
+    timestamp = datetime.now().strftime(FILE_TIME_FORMAT)
+    log_file = f"{config['path_log']}/{safe_job_name}_{timestamp}_{safe_user_name}"
+
+    return {
+        'stdout': f'{log_file}_stdout.log',
+        'stderr': f'{log_file}_stderr.log',
+    }
+
+
 def parse_run_result(execution: JobExecution, time_start: datetime, result: AnsibleRunner):
+    # events = list(result.events)
+
     job_result = JobExecutionResult(
         time_start=time_start,
         failed=result.errored,