database initialization

CONTRIBUTE.md

@@ -34,9 +34,3 @@ python3 -m pip install -r ${REPO}/requirements_test.txt
 bash ${REPO}/scripts/lint.sh
 bash ${REPO}/scripts/test.sh
 ```
-
-## Migrations
-
-Migrations should not be committed to `latest`.
-
-They will be generated & distributed cumulated for each release.

MANIFEST.in

@@ -1,2 +1,2 @@
-recursive-include ansible-webui/*
+recursive-include src/ansible-webui/*
 include requirements.txt

docs/source/usage/2_config.rst

@@ -39,3 +39,30 @@ Environmental variables
    Used in development.
    If unset or value is neither 'dev' nor 'staging' the webservice will be in production mode.
    'staging' mode is close to production behavior.
+
+* **AW_DB_MIGRATE**
+
+   Define to disable automatic database schema-upgrades.
+   After upgrading the module you might have to run the upgrade manually:
+
+   .. code-block:: bash
+
+       python3 -m ansible-webui.manage makemigrations
+       python3 -m ansible-webui.manage makemigrations aw
+       python3 -m ansible-webui.manage migrate
+
+* **AW_ADMIN**
+
+   Define the user-name for the initial admin user.
+
+* **AW_ADMIN_PWD**
+
+   Define the password for the initial admin user.
+
+* **AW_PATH_RUN**
+
+   Base directory for Ansible-Runner runtime files. Default: :code:`/tmp/ansible-webui`
+
+* **AW_PATH_PLAY**
+
+   Path to the [Ansible base/playbook directory](https://docs.ansible.com/ansible/2.8/user_guide/playbooks_best_practices.html#directory-layout). Default: current working directory (*when executing ansible-webui*)

pyproject.toml

@@ -28,3 +28,13 @@ Issues = "https://github.com/ansibleguy/issues"
 [tool.setuptools.dynamic]
 dependencies = {file = ["requirements.txt"]}
 version = {file = ["VERSION"]}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+"*" = ["*.html", "*.js", "*.css", "*.svg"]
+
+# [project.scripts]
+# ansible-webui = "ansible-webui.__main__"
+# ansible-webui-db = "ansible-webui.manage"

scripts/run_pip_build.sh

@@ -18,6 +18,7 @@ echo ''
 
 python3 -m virtualenv "$tmp_venv" >/dev/null
 source "${tmp_venv}/bin/activate"
+export AW_DB="${tmp_venv}/aw.dev.db"
 
 echo ''
 echo 'Building & Installing Module using PIP'

scripts/run_shared.sh

@@ -28,6 +28,7 @@ then
   echo "Creating DB ${TEST_DB}"
 fi
 
+export AW_DB="$TEST_DB"
 export DJANGO_SUPERUSER_USERNAME='ansible'
 export DJANGO_SUPERUSER_PASSWORD='automateMe'
 export DJANGO_SUPERUSER_EMAIL='ansible@localhost'

scripts/update_version.sh

@@ -4,5 +4,6 @@ set -e
 
 cd "$(dirname "$0")/.."
 
-last_tag="$(git describe --exact-match --abbrev=0)"
+#last_tag="$(git describe --exact-match --abbrev=0)"
+last_tag='0.0.1'
 echo "${last_tag}.dev" > "$(dirname "$0")/../VERSION"

src/ansible-webui/aw/config/environment.py

@@ -25,7 +25,7 @@ def get_existing_ansible_config_file() -> str:
         'keys': ['AW_SECRET'],
         'fallback': ''.join(random_choice(ascii_letters + digits + punctuation) for _ in range(50))
     },
-    'path_base': {'keys': ['AW_PATH_BASE'], 'fallback': '/tmp/ansible-webui'},
+    'path_run': {'keys': ['AW_PATH_RUN'], 'fallback': '/tmp/ansible-webui'},
     'path_play': {'keys': ['AW_PATH_PLAY', 'ANSIBLE_PLAYBOOK_DIR'], 'fallback': getcwd()},
     'ansible_config': {'keys': ['ANSIBLE_CONFIG'], 'fallback': get_existing_ansible_config_file()}
 }

src/ansible-webui/aw/config/main.py

@@ -15,6 +15,7 @@ def get_version() -> str:
         return version('ansible-webui')
 
     except PackageNotFoundError:
+        # NOTE: not able to use aw.utils.debug.log_warn because of circular dependency
         stderr.write('\x1b[1;33mWARNING: Module version could not be determined!\x1b[0m\n')
         return '0.0.0'
 

src/ansible-webui/aw/execute/util.py

@@ -43,12 +43,12 @@ def _runner_options(job: Job, execution: JobExecution) -> dict:
     #  limit, verbosity, envvars
 
     # build unique temporary execution directory
-    path_base = config['path_base']
-    if not path_base.endswith('/'):
-        path_base += '/'
+    path_run = config['path_run']
+    if not path_run.endswith('/'):
+        path_run += '/'
 
-    path_base += datetime.now().strftime(RUNNER_TMP_DIR_TIME_FORMAT)
-    path_base += ''.join(random_choice(digits) for _ in range(5))
+    path_run += datetime.now().strftime(RUNNER_TMP_DIR_TIME_FORMAT)
+    path_run += ''.join(random_choice(digits) for _ in range(5))
 
     # merge job + execution env-vars
     env_vars = {}
@@ -66,7 +66,7 @@ def _runner_options(job: Job, execution: JobExecution) -> dict:
 
     return {
         'runner_mode': 'pexpect',
-        'private_data_dir': path_base,
+        'private_data_dir': path_run,
         'project_dir': config['path_play'],
         'quiet': True,
         'limit': execution.limit if execution.limit is not None else job.limit,

src/ansible-webui/aw/settings.py

@@ -6,6 +6,9 @@
 from aw.utils.deployment import deployment_dev, deployment_prod
 
 BASE_DIR = Path(__file__).resolve().parent.parent
+TEMPLATE_DIRS = [
+    BASE_DIR / 'aw' / 'templates/'
+]
 
 DEBUG = deployment_dev()
 ALLOWED_HOSTS = ['*']
@@ -71,7 +74,8 @@
         if not DB_FILE.is_dir():
             raise ValueError(f"Home directory does not exist: '{DB_FILE}'")
 
-    DB_FILE = DB_FILE / 'aw.db'
+    if Path(DB_FILE).is_dir():
+        DB_FILE = DB_FILE / 'aw.db'
 
 else:
     DB_FILE = 'aw.dev.db' if deployment_dev() else 'aw.staging.db'

src/ansible-webui/aw/utils/debug.py

@@ -1,7 +1,8 @@
 from os import getpid
+from sys import stderr, stdout
 
 from aw.utils.util import datetime_w_tz
-from aw.utils.deployment import deployment_dev
+from aw.utils.deployment import deployment_dev, deployment_staging
 from aw.config.hardcoded import LOG_TIME_FORMAT
 
 PID = getpid()
@@ -17,9 +18,33 @@
 }
 
 
+def _log_prefix() -> str:
+    return f'[{datetime_w_tz().strftime(LOG_TIME_FORMAT)}] [{PID}]'
+
+
 def log(msg: str, level: int = 3):
     if level > 5 and not deployment_dev():
         return
 
     # time format adapted to the one used by gunicorn
-    print(f"[{datetime_w_tz().strftime(LOG_TIME_FORMAT)}] [{PID}] [{LEVEL_NAME_MAPPING[level]}] {msg}")
+    print(f"{_log_prefix()} [{datetime_w_tz().strftime(LOG_TIME_FORMAT)}] [{PID}] [{LEVEL_NAME_MAPPING[level]}] {msg}")
+
+
+def log_warn(msg: str, _stderr: bool = False):
+    if _stderr:
+        stderr.write(f'\x1b[1;33m{_log_prefix()} WARNING: {msg}\x1b[0m\n')
+
+    else:
+        stdout.write(f'\x1b[1;33m{_log_prefix()} WARNING: {msg}\x1b[0m\n')
+
+
+def log_error(msg: str):
+    stderr.write(f'\033[01;{_log_prefix()} ERROR: {msg}\x1b[0m\n')
+
+
+def warn_if_development():
+    if deployment_dev():
+        log_warn('Development mode!')
+
+    elif deployment_staging():
+        log_warn('Staging mode!')

src/ansible-webui/aw/utils/deployment.py

@@ -1,5 +1,4 @@
 from os import environ
-from sys import stdout
 
 from aw.config.hardcoded import ENV_KEY_DEPLOYMENT
 
@@ -14,15 +13,3 @@ def deployment_staging() -> bool:
 
 def deployment_prod() -> bool:
     return not deployment_dev() and not deployment_staging()
-
-
-def _print_warn(msg: str):
-    stdout.write('\x1b[1;33mWARNING: ' + msg + '\x1b[0m\n')
-
-
-def warn_if_development():
-    if deployment_dev():
-        _print_warn('Development mode!')
-
-    elif deployment_staging():
-        _print_warn('Staging mode!')

src/ansible-webui/aw/utils/subps.py

@@ -0,0 +1,26 @@
+import subprocess
+
+from aw.settings import BASE_DIR
+
+
+def process(cmd: list, timeout_sec: int = None, shell: bool = False) -> dict:
+    try:
+        with subprocess.Popen(
+            cmd,
+            shell=shell,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=BASE_DIR,
+        ) as p:
+            b_stdout, b_stderr = p.communicate(timeout=timeout_sec)
+            stdout, stderr, rc = b_stdout.decode('utf-8').strip(), b_stderr.decode('utf-8').strip(), p.returncode
+
+    except (subprocess.TimeoutExpired, subprocess.SubprocessError, subprocess.CalledProcessError,
+            OSError, IOError) as error:
+        stdout, stderr, rc = None, str(error), 1
+
+    return dict(
+        stdout=stdout,
+        stderr=stderr,
+        rc=rc,
+    )

src/ansible-webui/base/db.py

@@ -0,0 +1,97 @@
+from pathlib import Path
+from shutil import copy
+from datetime import datetime
+from sys import exit as sys_exit
+from os import environ
+from secrets import choice as random_choice
+from string import digits, ascii_letters
+
+from aw.settings import DB_FILE
+from aw.utils.subps import process
+from aw.utils.debug import log, log_error, log_warn
+
+
+ENV_KEY_INIT_ADMIN_NAME = 'AW_ADMIN'
+ENV_KEY_INIT_ADMIN_PWD = 'AW_ADMIN_PWD'
+
+
+def install_or_migrate_db():
+    if not Path(DB_FILE).is_file():
+        return install()
+
+    return migrate()
+
+
+def _manage_db(action: str, cmd: list, backup: str = None):
+    cmd2 = ['python3', 'manage.py']
+    cmd2.extend(cmd)
+
+    log(msg=f"Executing DB-management command: '{cmd2}'", level=6)
+    result = process(cmd=cmd2)
+
+    if result['rc'] != 0:
+        log_error(f'Database {action} failed!')
+        log(msg=f"Error:\n{result['stderr']}", level=1)
+        log(msg=f"Output:\n{result['stdout']}", level=3)
+
+        if backup is not None:
+            log_warn(
+                msg=f"Trying to restore database from automatic backup: {backup} => {DB_FILE}",
+                _stderr=True,
+            )
+            copy(src=DB_FILE, dst=f'{backup}.failed')
+            copy(src=backup, dst=DB_FILE)
+
+        else:
+            sys_exit(1)
+
+
+def install():
+    log(msg=f"Initializing database {DB_FILE}..", level=3)
+    _make_migrations()
+    _manage_db(action='initialization', cmd=['migrate'])
+
+
+def migrate():
+    _make_migrations()
+
+    backup = f"{DB_FILE}.{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.bak"
+    log(msg=f"Creating database backup: '{backup}'", level=6)
+    copy(src=DB_FILE, dst=backup)
+
+    if 'AW_DB_MIGRATE' not in environ:
+        log(msg=f"Migrating database {DB_FILE}..", level=3)
+        _manage_db(action='migration', cmd=['migrate'], backup=backup)
+
+
+def _make_migrations():
+    _manage_db(action='schema-creation', cmd=['makemigrations'])
+    _manage_db(action='schema-creation', cmd=['makemigrations', 'aw'])
+
+
+def create_first_superuser():
+    from django.contrib.auth.models import User
+    if len(User.objects.filter(is_superuser=True)) == 0:
+        name = 'ansible'
+        pwd = ''.join(random_choice(ascii_letters + digits + '!.-+') for _ in range(14))
+
+        if ENV_KEY_INIT_ADMIN_NAME in environ:
+            name = environ[ENV_KEY_INIT_ADMIN_NAME]
+
+        if ENV_KEY_INIT_ADMIN_PWD in environ:
+            pwd = environ[ENV_KEY_INIT_ADMIN_PWD]
+
+        User.objects.create_superuser(
+            username=name,
+            email=f"{name}@localhost",
+            password=pwd
+        )
+
+        log_warn('No admin was found in the database!')
+        if ENV_KEY_INIT_ADMIN_PWD in environ:
+            log(msg='The user was created as provided!', level=4)
+
+        else:
+            log(msg=f"Generated user: '{name}'", level=3)
+            log(msg=f"Generated pwd: '{pwd}'", level=3)
+            log_warn('Make sure to change the password!')

src/ansible-webui/base/webserver.py

@@ -6,7 +6,8 @@
 from gunicorn.app.wsgiapp import WSGIApplication
 
 from aw.config.hardcoded import PORT_WEB
-from aw.utils.deployment import deployment_dev, warn_if_development
+from aw.utils.deployment import deployment_dev
+from aw.utils.debug import log, warn_if_development
 
 # https://docs.gunicorn.org/en/stable/settings.html
 OPTIONS_DEV = {
@@ -47,7 +48,7 @@ def init_webserver():
         warn_if_development()
         opts = {**opts, **OPTIONS_DEV}
 
-    print(f"Listening on http://{opts['bind']}")
+    log(msg=f"Listening on http://{opts['bind']}", level=5)
 
     StandaloneApplication(
         app_uri="aw.main:app",

src/ansible-webui/main.py

@@ -6,16 +6,22 @@
 
 init_globals()
 
+# pylint: disable=C0415
+from base.db import install_or_migrate_db
+
 
 def main():
     if uname().system.lower() != 'linux':
         raise SystemError('Currently only linux systems are supported!')
 
+    install_or_migrate_db()
+
     django_setup()
 
-    # pylint: disable=C0415
+    from base.db import create_first_superuser
     from base.webserver import init_webserver
     from base.scheduler import init_scheduler
 
+    create_first_superuser()
     init_scheduler()
     init_webserver()