From 316f317a3a024dfc67c68be6df872bbf701bfde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Sun, 21 Jul 2024 01:19:53 +0200 Subject: [PATCH 1/5] rewrite to flask --- astrid.service | 14 --- build.sh | 4 - config.ini.sample | 14 --- config.toml.sample | 38 +++++++ doc/initscript | 40 ------- docker/Dockerfile | 4 + docker/docker-compose.yml | 4 +- docker/entrypoint.sh | 10 +- main | 36 ------- repos.ini.sample | 6 -- repos.toml.sample | 12 +++ requirements.txt | 5 +- setup.cfg | 2 - setup.py | 46 -------- src/app.py | 114 ++++++++++++++++++++ src/gunicorn.conf.py | 12 +++ src/logger.py | 61 +++++++++++ src/oauth.py | 52 +++++++++ src/repository.py | 168 ++++++++++++++++++++++++++++++ src/static/style.css | 3 + src/templates/base.html.jinja | 15 +++ src/templates/buildlog.html.jinja | 8 ++ src/templates/index.html.jinja | 14 +++ src/templates/info.html.jinja | 25 +++++ src/templates/listdir.html.jinja | 35 +++++++ src/templates/signin.html.jinja | 8 ++ 26 files changed, 583 insertions(+), 167 deletions(-) delete mode 100644 astrid.service delete mode 100755 build.sh delete mode 100644 config.ini.sample create mode 100644 config.toml.sample delete mode 100644 doc/initscript delete mode 100755 main delete mode 100644 repos.ini.sample create mode 100644 repos.toml.sample delete mode 100644 setup.cfg delete mode 100644 setup.py create mode 100644 src/app.py create mode 100644 src/gunicorn.conf.py create mode 100644 src/logger.py create mode 100644 src/oauth.py create mode 100644 src/repository.py create mode 100644 src/static/style.css create mode 100644 src/templates/base.html.jinja create mode 100644 src/templates/buildlog.html.jinja create mode 100644 src/templates/index.html.jinja create mode 100644 src/templates/info.html.jinja create mode 100644 src/templates/listdir.html.jinja create mode 100644 src/templates/signin.html.jinja diff --git a/astrid.service b/astrid.service deleted file mode 100644 index 199cdbf..0000000 --- a/astrid.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Astrid minimalistic content integration server -After=network-online.target -Requires=network-online.target - -[Service] -ExecStart=/network/home/astrid/.local/bin/astrid -User=astrid -Restart=on-failure -; Give user some time to fix wrong configuration -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/build.sh b/build.sh deleted file mode 100755 index 857c552..0000000 --- a/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -VERSION=`git describe HEAD --tags` -python3 setup.py $VERSION bdist_egg diff --git a/config.ini.sample b/config.ini.sample deleted file mode 100644 index a187c1f..0000000 --- a/config.ini.sample +++ /dev/null @@ -1,14 +0,0 @@ -[global] -log.screen = False -server.socket_port = 8080 -server.socket_host = '0.0.0.0' -server.thread_pool = 10 -tools.auth_basic.on: True -tools.auth_basic.realm: 'Astrid authorization' -tools.auth_basic.checkpassword: cherrypy.lib.auth_basic.checkpassword_dict({'user': 'passwd'}) -tools.expires.sec = 0 -repodir = '/data/repos' - -[/style.css] -tools.staticfile.on = True -tools.staticfile.filename = os.path.join(os.path.abspath("."), "astrid/static/style.css") diff --git a/config.toml.sample b/config.toml.sample new file mode 100644 index 0000000..fd6c87c --- /dev/null +++ b/config.toml.sample @@ -0,0 +1,38 @@ +# Flask config file +# +# This file is loaded by flask to app.config. +# It can overload flask configuration (https://flask.palletsprojects.com/en/3.0.x/config/) +# and it load custom config values for the app. +# All variable names must be upper case for flask to load them. +# + +# Flask debug mode +# set to true for development +DEBUG = false + +# Flask tests +# set to true for development +TESTING = false + +# Secret key used for session encryption +# set to long random string for security +SECRET_KEY = '' # CHANGE THIS! + +[OAUTH] + +# Client ID created in OIDC provider (keycloak) +CLIENT_ID = '' + +# Client secret from OIDC provider (keycloak) +CLIENT_SECRET = '' + +# URL for oauth client to pull information about openid configuration +SERVER_METADATA_URL = 'https:///realms/master/.well-known/openid-configuration' + +[GUNICORN] + +# The number of worker processes for handling requests +THREADS = 2 + +# The number of worker threads in each process for handling requests +WORKERS = 4 diff --git a/doc/initscript b/doc/initscript deleted file mode 100644 index 830daa8..0000000 --- a/doc/initscript +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# -### BEGIN INIT INFO -# Provides: astrid -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start astrid at boot time -# Description: Enable astrid (simple continuous integration server) services. -### END INIT INFO - -PIDFILE="/tmp/astrid.pid" -case "$1" in - start) - if [ -f $PIDFILE ] ; then - echo "Already running." - exit 1 - else - echo "Starting astrid." - su -c "export PYTHONPATH=\"/home/koutny/local/lib/python\"; python /home/koutny/scripts/cis/start.py $PIDFILE;" koutny - fi - ;; - stop) - if [ -f $PIDFILE ] ; then - echo "Stopping astrid." - kill -9 $(cat $PIDFILE) - rm -f $PIDFILE - else - echo "Not running." - exit 1 - fi - ;; - *) - echo "Usage: /etc/init.d/astrid {start|stop}" - exit 1 - ;; -esac - -exit 0 diff --git a/docker/Dockerfile b/docker/Dockerfile index 6a93e43..37d9778 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,9 @@ FROM python:3.12 +# tells entrypoint if it's a production or development environment +# and should run gunicorn or not +ENV MODE="production" + # install docker RUN apt update && apt install -y docker.io containers-storage diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2180bee..a129fea 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +name: astrid services: astrid: @@ -12,8 +12,10 @@ services: TZ: 'Europe/Prague' PUID: 1000 GUID: 65534 + MODE: 'development' privileged: true # needed for containers volumes: - ./data:/data + - ..:/app ports: - 8080:8080 # opened port mapping, not needed with proxy diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 90205fc..e8bcc2a 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -49,12 +49,16 @@ chown "$PUID:$GUID" /data # create needed files if missing su - $USER -c "mkdir -p /data/config /data/containers /data/log /data/repos /data/ssh" -su - $USER -c "cp -n /app/config.ini.sample /data/config/config.ini" -su - $USER -c "cp -n /app/repos.ini.sample /data/config/repos.ini" +su - $USER -c "cp -n /app/config.toml.sample /data/config/config.toml" +su - $USER -c "cp -n /app/repos.toml.sample /data/config/repos.toml" if [ $(ls "/data/ssh" | grep ".pub" | wc -l) -eq 0 ]; then su - $USER -c "ssh-keygen -t ed25519 -f /data/ssh/id_ed25519" fi dockerd & -su - $USER -c "python3 -u /app/main" +if [ "$MODE" == "development" ]; then + su - $USER -c "python3 -u /app/src/app.py" +else + su - $USER -c "gunicorn --config /app/src/gunicorn.conf.py --chdir /app/src" +fi diff --git a/main b/main deleted file mode 100755 index ccffe5b..0000000 --- a/main +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import cherrypy - -from cherrypy.process.plugins import Daemonizer -from cherrypy.process.plugins import PIDFile - -import astrid - -print("Starting Astrid service") - -daemon = False -argc = len(sys.argv) - -if argc == 1: - daemon = False -elif argc == 2 and sys.argv[1] == '-d': - pidfile = os.path.expanduser("/data/pidfile") - daemon = True -elif argc == 3 and sys.argv[1] == '-d': - pidfile = sys.argv[2] -else: - print("Usage: {} [-d [pidfile]]".format(sys.argv[0])) - sys.exit(1) - -if daemon: - Daemonizer(cherrypy.engine).subscribe() - PIDFile(cherrypy.engine, pidfile=pidfile).subscribe() - -print("Starting web server") - -cherrypy.tree.mount(astrid.root, "/", config=astrid.config) -cherrypy.engine.start() -cherrypy.engine.block() diff --git a/repos.ini.sample b/repos.ini.sample deleted file mode 100644 index fddac6e..0000000 --- a/repos.ini.sample +++ /dev/null @@ -1,6 +0,0 @@ -[fykos37] -path=gitea@fykos.cz:FYKOS/fykos37.git -users=user -image_version=latest -build_usr=astrid -build_cmd=make -k all diff --git a/repos.toml.sample b/repos.toml.sample new file mode 100644 index 0000000..5165995 --- /dev/null +++ b/repos.toml.sample @@ -0,0 +1,12 @@ +# Repos config file +# +# Declares all repositories in this app. Each repository is its own section +# specified by [reponame] with listed config values. +# + +[fykos37] # reponame +git_path='gitea@fykos.cz:FYKOS/fykos37.git' # ssh address of git repository, string +allowed_roles=['fksdb-fykos'] # array of allowed users, array of strings +build_cmd='make -k all' # build command, string +image_version='latest' # docker image version of buildtools, string, optional (default 'latest') +submodules=false # does repo have submodules, bool, optional (default false) diff --git a/requirements.txt b/requirements.txt index 9e3f2a0..1511618 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ -CherryPy GitPython +Flask +AuthLib +requests +gunicorn diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 21cced6..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[egg_info] -tag_svn_revision = false diff --git a/setup.py b/setup.py deleted file mode 100644 index d49bdac..0000000 --- a/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -from setuptools import setup, find_packages -import sys, os - -if len(sys.argv) < 3: - sys.stderr.write("Missing 2nd argument 'version'.\n") - sys.exit(1) -else: - version = sys.argv[1] - del sys.argv[1] - - -setup(name='astrid', - version=version, - description="Minimalistic continous integration", - long_description="""\ -""", - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers - keywords='Git build continous integration', - author='Michal Koutn\xc3\xbd', - author_email='michal@fykos.cz', - url='', - license='WTFPL', - packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), - include_package_data=True, - zip_safe=False, - install_requires=[ - 'GitPython', - 'cherrypy' - ], - scripts=[ - 'main', - 'bin/touch-astrid.sample' - ], - package_dir={ - 'astrid': 'astrid' - }, - package_data={ - 'astrid': ['templates/*', 'static/style.css'] - }, - data_files=[ - ('config', ['repos.ini.sample', 'config.ini.sample']) - ], - entry_points=""" - # -*- Entry points: -*- - """, - ) diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..527bd0f --- /dev/null +++ b/src/app.py @@ -0,0 +1,114 @@ +from flask import Flask, abort, redirect, render_template, send_from_directory, session, url_for +import configparser +import os +import tomllib + +from repository import Repository +from oauth import isLoggedIn, registerAuthRoutes, requireLogin + +# TODO run using gunicorn for more threads + +app = Flask(__name__, + static_folder='static', + template_folder='templates') + +# load config +app.config.from_file("/data/config/config.toml", load=tomllib.load, text=False) + +registerAuthRoutes(app) + +# load repos +#reposConfig = configparser.ConfigParser() +#reposConfig.read('/data/config/repos.ini') +with open('/data/config/repos.toml', 'rb') as file: + reposConfig = tomllib.load(file) + +repos = {repo: Repository(repo, '/data/repos/', reposConfig[repo]) for repo in reposConfig} + +# automatically inject user to every rendered template +@app.context_processor +def inject_user(): + return dict(user = session.get('user')) + +# +# routes +# + +@app.route('/') +def index(): + if isLoggedIn(): + return render_template('index.html.jinja', repos=repos) + return render_template('signin.html.jinja') + +@app.route('/info/') +@requireLogin +def info(repoName): + if repoName not in repos: + abort(404) + repo = repos[repoName] + if not repo.checkAccess(): + abort(401) + return render_template('info.html.jinja', repo=repoName, log=repo.logger.getLogs()) + +@app.route('/buildlog/') +@requireLogin +def buildLog(repoName): + if repoName not in repos: + abort(404) + repo = repos[repoName] + if not repo.checkAccess(): + abort(401) + return render_template('buildlog.html.jinja', repo=repoName, log=repo.logger.getBuildLog()) + +@app.route('/build/') +@requireLogin +def build(repoName): + if repoName not in repos: + abort(404) + repo = repos[repoName] + if not repo.checkAccess(): + abort(401) + + repo.execute() + + return redirect(url_for('index')) + +# TODO redirect to url with trailing / when showing folder for correct html hrefs +@app.route('//') +@app.route('//') +@requireLogin +def repository(repoName, path=''): + if repoName not in repos: + abort(404) + repo = repos[repoName] + if not repo.checkAccess(): + abort(401) + + repoDir = os.path.normpath(os.path.join('/data/repos', repoName)) + normalizedPath = os.path.normpath(path) + targetPath = os.path.normpath(os.path.join(repoDir, normalizedPath)) + + # check that listed directory is child of repository directory + # for potencial directory travelsal attacks + if not targetPath.startswith(repoDir): + abort(404) + + if not os.path.exists(targetPath): + abort(404) + + if os.path.isdir(targetPath): + directoryContent = next(os.walk(targetPath)) + # filter directories and files starting with . + dirs=[d for d in sorted(directoryContent[1]) if not d.startswith('.')] + files=[f for f in sorted(directoryContent[2]) if not f.startswith('.')] + + return render_template('listdir.html.jinja', path=os.path.normpath(os.path.join(repoName, normalizedPath)), repo=repoName, dirs=dirs, files=files) + + if os.path.isfile(targetPath): + return send_from_directory(repoDir, normalizedPath) + + abort(404) + + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=8080) diff --git a/src/gunicorn.conf.py b/src/gunicorn.conf.py new file mode 100644 index 0000000..f2b2db7 --- /dev/null +++ b/src/gunicorn.conf.py @@ -0,0 +1,12 @@ +import tomllib + +def get_config(): + with open('/data/config/config.toml', 'rb') as file: + return tomllib.load(file) + +wsgi_app = 'app:app' +bind = "0.0.0.0:8080" +preload_app = True # must be set for locks to work between different gunicorn workers + +workers = get_config()['GUNICORN']['WORKERS'] +threads = get_config()['GUNICORN']['THREADS'] diff --git a/src/logger.py b/src/logger.py new file mode 100644 index 0000000..c96b7c0 --- /dev/null +++ b/src/logger.py @@ -0,0 +1,61 @@ +from datetime import datetime +from email.mime.text import MIMEText +import smtplib + +class Logger: + separator = ";" + + def __init__(self, repoName): + self.repoName = repoName + + def _getLogfile(self): + return f'/data/log/{self.repoName}.log' + + def _getBuildlogfile(self): + return f'/data/log/{self.repoName}.build.log' + + def log(self, user, message, sendMail = False): + logfile = self._getLogfile() + f = open(logfile, "a") + f.write(Logger.separator.join([datetime.now().isoformat(' '), message, user]) + "\n") + f.close() + + if sendMail: + self._sendMail(message) + + def getLogs(self): + logfile = self._getLogfile() + try: + f = open(logfile, "r") + + records = [row.split(Logger.separator) for row in reversed(list(f))] + f.close() + return records + except: + return [] + + def getBuildLog(self): + fn = self._getBuildlogfile() + try: + f = open(fn, "r") + res = f.read() + f.close() + return res + except: + return None + + def _sendMail(self, message): + headerFrom = cherrypy.config.get("mail_from") + headerTo = cherrypy.config.get("mail_to") + + msg = MIMEText(message) + msg['Subject'] = 'Astrid coughed on repo {}'.format(self.repoName) + msg['From'] = headerFrom + msg['To'] = headerTo + + try: + s = smtplib.SMTP('localhost') + s.sendmail(headerFrom, [headerTo], msg.as_string()) + s.quit() + except: + pass diff --git a/src/oauth.py b/src/oauth.py new file mode 100644 index 0000000..5d19d6b --- /dev/null +++ b/src/oauth.py @@ -0,0 +1,52 @@ +from functools import wraps +from flask import redirect, request, session, url_for +from authlib.integrations.flask_client import OAuth + +def isLoggedIn() -> bool: + return 'user' in session and session['user'] is not None + +def requireLogin(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if not isLoggedIn(): + print(request.url) + session['next'] = request.url + return redirect(url_for('login')) + return f(*args, **kwargs) + return decorated_function + +# +# Add all routes needed for auth to passed app +# +# This is a workaround, because app variable is needed for oauth variable, +# which is then used in routes. Flask blueprints cannot be used because it +# would create circular import dependency with app.py. +# +def registerAuthRoutes(app): + oauth = OAuth(app) + oauth.register( + name='keycloak', + client_id=app.config['OAUTH']['CLIENT_ID'], + client_secret=app.config['OAUTH']['CLIENT_SECRET'], + server_metadata_url=app.config['OAUTH']['SERVER_METADATA_URL'], + client_kwargs={ + 'scope': 'openid profile roles' + } + ) + + @app.route('/login') + def login(): + redirect_uri = url_for('auth', _external=True) + return oauth.keycloak.authorize_redirect(redirect_uri) + + @app.route('/auth') + def auth(): + token = oauth.keycloak.authorize_access_token() + session['user'] = token['userinfo'] + next_url = session.pop('next', None) + return redirect(next_url or url_for('index')) + + @app.route('/logout') + def logout(): + session.clear() + return redirect('/') diff --git a/src/repository.py b/src/repository.py new file mode 100644 index 0000000..7bd9fb8 --- /dev/null +++ b/src/repository.py @@ -0,0 +1,168 @@ +from flask import session +from logger import Logger +import time +from threading import Thread, Lock +from git import Repo, Git, GitCommandError +import os +import subprocess + +from oauth import isLoggedIn + +class Repository: + def __init__(self, name: str, repodir: str, repoConfig: dict): + self.name = name + self.repodir = repodir + self.config = repoConfig + + self.runningLock = Lock() + self.waitingLock = Lock() + self.logger = Logger(name) + + # validate config + if 'git_path' not in self.config or not self.config['git_path']: + raise ValueError(f'Config value `git_path` not specified or empty in repository {name}') + if not isinstance(self.config['git_path'], str): + raise TypeError(f'Config value `git_path` must be a string in repository {name}') + + if 'allowed_roles' not in self.config: + raise ValueError(f'Config value `allowed_roles` not specified in repository {name}') + if not isinstance(self.config['allowed_roles'], list) or not all(isinstance(role, str) for role in self.config['allowed_roles']): + raise TypeError(f'Config value `allowed_roles` must be a list of strings in repository {name}') + + if 'build_cmd' not in self.config: + raise ValueError(f'Config value `build_cmd` not specified in repository {name}') + if not isinstance(self.config['build_cmd'], str): + raise TypeError(f'Config value `build_cmd` must be a string in repository {name}') + + if 'image_version' in self.config: + if not isinstance(self.config['image_version'], str): + raise TypeError(f'Config value `image_version` must be a string in repository {name}') + else: + self.config['image_version'] = 'latest' # default value + + if 'submodules' in self.config: + if not isinstance(self.config['submodules'], bool): + raise TypeError(f'Config value `submodules` must be a bool in repository {name}') + else: + self.config['submodules'] = False # default value + + def checkAccess(self) -> bool: + if not isLoggedIn(): + return False + user = session['user'] + if not user['realm_access']['roles']: + return False + # check that user has at least one group specified in config + if len(set(self.config['allowed_roles']).intersection(user['realm_access']['roles'])) > 0: + return True + return False + + def isBuilding(self) -> bool: + return self.runningLock.locked() + + def execute(self): + Thread(target=self._queueJob, args=(self.name, session['user']['name'])).start() + + def _queueJob(self, reponame, user): + + # if job is not running + if self.runningLock.acquire(blocking=False): + # lock job running and run a job + try: + self._runJob(reponame, user) + finally: # ensure lock release on exception + self.runningLock.release() # release lock for job run + return + + # if job is already running + if not self.waitingLock.acquire(blocking=False): # if another process is already waiting + return # skip waiting + + # if it's not waiting, waiting lock is acquired + + # wait for previos job to complete, wait for running lock release and acquire it + self.runningLock.acquire() + self.waitingLock.release() # relase lock for waiting + try: + self._runJob(reponame, user) # run job + finally: # ensure lock release on exception + self.runningLock.release() # release lock for job run + + def _runJob(self, reponame, user): + msg = None + sendMail = False + + time_start = time.time() + time_end = None + + try: + repo = self._updateRepo(reponame) + try: + if self._build(reponame): + msg = f"#{repo.head.commit.hexsha[0:6]} Build succeeded." + time_end = time.time() + else: + msg = f"#{repo.head.commit.hexsha[0:6]} Build failed." + time_end = time.time() + except: + msg = f"#{repo.head.commit.hexsha[0:6]} Build error." + raise + except GitCommandError as e: + msg = f"Pull error. ({e})" + sendMail = True + raise + + if time_end is not None: + msg += " ({:.2f} s)".format(time_end - time_start) + + self.logger.log(user, msg, sendMail) + + def _updateRepo(self, reponame): + remotepath = self.config['path'] + submodules = self.config['submodules'] if 'submodules' in self.config else False + localpath = os.path.join(self.repodir, reponame) + + os.umask(0o007) # create repo content not readable to others + if not os.path.isdir(localpath): + print(f"Repository {reponame} empty, cloning") + g = Git() + g.clone(remotepath, localpath) + + repo = Repo(localpath) + + if submodules: + repo.git.submodule("init") + + # not-working umask workaround + subprocess.run(["chmod", "g+w", localpath], check=False) + else: + print(f"Pulling repository {reponame}") + repo = Repo(localpath) + try: + repo.git.update_index("--refresh") + except GitCommandError: + pass # it's fine, we'll reset + # We use wrapped API (direct git command calls) rather than calling semantic GitPython API. + # (e.g. repo.remotes.origin.pull() was replaced by repo.git.pull("origin")) + repo.git.reset("--hard", "master") + #repo.git.clean("-f") + repo.git.pull("origin") + + if submodules: + repo.git.submodule("init") + repo.git.submodule("foreach", "git", "fetch") + repo.git.submodule("update") + + return repo + + def _build(self, reponame): + cmd = self.config['build_cmd'] + cwd = os.path.join(self.repodir, reponame) # current working directory + image_version = self.config['image_version'] + + print(f"Building {reponame}") + logfilename = os.path.expanduser(f"/data/log/{reponame}.build.log") + logfile = open(logfilename, "w") + p = subprocess.run(["docker", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"--user={os.getuid()}:{os.getgid()}", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) + logfile.close() + return p.returncode == 0 diff --git a/src/static/style.css b/src/static/style.css new file mode 100644 index 0000000..d655422 --- /dev/null +++ b/src/static/style.css @@ -0,0 +1,3 @@ +table td { + padding: 1px 10px 1px 10px; +} \ No newline at end of file diff --git a/src/templates/base.html.jinja b/src/templates/base.html.jinja new file mode 100644 index 0000000..a72a9fd --- /dev/null +++ b/src/templates/base.html.jinja @@ -0,0 +1,15 @@ + + + + + + {% block title %}{% endblock %} + + + + + + {% if user %}

User: {{user['name']}} Logout

{% endif %} +{% block content %}{% endblock %} + + diff --git a/src/templates/buildlog.html.jinja b/src/templates/buildlog.html.jinja new file mode 100644 index 0000000..0484614 --- /dev/null +++ b/src/templates/buildlog.html.jinja @@ -0,0 +1,8 @@ +{% extends "base.html.jinja" %} +{% block title %}{{repo}} buildlog{% endblock %} +{% block content %} +

Build log for repository {{repo}}

+
+{{log}}
+
+{% endblock %} diff --git a/src/templates/index.html.jinja b/src/templates/index.html.jinja new file mode 100644 index 0000000..f425600 --- /dev/null +++ b/src/templates/index.html.jinja @@ -0,0 +1,14 @@ +{% extends "base.html.jinja" %} +{% block title %}Astrid{% endblock %} +{% block content %} +

Astrid

+

Please choose a repository:

+
    +{% for repo in repos.values() %} + {% if repo.checkAccess() %} +
  • {{ repo.name }} (info{% if repo.isBuilding() %}, building{% endif %})
    • + {% endif %} +{% endfor %} +
+{% endblock %} + diff --git a/src/templates/info.html.jinja b/src/templates/info.html.jinja new file mode 100644 index 0000000..4144fdf --- /dev/null +++ b/src/templates/info.html.jinja @@ -0,0 +1,25 @@ +{% extends "base.html.jinja" %} +{% block title %}{{repo}} info{% endblock %} +{% block content %} +

Information for {{repo}}

+

Buildlog

+ + + + + + + + + + {% for row in log %} + + + + + + {% endfor %} + +
TimeMessageUser
{{row[0]}}{{row[1]}}{{row[2]}}
+{% endblock %} + diff --git a/src/templates/listdir.html.jinja b/src/templates/listdir.html.jinja new file mode 100644 index 0000000..b7a762e --- /dev/null +++ b/src/templates/listdir.html.jinja @@ -0,0 +1,35 @@ +{% extends "base.html.jinja" %} +{% block title %}{{repo}}{% endblock %} +{% block content %} +

Directory listing for {{path}}

+ + + + + + + + + + + + + + + {% for dir in dirs %} + + + + + + {% endfor %} + {% for file in files %} + + + + + + {% endfor %} + +
FileSizeLast modified
../
{{dir}}/
{{file}}
+{% endblock %} diff --git a/src/templates/signin.html.jinja b/src/templates/signin.html.jinja new file mode 100644 index 0000000..c81191b --- /dev/null +++ b/src/templates/signin.html.jinja @@ -0,0 +1,8 @@ +{% extends "base.html.jinja" %} +{% block title %}{{repo}} buildlog{% endblock %} +{% block content %} +

You are currently signed out

+

+ login +

+{% endblock %} From 32a8f3b6e9e7213e4a80b0dd855b8b9951a60772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Sun, 21 Jul 2024 22:32:18 +0200 Subject: [PATCH 2/5] basic auth option --- docker/entrypoint.sh | 1 + src/app.py | 37 +++++++++++--- src/auth.py | 88 ++++++++++++++++++++++++++++++++ src/oauth.py | 52 ------------------- src/repository.py | 5 +- src/templates/listdir.html.jinja | 12 ++--- touch.sh | 10 ++++ users.toml.sample | 8 +++ 8 files changed, 145 insertions(+), 68 deletions(-) create mode 100644 src/auth.py delete mode 100644 src/oauth.py create mode 100755 touch.sh create mode 100644 users.toml.sample diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e8bcc2a..5324e83 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -51,6 +51,7 @@ su - $USER -c "mkdir -p /data/config /data/containers /data/log /data/repos /dat su - $USER -c "cp -n /app/config.toml.sample /data/config/config.toml" su - $USER -c "cp -n /app/repos.toml.sample /data/config/repos.toml" +su - $USER -c "cp -n /app/users.toml.sample /data/config/users.toml" if [ $(ls "/data/ssh" | grep ".pub" | wc -l) -eq 0 ]; then su - $USER -c "ssh-keygen -t ed25519 -f /data/ssh/id_ed25519" diff --git a/src/app.py b/src/app.py index 527bd0f..104e98d 100644 --- a/src/app.py +++ b/src/app.py @@ -1,12 +1,10 @@ +from datetime import datetime from flask import Flask, abort, redirect, render_template, send_from_directory, session, url_for -import configparser import os import tomllib from repository import Repository -from oauth import isLoggedIn, registerAuthRoutes, requireLogin - -# TODO run using gunicorn for more threads +from auth import isLoggedIn, registerAuthRoutes, requireLogin app = Flask(__name__, static_folder='static', @@ -18,8 +16,6 @@ registerAuthRoutes(app) # load repos -#reposConfig = configparser.ConfigParser() -#reposConfig.read('/data/config/repos.ini') with open('/data/config/repos.toml', 'rb') as file: reposConfig = tomllib.load(file) @@ -30,6 +26,13 @@ def inject_user(): return dict(user = session.get('user')) +def human_readable_size(size): + for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']: + if size < 1024.0 or unit == 'PiB': + value = f"{size:.2f}".rstrip('0').rstrip('.') + return f"{value} {unit}" + size /= 1024.0 + # # routes # @@ -99,8 +102,26 @@ def repository(repoName, path=''): if os.path.isdir(targetPath): directoryContent = next(os.walk(targetPath)) # filter directories and files starting with . - dirs=[d for d in sorted(directoryContent[1]) if not d.startswith('.')] - files=[f for f in sorted(directoryContent[2]) if not f.startswith('.')] + dirNames=[d for d in sorted(directoryContent[1]) if not d.startswith('.')] + fileNames=[f for f in sorted(directoryContent[2]) if not f.startswith('.')] + + # get file metadata + dirs = [] + for dirName in dirNames: + filepath = os.path.join(targetPath, dirName) + modifiedTime = datetime.fromtimestamp(os.path.getmtime(filepath)).isoformat(' ', timespec='seconds') + dirs.append([ + dirName, '-', modifiedTime + ]) + + files = [] + for fileName in fileNames: + filepath = os.path.join(targetPath, fileName) + size = human_readable_size(os.path.getsize(filepath)) + modifiedTime = datetime.fromtimestamp(os.path.getmtime(filepath)).isoformat(' ', timespec='seconds') + files.append([ + fileName, size, modifiedTime + ]) return render_template('listdir.html.jinja', path=os.path.normpath(os.path.join(repoName, normalizedPath)), repo=repoName, dirs=dirs, files=files) diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..3ea599a --- /dev/null +++ b/src/auth.py @@ -0,0 +1,88 @@ +from functools import wraps +import tomllib +from authlib.oidc.core.claims import hmac +from flask import abort, redirect, request, session, url_for +from authlib.integrations.flask_client import OAuth +from werkzeug.datastructures import Authorization + +with open('/data/config/users.toml', 'rb') as file: + basicAuthUsers = tomllib.load(file) + +def isLoggedIn() -> bool: + return session.get('user') is not None + +def loginBasicAuthUser(auth: Authorization) -> None: + if auth.username not in basicAuthUsers: + abort(401) + + user = basicAuthUsers[auth.username] + if auth.password is None or user['password'] is None: + abort(401) + + # compare passwords safely + if not hmac.compare_digest(auth.password, user['password']): + abort(401) + + if user['roles'] is None: + raise ValueError(f'User {auth.username} is missing field `roles` in configuration.') + + if not isinstance(user['roles'], list) or not all(isinstance(role, str) for role in user['roles']): + raise TypeError(f'Config value `roles` must be a list of strings in basic auth user {auth.username}') + + session['user'] = { + 'name': auth.username, + 'realm_access': { + 'roles': user['roles'] + } + } + +def requireLogin(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if not isLoggedIn(): + # basic auth + auth = request.authorization + if auth is not None: + loginBasicAuthUser(auth) + else: + # redirect to oauth + session['next'] = request.url + return redirect(url_for('login')) + return f(*args, **kwargs) + return decorated_function + +# +# Add all routes needed for auth to passed app +# +# This is a workaround, because app variable is needed for oauth variable, +# which is then used in routes. Flask blueprints cannot be used because it +# would create circular import dependency with app.py. +# +def registerAuthRoutes(app): + oauth = OAuth(app) + oauth.register( + name='keycloak', + client_id=app.config['OAUTH']['CLIENT_ID'], + client_secret=app.config['OAUTH']['CLIENT_SECRET'], + server_metadata_url=app.config['OAUTH']['SERVER_METADATA_URL'], + client_kwargs={ + 'scope': 'openid profile roles' + } + ) + + @app.route('/login') + def login(): + redirect_uri = url_for('auth', _external=True) + return oauth.keycloak.authorize_redirect(redirect_uri) + + @app.route('/auth') + def auth(): + token = oauth.keycloak.authorize_access_token() + session['user'] = token['userinfo'] + next_url = session.pop('next', None) + return redirect(next_url or url_for('index')) + + @app.route('/logout') + def logout(): + session.clear() + return redirect('/') diff --git a/src/oauth.py b/src/oauth.py deleted file mode 100644 index 5d19d6b..0000000 --- a/src/oauth.py +++ /dev/null @@ -1,52 +0,0 @@ -from functools import wraps -from flask import redirect, request, session, url_for -from authlib.integrations.flask_client import OAuth - -def isLoggedIn() -> bool: - return 'user' in session and session['user'] is not None - -def requireLogin(f): - @wraps(f) - def decorated_function(*args, **kwargs): - if not isLoggedIn(): - print(request.url) - session['next'] = request.url - return redirect(url_for('login')) - return f(*args, **kwargs) - return decorated_function - -# -# Add all routes needed for auth to passed app -# -# This is a workaround, because app variable is needed for oauth variable, -# which is then used in routes. Flask blueprints cannot be used because it -# would create circular import dependency with app.py. -# -def registerAuthRoutes(app): - oauth = OAuth(app) - oauth.register( - name='keycloak', - client_id=app.config['OAUTH']['CLIENT_ID'], - client_secret=app.config['OAUTH']['CLIENT_SECRET'], - server_metadata_url=app.config['OAUTH']['SERVER_METADATA_URL'], - client_kwargs={ - 'scope': 'openid profile roles' - } - ) - - @app.route('/login') - def login(): - redirect_uri = url_for('auth', _external=True) - return oauth.keycloak.authorize_redirect(redirect_uri) - - @app.route('/auth') - def auth(): - token = oauth.keycloak.authorize_access_token() - session['user'] = token['userinfo'] - next_url = session.pop('next', None) - return redirect(next_url or url_for('index')) - - @app.route('/logout') - def logout(): - session.clear() - return redirect('/') diff --git a/src/repository.py b/src/repository.py index 7bd9fb8..dea8a2b 100644 --- a/src/repository.py +++ b/src/repository.py @@ -6,7 +6,7 @@ import os import subprocess -from oauth import isLoggedIn +from auth import isLoggedIn class Repository: def __init__(self, name: str, repodir: str, repoConfig: dict): @@ -118,7 +118,7 @@ def _runJob(self, reponame, user): self.logger.log(user, msg, sendMail) def _updateRepo(self, reponame): - remotepath = self.config['path'] + remotepath = self.config['git_path'] submodules = self.config['submodules'] if 'submodules' in self.config else False localpath = os.path.join(self.repodir, reponame) @@ -165,4 +165,5 @@ def _build(self, reponame): logfile = open(logfilename, "w") p = subprocess.run(["docker", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"--user={os.getuid()}:{os.getgid()}", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) logfile.close() + print(f"Building {reponame} completed") return p.returncode == 0 diff --git a/src/templates/listdir.html.jinja b/src/templates/listdir.html.jinja index b7a762e..da386f8 100644 --- a/src/templates/listdir.html.jinja +++ b/src/templates/listdir.html.jinja @@ -18,16 +18,16 @@ {% for dir in dirs %} - {{dir}}/ - - + {{dir[0]}}/ + {{dir[1]}} + {{dir[2]}} {% endfor %} {% for file in files %} - {{file}} - - + {{file[0]}} + {{file[1]}} + {{file[2]}} {% endfor %} diff --git a/touch.sh b/touch.sh new file mode 100755 index 0000000..d40f193 --- /dev/null +++ b/touch.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Astrid connection configuration +USER= +PASS= +URL="https:///build/$GITEA_REPO_NAME" + +curl --silent --output /dev/null --user $USER:$PASS $URL \ +&& echo "$GITEA_REPO_NAME: rebuild request sent to Astrid." \ +|| echo "$GITEA_REPO_NAME: rebuild request sending failed." diff --git a/users.toml.sample b/users.toml.sample new file mode 100644 index 0000000..7c14801 --- /dev/null +++ b/users.toml.sample @@ -0,0 +1,8 @@ +# Basic auth users +# +# Specifies user by using username as [section] and setting password +# and roles properties. + +[user] +password = "password" +roles = ["role1", "role2"] From 1c72faae6f59a7c43b588fa7bbd9460af1942bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Sun, 21 Jul 2024 23:46:27 +0200 Subject: [PATCH 3/5] cleanup --- .gitignore | 6 +- README | 43 --- README.md | 49 ++++ astrid/__init__.py | 91 ------- astrid/pages.py | 256 ------------------ astrid/server.py | 178 ------------ astrid/static/style.css | 3 - astrid/templates/build.html | 6 - astrid/templates/buildlog.html | 15 - astrid/templates/dir.html | 7 - astrid/templates/error.html | 4 - astrid/templates/footer.inc | 2 - astrid/templates/header.inc | 7 - astrid/templates/home.html | 10 - astrid/templates/info.html | 11 - astrid/templates/textfile.html | 6 - astrid/templating.py | 42 --- bin/touch-astrid-https.sample | 21 -- bin/touch-astrid.sample | 28 -- .../config.toml.sample | 2 +- repos.toml.sample => config/repos.toml.sample | 0 users.toml.sample => config/users.toml.sample | 6 +- doc/http_touch.sh | 28 -- docker/entrypoint.sh | 6 +- src/repository.py | 16 +- 25 files changed, 74 insertions(+), 769 deletions(-) delete mode 100644 README create mode 100644 README.md delete mode 100644 astrid/__init__.py delete mode 100644 astrid/pages.py delete mode 100644 astrid/server.py delete mode 100644 astrid/static/style.css delete mode 100644 astrid/templates/build.html delete mode 100644 astrid/templates/buildlog.html delete mode 100644 astrid/templates/dir.html delete mode 100644 astrid/templates/error.html delete mode 100644 astrid/templates/footer.inc delete mode 100644 astrid/templates/header.inc delete mode 100644 astrid/templates/home.html delete mode 100644 astrid/templates/info.html delete mode 100644 astrid/templates/textfile.html delete mode 100644 astrid/templating.py delete mode 100644 bin/touch-astrid-https.sample delete mode 100644 bin/touch-astrid.sample rename config.toml.sample => config/config.toml.sample (95%) rename repos.toml.sample => config/repos.toml.sample (100%) rename users.toml.sample => config/users.toml.sample (71%) delete mode 100644 doc/http_touch.sh diff --git a/.gitignore b/.gitignore index 21332cf..3729227 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,2 @@ -build -dist -astrid.egg-info/ -*~ -*.pyc +**__pycache__ docker/data diff --git a/README b/README deleted file mode 100644 index dc61299..0000000 --- a/README +++ /dev/null @@ -1,43 +0,0 @@ -About -===== -Astrid is a minimalistic content integration server. It tracks Git -repositories, it can run build tasks on them and expose the built repositories -on the web. Astrid provides simple isolation of different build users with the -help of file permissions. - -Installation -============ - -Quickest way is to install Astrid from Python egg (run `./build.sh` to build -the egg) on the destination machine. - -After egg installation, copy configuration files located in 'config' directory -into '~/.astrid' and strip the '.sample' extension. (It is recommended to -create separate user account for astrid daemon.) - -If you want to integrate Astrid with systemd, copy 'astrid.service' file into -/etc/systemd/system/astrid.service on target machine and run 'systemctl -daemon-reload && systemctl enable astrid.service'. -(NOTE: It assumes existence of user 'astrid' and hardcoded path to its home -directory.) - -Eventually edit the settings for your site. - -Manual running -============== - -Start Astrid with script 'bin/astrid -d [pidfile]', it will spawn a background -process and write its PID into the pidfile. - -Running as systemd service -========================== - -If you installed service unit file as described in Installation, you can -control Astrid via systemctl commands. - - -Automatic rebuilds -================== - -Configure post-receive hook in your Git repository to let Astrid know about -incoming updates. Sample hook is in 'bin/touch-astrid.sh'. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1ec97d4 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# Astrid +Astrid (Automatická Sazba Textů -rid) je minimalistický server určen pro +automatickou kompilaci LaTeX souborů v rámci gitového repozitáře v +perzistentním uložišti. Kompilace probíhá v rámci `docker` podporocesu, který +využívá [buildtools image](https://github.com/fykosak/docker-buildtools). + +## Spuštění +Aplikace je navržena pro použití uvnitř docker kontejneru. Pro deployment lze +využít sample [docker compose](docker/docker-compose.prod.yml). [Docker +image](https://github.com/fykosak/astrid/pkgs/container/astrid) se vytvoří +automaticky pomocí github actions. + +Po spuštění se automaticky ve složce `data` vytvoří potřebné složky +a konfigurace. +- `config` - obsahuje konfiguraci ve formátu `TOML` +- `containers` - složka sloužící jako cache pro docker image uvnitř Astrid +- `log` - log soubory o historii jednotlivých spuštění a posledním spuštění kompilace +- `ssh` - ssh soubory pro uživatele uvnitř Astrid, potřeba pro přístup ke gitovým repozitářům + +> [!WARNING] +> Konfigurační soubor `config.toml` obsahuje proměnnou `SECRET_KEY`. Tu je +> potřeba nastavit na dostatečně dlouhý náhodný řetězec znaků, aby byla zajištěno +> bezpečné ukládání `SESSION` a přihlašování. + +### Vývoj +Pro lokální spuštění určeno pro vývoj stačí přejít do složky `docker` a v ní +spustit `docker compose up`. Vše ostatní se vytvoří automaticky. Po prvním +spuštění je třeba upravit konfigurační soubor `config.toml`, ve kterém je třeba +odkomentovat proměnnou `SECRET_KEY`. Dále je doporučeno nastavit ostatní +proměnné určené pro vývoj. + +## Spuštění buildu +Kompilace repozitáře lze ručně spustit pomocí zadání url +`https:///build/`. Automaticky lze spouštět +zavoláním git post-receive hooku, který obdobný request provede. +Sample script je v souboru [touch.sh](touch.sh). + +## Implementace +Astrid je napsána v Pythonu pomocí knihovny `Flask` (dříve `CherryPy`). +Společní s ní je využita integrace knihovny `authlib`. Aplikace předpokládá, že +uživatelé se přihlašují pomoci OIDC poskytovatele Keycloak, který pošle při +přihlášení přidělené role, které jsou následně kontrolovány dle nastavení +repozitářů. + +Pro automatizovaný přístup (spuštění buildu, stahování PDF) je implementované +i přihlašování přes http basic auth pro uživatele specifikované v `users.toml`. + +Při vývoji je spuštěna přímo Flask aplikace, pro produkci Astrid využívá WSGI +server `Gunicorn`, který umožňuje aplikaci běžet na více jádrech a vláknech. diff --git a/astrid/__init__.py b/astrid/__init__.py deleted file mode 100644 index 90d7a97..0000000 --- a/astrid/__init__.py +++ /dev/null @@ -1,91 +0,0 @@ -from datetime import datetime -from email.mime.text import MIMEText - -import cherrypy -import os.path -import smtplib -import threading - -class BuildLogger: - separator = ";" - - def __init__(self, repodir): - self.repodir = repodir - - def _getLogfile(self, reponame): - return f'/data/log/{reponame}.log' - - def _getBuildlogfile(self, reponame): - return f'/data/log/{reponame}.build.log' - - def log(self, reponame, user, message, sendMail = False): - logfile = self._getLogfile(reponame) - f = open(logfile, "a") - f.write(BuildLogger.separator.join([datetime.now().isoformat(' '), message, user]) + "\n") - f.close() - - if sendMail: - self._sendMail(reponame, message) - - def getLogs(self, reponame): - logfile = self._getLogfile(reponame) - try: - f = open(logfile, "r") - - records = [row.split(BuildLogger.separator) for row in reversed(list(f))] - f.close() - return records - except: - return [] - - def getBuildlog(self, reponame): - fn = self._getBuildlogfile(reponame) - try: - f = open(fn, "r") - res = f.read() - f.close() - return res - except: - return None - - def _sendMail(self, reponame, message): - headerFrom = cherrypy.config.get("mail_from") - headerTo = cherrypy.config.get("mail_to") - - msg = MIMEText(message) - msg['Subject'] = 'Astrid coughed on repo {}'.format(reponame) - msg['From'] = headerFrom - msg['To'] = headerTo - - try: - s = smtplib.SMTP('localhost') - s.sendmail(headerFrom, [headerTo], msg.as_string()) - s.quit() - except: - pass - - -from configparser import ConfigParser - -REPOS_INI = '/data/config/repos.ini' -CONFIG_INI = '/data/config/config.ini' - -repos = ConfigParser() -repos.read(os.path.expanduser(REPOS_INI)) -cherrypy.engine.autoreload.files.add(REPOS_INI) - -config = os.path.expanduser(CONFIG_INI) -cherrypy.config.update(config) - -# prepare locks for each repository -locks = {section: threading.Lock() for section in repos.sections()} -waitLocks = {section: threading.Lock() for section in repos.sections()} - -from astrid.pages import BuilderPage, InfoPage, DashboardPage, BuildlogPage - -repodir = cherrypy.config.get("repodir") - -root = DashboardPage(repodir, repos, locks, waitLocks) -root.build = BuilderPage(repodir, repos, locks, waitLocks) -root.info = InfoPage(repodir, repos, locks, waitLocks) -root.buildlog = BuildlogPage(repodir, repos, locks, waitLocks) diff --git a/astrid/pages.py b/astrid/pages.py deleted file mode 100644 index 8969856..0000000 --- a/astrid/pages.py +++ /dev/null @@ -1,256 +0,0 @@ -import os.path -import os -import subprocess -import time -from threading import Thread -import cherrypy - -from git import Repo, Git, GitCommandError - -from astrid import BuildLogger -from astrid.templating import Template -import astrid.server - -class BasePage(object): - def __init__(self, repodir, repos, locks, waitLocks): - self.repos = repos - self.repodir = repodir - self.locks = locks - self.waitLocks = waitLocks - - def _checkAccess(self, reponame): - user = cherrypy.request.login - if user not in self.repos.get(reponame, "users"): - raise cherrypy.HTTPError(401) - - def _isBuilding(self, reponame): - lock = self.locks[reponame] - building = not lock.acquire(False) - if not building: # repo's not building, we release the lock - lock.release() - return building - - -class BuilderPage(BasePage): - - @cherrypy.expose - def default(self, reponame): - if reponame not in self.repos.sections(): - raise cherrypy.HTTPError(404) - - self._checkAccess(reponame) - - user = cherrypy.request.login - - thread = Thread(target=self._queueJob, args=(reponame,user)) - thread.daemon = True - thread.start() - - raise cherrypy.HTTPRedirect("/") - - def _queueJob(self, reponame, user): - lock = self.locks[reponame] - waitLock = self.waitLocks[reponame] - - # if job is not running - if lock.acquire(blocking=False): - # lock job running and run job - try: - self._runJob(reponame, user) - finally: # ensure lock release on exception - lock.release() # release lock for job run - return - - # if job is already running - if not waitLock.acquire(blocking=False): # if another process is already waiting - return # skip waiting - - # if it's not waiting - # we now own lock for waiting - lock.acquire() # wait for job lock to release and acquire it - waitLock.release() # relase lock for waiting - try: - self._runJob(reponame, user) # run job - finally: # ensure lock release on exception - lock.release() # release lock for job run - - def _runJob(self, reponame, user): - msg = None - sendMail = False - - time_start = time.time() - time_end = None - - try: - repo = self._updateRepo(reponame) - try: - if self._build(reponame): - msg = f"#{repo.head.commit.hexsha[0:6]} Build succeeded." - time_end = time.time() - else: - msg = f"#{repo.head.commit.hexsha[0:6]} Build failed." - time_end = time.time() - except: - msg = f"#{repo.head.commit.hexsha[0:6]} Build error." - raise - except GitCommandError as e: - msg = f"Pull error. ({e})" - sendMail = True - raise - - if time_end is not None: - msg += " ({:.2f} s)".format(time_end - time_start) - - logger = BuildLogger(self.repodir) - logger.log(reponame, user, msg, sendMail) - - def _updateRepo(self, reponame): - remotepath = self.repos.get(reponame, "path") - submodules = self.repos.get(reponame, "submodules") if self.repos.has_option(reponame, "submodules") else False - localpath = os.path.join(self.repodir, reponame) - - os.umask(0o007) # create repo content not readable to others - if not os.path.isdir(localpath): - print(f"Repository {reponame} empty, cloning") - g = Git() - g.clone(remotepath, localpath) - - repo = Repo(localpath) - - if submodules: - repo.git.submodule("init") - - # not-working umask workaround - subprocess.run(["chmod", "g+w", localpath], check=False) - else: - print(f"Pulling repository {reponame}") - repo = Repo(localpath) - try: - repo.git.update_index("--refresh") - except GitCommandError: - pass # it's fine, we'll reset - # We use wrapped API (direct git command calls) rather than calling semantic GitPython API. - # (e.g. repo.remotes.origin.pull() was replaced by repo.git.pull("origin")) - repo.git.reset("--hard", "master") - #repo.git.clean("-f") - repo.git.pull("origin") - - if submodules: - repo.git.submodule("init") - repo.git.submodule("foreach", "git", "fetch") - repo.git.submodule("update") - - return repo - - def _build(self, reponame): - cmd = self.repos.get(reponame, "build_cmd") - cwd = os.path.join(self.repodir, reponame) # current working directory - image_version = self.repos.get(reponame, "image_version") - - print(f"Building {reponame}") - logfilename = os.path.expanduser(f"/data/log/{reponame}.build.log") - logfile = open(logfilename, "w") - p = subprocess.run(["docker", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"--user={os.getuid()}:{os.getgid()}", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) - logfile.close() - return p.returncode == 0 - -class InfoPage(BasePage): - - @cherrypy.expose - def default(self, reponame): - if reponame not in self.repos.sections(): - raise cherrypy.HTTPError(404) - - self._checkAccess(reponame) - - logger = BuildLogger(self.repodir) - - msg = "" - first = True - for record in logger.getLogs(reponame): - record += ['']*(3-len(record)) - if first: - msg += f'' - first = False - else: - msg += f"" - - msg += "
TimeMessageUser
{record[0]}{record[1]}{record[2]}
{record[0]}{record[1]}{record[2]}
" - - template = Template("templates/info.html") - template.assignData("reponame", reponame) - for k, v in self.repos.items(reponame): - template.assignData("repo." + k, v) - - template.assignData("messages", msg) - template.assignData("pagetitle", reponame + " info") - - return template.render() - -class BuildlogPage(BasePage): - - @cherrypy.expose - def default(self, reponame): - if reponame not in self.repos.sections(): - raise cherrypy.HTTPError(404) - - self._checkAccess(reponame) - - logger = BuildLogger(self.repodir) - building = self._isBuilding(reponame) - building = ', building…' if building else '' - - template = Template("templates/buildlog.html") - template.assignData("reponame", reponame) - template.assignData("building", building) - template.assignData("buildlog", logger.getBuildlog(reponame)) - - template.assignData("pagetitle", reponame + " build log") - - return template.render() - - -class DashboardPage(BasePage): - _cp_config = {'tools.staticdir.on' : True, - 'tools.staticdir.dir' : cherrypy.config.get("repodir"), - 'tools.staticdir.indexlister': astrid.server.htmldir, - 'tools.staticdir.content_types': { - 'aux': 'text/plain; charset=utf-8', - 'log': 'text/plain; charset=utf-8', - 'toc': 'text/plain; charset=utf-8', - 'out': 'text/plain; charset=utf-8', - 'tex': 'text/plain; charset=utf-8', - 'mpx': 'text/plain; charset=utf-8', - 'mp': 'text/plain; charset=utf-8', - 'plt': 'text/plain; charset=utf-8', - 'dat': 'text/plain; charset=utf-8', - 'csv': 'text/csv; charset=utf-8', - 'py': 'text/x-python; charset=utf-8', - 'sh': 'text/x-sh; charset=utf-8', - 'soap': 'application/soap+xml', - 'ipe': 'application/xml' - #'inc': ??? or download like .tex - #'sample': ??? how about this stuff? - #'Makefile': ??? how about Makefiles - } - #'tools.staticdir.index' : 'index.html', - } - - @cherrypy.expose - def index(self, **params): - template = Template('templates/home.html') - repos = "" - user = cherrypy.request.login - - for section in self.repos.sections(): - if user in self.repos.get(section, "users").split(","): - building = self._isBuilding(section) - building = ', building…' if building else '' - repos += f'
  • {section} (info{building})
    • ' - - template.assignData("pagetitle", "Astrid") - template.assignData("repos", repos) - template.assignData("user", user) - return template.render() - - index._cp_config = {'tools.staticdir.on': False} diff --git a/astrid/server.py b/astrid/server.py deleted file mode 100644 index c649f75..0000000 --- a/astrid/server.py +++ /dev/null @@ -1,178 +0,0 @@ -import os.path -import re -import stat -import urllib.request, urllib.parse, urllib.error -import datetime -import cherrypy -import cherrypy.lib.auth_basic -from cherrypy.lib import cptools, httputil -from cherrypy.lib.static import staticdir - -import astrid - -def htmldir( section="", dir="", path="", hdr=True, **kwargs ): - fh = '' - url = "http://" + cherrypy.request.headers.get('Host', '') + \ - cherrypy.request.path_info - - # preamble - if hdr: - fh += """ - -Directory listing for: {} - - - -""".format(url) - - path = path.rstrip(r"\/") - fh += '

    Directory listing for: ' + url + \ - '

    \n' - fh += '' - - fh += '' - for dpath, ddirs, dfiles in os.walk( path ): - - for dn in sorted( ddirs ): - if dn.startswith("."): - continue - fdn = os.path.join( dpath, dn ) - dmtime = os.path.getmtime( fdn ) - dtim = datetime.datetime.fromtimestamp( dmtime ).isoformat(' ') - fh += f'' - - del ddirs[:] # limit to one level - - for fil in sorted( dfiles ): - if fil.startswith("."): - continue - fn = os.path.join( dpath, fil ) - siz = os.path.getsize( fn ) - fmtime = os.path.getmtime( fn ) - ftim = datetime.datetime.fromtimestamp( fmtime ).isoformat(' ') - fh += f'' - - fh += '
    FileSizeLast mod
    ../
    {dn}/{dtim}
    {fil}{siz}{ftim}
    ' - # postamble - if hdr: - fh += """ - -""" - - return fh - -def staticdirindex(section, dir, root="", match="", content_types=None, index="", indexlistermatch="", indexlister=None, **kwargs): - """Serve a directory index listing for a dir. - - Compatibility alert: staticdirindex is built on and is dependent on - staticdir and its configurations. staticdirindex only works effectively - in locations where staticdir is also configured. staticdirindex is - coded to allow easy integration with staticdir, if demand warrants. - - indexlister must be configured, or no function is performed. - indexlister should be a callable that accepts the following parameters: - section: same as for staticdir (and implicitly calculated for it) - dir: same as for staticdir, but already combined with root - path: combination of section and dir - - Other parameters that are configured for staticdirindex will be passed - on to indexlister. - - Should use priorty > than that of staticdir, so that only directories not - served by staticdir, call staticdirindex. - -""" - req = cherrypy.request - response = cherrypy.response - - user = req.login - reponame = req.path_info.split("/")[1] - if reponame not in ["info", "build", "buildlog"]: - if reponame not in astrid.repos.sections(): - raise cherrypy.HTTPError(404) - - if user not in astrid.repos.get(reponame, "users"): - raise cherrypy.HTTPError(401) - - - match = indexlistermatch - - # first call old staticdir, and see if it does anything - sdret = staticdir( section, dir, root, match, content_types, index ) - if sdret: - return True - - # if not, then see if we are configured to do anything - if indexlister is None: - return False - - # N.B. filename ending in a slash or not does not imply a directory - # the following block of code directly copied from static.py staticdir - if match and not re.search(match, cherrypy.request.path_info): - return False - - # Allow the use of '~' to refer to a user's home directory. - dir = os.path.expanduser(dir) - - # If dir is relative, make absolute using "root". - if not os.path.isabs(dir): - if not root: - msg = "Static dir requires an absolute dir (or root)." - raise ValueError(msg) - dir = os.path.join(root, dir) - - # Determine where we are in the object tree relative to 'section' - # (where the static tool was defined). - if section == 'global': - section = "/" - section = section.rstrip(r"\/") - branch = cherrypy.request.path_info[len(section) + 1:] - branch = urllib.parse.unquote(branch.lstrip(r"\/")) - - # If branch is "", filename will end in a slash - filename = os.path.join(dir, branch) - - # There's a chance that the branch pulled from the URL might - # have ".." or similar uplevel attacks in it. Check that the final - # filename is a child of dir. - if not os.path.normpath(filename).startswith(os.path.normpath(dir)): - raise cherrypy.HTTPError(403) # Forbidden - # the above block of code directly copied from static.py staticdir - # N.B. filename ending in a slash or not does not imply a directory - - # Check if path is a directory. - - path = filename - # The following block of code copied from static.py serve_file - - # If path is relative, users should fix it by making path absolute. - # That is, CherryPy should not guess where the application root is. - # It certainly should *not* use cwd (since CP may be invoked from a - # variety of paths). If using tools.static, you can make your relative - # paths become absolute by supplying a value for "tools.static.root". - if not os.path.isabs(path): - raise ValueError(f"'{path}' is not an absolute path.") - - try: - st = os.stat(path) - except OSError: - # The above block of code copied from static.py serve_file - - return False - - if stat.S_ISDIR(st.st_mode): - - # Set the Last-Modified response header, so that - # modified-since validation code can work. - response.headers['Last-Modified'] = httputil.HTTPDate(st.st_mtime) - cptools.validate_since() - response.body = indexlister( section=section, dir=dir, path=path, - **kwargs ).encode() - response.headers['Content-Type'] = 'text/html' - req.is_index = True - return True - - return False - -# Replace the real staticdir with our version -cherrypy.tools.staticdir = cherrypy._cptools.HandlerTool( staticdirindex ) diff --git a/astrid/static/style.css b/astrid/static/style.css deleted file mode 100644 index d655422..0000000 --- a/astrid/static/style.css +++ /dev/null @@ -1,3 +0,0 @@ -table td { - padding: 1px 10px 1px 10px; -} \ No newline at end of file diff --git a/astrid/templates/build.html b/astrid/templates/build.html deleted file mode 100644 index 6f1a4a4..0000000 --- a/astrid/templates/build.html +++ /dev/null @@ -1,6 +0,0 @@ -{include header.inc} - -

    Build

    -

    Building {reponame}...

    -

    {message}

    -{include footer.inc} \ No newline at end of file diff --git a/astrid/templates/buildlog.html b/astrid/templates/buildlog.html deleted file mode 100644 index 2185a79..0000000 --- a/astrid/templates/buildlog.html +++ /dev/null @@ -1,15 +0,0 @@ -{include header.inc} - -

    Build log for repository {reponame}

    -

    -{building} -

    - -
    -{!buildlog}
-
    - -

    -{building} -

    -{include footer.inc} diff --git a/astrid/templates/dir.html b/astrid/templates/dir.html deleted file mode 100644 index 570769c..0000000 --- a/astrid/templates/dir.html +++ /dev/null @@ -1,7 +0,0 @@ -{include header.inc} - -

    Listing {dirname}

    -
      -{files} -
        -{include footer.inc} \ No newline at end of file diff --git a/astrid/templates/error.html b/astrid/templates/error.html deleted file mode 100644 index 3cf8697..0000000 --- a/astrid/templates/error.html +++ /dev/null @@ -1,4 +0,0 @@ -{include header.inc} - -

        Error {code} {message}

        -{include footer.inc} \ No newline at end of file diff --git a/astrid/templates/footer.inc b/astrid/templates/footer.inc deleted file mode 100644 index 691287b..0000000 --- a/astrid/templates/footer.inc +++ /dev/null @@ -1,2 +0,0 @@ - - \ No newline at end of file diff --git a/astrid/templates/header.inc b/astrid/templates/header.inc deleted file mode 100644 index e0686af..0000000 --- a/astrid/templates/header.inc +++ /dev/null @@ -1,7 +0,0 @@ - - - - -{pagetitle} - - diff --git a/astrid/templates/home.html b/astrid/templates/home.html deleted file mode 100644 index e5f6d58..0000000 --- a/astrid/templates/home.html +++ /dev/null @@ -1,10 +0,0 @@ -{include header.inc} - -

        {pagetitle}

        -

        Please choose a repository:

        -
          -{repos} -
        - -

        User: {user}

        -{include footer.inc} diff --git a/astrid/templates/info.html b/astrid/templates/info.html deleted file mode 100644 index b31cfe9..0000000 --- a/astrid/templates/info.html +++ /dev/null @@ -1,11 +0,0 @@ -{include header.inc} - -

        Information for repository {reponame}

        -

        -Address: {!repo.path}
        -Permitted users: {!repo.users}
        -Build user: {!repo.build_usr}
        -Build command: {!repo.build_cmd} -

        -{messages} -{include footer.inc} diff --git a/astrid/templates/textfile.html b/astrid/templates/textfile.html deleted file mode 100644 index a127ff9..0000000 --- a/astrid/templates/textfile.html +++ /dev/null @@ -1,6 +0,0 @@ -{include header.inc} - -
        -{!filecontent}
-
        -{include footer.inc} \ No newline at end of file diff --git a/astrid/templating.py b/astrid/templating.py deleted file mode 100644 index 159e354..0000000 --- a/astrid/templating.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import html -import re -import os.path -import pkg_resources - -class Template: - def __init__(self, filename): - self.filename = filename - self.includes = {} - self.data = {} - - def assignData(self, name, data): - self.data[name] = str(data) - - def render(self): - # firstly include files - template = pkg_resources.resource_stream('astrid', self.filename).read().decode() - template = re.sub('{include ([^}]*)}', self._include, template) - # secondly expand variables - return re.sub('{(!)?([^}]*)}', self._expand, template) - - def _include(self, mo): - filename = os.path.dirname(self.filename) + "/" + mo.group(1) - if filename not in self.includes: - f = pkg_resources.resource_stream('astrid', filename) - if not f: - self.includes[filename] = "NOT FOUND" - else: - self.includes[filename] = f.read().decode() - f.close() - - return self.includes[filename] - - def _expand(self, mo): - if mo.group(2) in self.data: - data = self.data[mo.group(2)] - else: - data = "undefined" - if mo.group(1) == "!": - return html.escape(data) - return data diff --git a/bin/touch-astrid-https.sample b/bin/touch-astrid-https.sample deleted file mode 100644 index a27aefa..0000000 --- a/bin/touch-astrid-https.sample +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - - -USER=user -PASS=password -HOST=astrid.example.org -URL="/build/reponame" - - -function touch_astrid { - # timeout 1 second, 1 try - wget -T 1 -t 1 --http-user=$USER --http-password=$PASS -O /dev/null https://${HOST}$URL 2>/dev/null - if [ $? = 0 ] ; then - echo "$URL: rebuild request sent to Astrid." - else - echo "$URL: rebuild request sending failed." - fi -} - - -touch_astrid & diff --git a/bin/touch-astrid.sample b/bin/touch-astrid.sample deleted file mode 100644 index 2cd2ee6..0000000 --- a/bin/touch-astrid.sample +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Astrid connection configuration -USER=HTTP_USER -PASS=HTTP_PASSWORD -HOST=localhost:8080 -URL="/build/$GL_REPO" - -# Auto deploy confiration -REPOSITORY=origin - - -function touch_astrid { - # Create the request - AUTH=$(echo -n "$USER:$PASS" | base64) - - REQUEST="HEAD $URL HTTP/1.0\r -Host: $HOST\r -Authorization: Basic $AUTH\r -\r -" - - echo -ne "$REQUEST" | /bin/nc.traditional -q 1 ${HOST/:/ } \ - && echo "$GL_REPO: rebuild request sent to Astrid." \ - || echo "$GL_REPO: rebuild request sending failed." -} - -touch_astrid diff --git a/config.toml.sample b/config/config.toml.sample similarity index 95% rename from config.toml.sample rename to config/config.toml.sample index fd6c87c..3c26f2e 100644 --- a/config.toml.sample +++ b/config/config.toml.sample @@ -16,7 +16,7 @@ TESTING = false # Secret key used for session encryption # set to long random string for security -SECRET_KEY = '' # CHANGE THIS! +# SECRET_KEY = '' # CHANGE THIS! [OAUTH] diff --git a/repos.toml.sample b/config/repos.toml.sample similarity index 100% rename from repos.toml.sample rename to config/repos.toml.sample diff --git a/users.toml.sample b/config/users.toml.sample similarity index 71% rename from users.toml.sample rename to config/users.toml.sample index 7c14801..7167c16 100644 --- a/users.toml.sample +++ b/config/users.toml.sample @@ -3,6 +3,10 @@ # Specifies user by using username as [section] and setting password # and roles properties. -[user] +[username1] password = "password" roles = ["role1", "role2"] + +[username2] +password = "password" +roles = ["role1"] diff --git a/doc/http_touch.sh b/doc/http_touch.sh deleted file mode 100644 index e55b609..0000000 --- a/doc/http_touch.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [ $(git rev-parse --is-bare-repository) = true ] -then - REPOSITORY_BASENAME=$(basename $(readlink -nf "$PWD"/)) - REPOSITORY_BASENAME=${REPOSITORY_BASENAME%.git} -else - REPOSITORY_BASENAME=$(basename $(readlink -nf "$PWD"/..)) -fi - -echo $REPOSITORY_BASENAME - -USER=user -PWD=passwd - -AUTH=$(echo -n "$USER:$PWD" | base64) - -HOST="localhost:8080" -PATH="/build/$REPOSITORY_BASENAME" - -REQUEST="HEAD $PATH HTTP/1.0\r -Host: $HOST\r -Authorization: Basic $AUTH\r -\r -" -#echo -ne "$REQUEST" | netcat -q 1 ${HOST/:/ } -echo -ne "$REQUEST" | /bin/nc.traditional -q 1 ${HOST/:/ } -echo "Rebuild request sent." diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 5324e83..886f56c 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -49,9 +49,9 @@ chown "$PUID:$GUID" /data # create needed files if missing su - $USER -c "mkdir -p /data/config /data/containers /data/log /data/repos /data/ssh" -su - $USER -c "cp -n /app/config.toml.sample /data/config/config.toml" -su - $USER -c "cp -n /app/repos.toml.sample /data/config/repos.toml" -su - $USER -c "cp -n /app/users.toml.sample /data/config/users.toml" +su - $USER -c "cp -n /app/config/config.toml.sample /data/config/config.toml" +su - $USER -c "cp -n /app/config/repos.toml.sample /data/config/repos.toml" +su - $USER -c "cp -n /app/config/users.toml.sample /data/config/users.toml" if [ $(ls "/data/ssh" | grep ".pub" | wc -l) -eq 0 ]; then su - $USER -c "ssh-keygen -t ed25519 -f /data/ssh/id_ed25519" diff --git a/src/repository.py b/src/repository.py index dea8a2b..6671b68 100644 --- a/src/repository.py +++ b/src/repository.py @@ -64,7 +64,16 @@ def execute(self): Thread(target=self._queueJob, args=(self.name, session['user']['name'])).start() def _queueJob(self, reponame, user): - + """ + Queue a job run. + + If no job is running for repository, exclusive lock is obtained + and job is executed. If another job is already running, lock for + waiting and job is waiting for first job to finish, then runs. + If one job is running and second is waiting, job run is skipped + as git pull of the waiting job will update to the newest state + of repository anyway. + """ # if job is not running if self.runningLock.acquire(blocking=False): # lock job running and run a job @@ -89,6 +98,11 @@ def _queueJob(self, reponame, user): self.runningLock.release() # release lock for job run def _runJob(self, reponame, user): + """ + Run a build job + + The repository is pulled first, than build is started. + """ msg = None sendMail = False From deadf6039649eeaa8f2eff3c2420f001bb8074f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Thu, 3 Oct 2024 01:57:36 +0200 Subject: [PATCH 4/5] autogenerate https external links --- src/app.py | 7 +++++++ src/templates/base.html.jinja | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index 104e98d..184b695 100644 --- a/src/app.py +++ b/src/app.py @@ -10,6 +10,13 @@ static_folder='static', template_folder='templates') +# This section is needed for url_for("foo", _external=True) to automatically +# generate http scheme when this sample is running on localhost, +# and to generate https scheme when it is deployed behind reversed proxy. +# See also https://flask.palletsprojects.com/en/2.2.x/deploying/proxy_fix/ +from werkzeug.middleware.proxy_fix import ProxyFix +app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1) + # load config app.config.from_file("/data/config/config.toml", load=tomllib.load, text=False) diff --git a/src/templates/base.html.jinja b/src/templates/base.html.jinja index a72a9fd..275d6f7 100644 --- a/src/templates/base.html.jinja +++ b/src/templates/base.html.jinja @@ -3,13 +3,14 @@ + {% block title %}{% endblock %} - {% if user %}

        User: {{user['name']}} Logout

        {% endif %} +{% if user %}

        User: {{user['name']}} Logout

        {% endif %} {% block content %}{% endblock %} From 5f298de46922296d090a53cacfe2e85023e31734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Thu, 3 Oct 2024 02:00:35 +0200 Subject: [PATCH 5/5] gh image build --- .github/workflows/deploy-image.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index ee65abb..52e42c3 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -4,13 +4,6 @@ on: push: branchers: - master - - dev-docker - paths: - - 'astrid/**' - - 'docker/**' - - 'main' - - 'requirements.txt' - - '.github/workflows/deploy-image.yml' env: REGISTRY: ghcr.io