From 8065e12e974039af33fc37283caeec013a992b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Wed, 1 Nov 2023 14:02:04 +0100 Subject: [PATCH 01/17] basic docker image --- .dockerignore | 1 + astrid/__init__.py | 14 ++++----- astrid/pages.py | 67 ++++++++++++++++++++------------------------ astrid/server.py | 18 ++++++------ astrid/templating.py | 13 ++++----- docker/Dockerfile | 26 +++++++++++++++++ docker/README.md | 11 ++++++++ docker/entrypoint.sh | 4 +++ bin/astrid => main | 4 +++ requirements.txt | 2 ++ setup.py | 2 +- 11 files changed, 100 insertions(+), 62 deletions(-) create mode 100644 .dockerignore create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/entrypoint.sh rename bin/astrid => main (91%) create mode 100644 requirements.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9a4b405 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +docker/Dockerfile diff --git a/astrid/__init__.py b/astrid/__init__.py index e9fc3f2..8ce06e1 100644 --- a/astrid/__init__.py +++ b/astrid/__init__.py @@ -8,31 +8,31 @@ class BuildLogger: separator = ";" - + def __init__(self, repodir): self.repodir = repodir - + def _getLogfile(self, reponame): return os.path.expanduser('~/.astrid/{}.log'.format(reponame)) def _getBuildlogfile(self, reponame): return os.path.expanduser('~/.astrid/{}.build.log'.format(reponame)) - + def log(self, reponame, message, sendMail = False): logfile = self._getLogfile(reponame) f = open(logfile, "a") user = cherrypy.request.login f.write(BuildLogger.separator.join([datetime.now().isoformat(' '), message, user]) + "\n") f.close() - + if sendMail: self._sendMail(reponame, message) - + def getLogs(self, reponame): logfile = self._getLogfile(reponame) try: f = open(logfile, "r") - + records = [row.split(BuildLogger.separator) for row in reversed(list(f))] f.close() return records @@ -89,5 +89,3 @@ def _sendMail(self, reponame, message): root.build = BuilderPage(repodir, repos, locks) root.info = InfoPage(repodir, repos, locks) root.buildlog = BuildlogPage(repodir, repos, locks) - - diff --git a/astrid/pages.py b/astrid/pages.py index 43657df..fdb5cd2 100644 --- a/astrid/pages.py +++ b/astrid/pages.py @@ -19,7 +19,7 @@ def __init__(self, repodir, repos, locks): self.repos = repos self.repodir = repodir self.locks = locks - + def _checkAccess(self, reponame): user = cherrypy.request.login if user not in self.repos.get(reponame, "users"): @@ -31,22 +31,22 @@ def _isBuilding(self, reponame): if not building: # repo's not building, we release the lock lock.release() return building - + class BuilderPage(BasePage): - + @cherrypy.expose def default(self, reponame): if reponame not in self.repos.sections(): raise cherrypy.HTTPError(404) - + self._checkAccess(reponame) lock = self.locks[reponame] msg = None sendMail = False - + lock.acquire() - try: + try: try: time_start = time.time() time_end = None @@ -68,11 +68,11 @@ def default(self, reponame): msg_class = "red" sendMail = True raise - + if time_end != None: msg += " ({:.2f} s)".format(time_end - time_start) - + template = Template("templates/build.html") template.assignData("reponame", reponame) template.assignData("message", msg) @@ -82,15 +82,17 @@ def default(self, reponame): finally: logger = BuildLogger(self.repodir) logger.log(reponame, msg, sendMail) - lock.release() - + lock.release() + def _updateRepo(self, reponame): + print(f"Updating repository {reponame}") remotepath = self.repos.get(reponame, "path") submodules = self.repos.get(reponame, "submodules") if self.repos.has_option(reponame, "submodules") else False localpath = os.path.join(self.repodir, reponame) os.umask(0o007) # create repo content not readable to others if not os.path.isdir(localpath): + print(f"Repository {reponame} empty, creating") g = Git() g.clone(remotepath, localpath) @@ -101,8 +103,9 @@ def _updateRepo(self, reponame): # not-working umask workaround p = subprocess.Popen(["chmod", "g+w", localpath]) - p.wait() + p.wait() else: + print(f"Pulling repository {reponame}") repo = Repo(localpath) try: repo.git.update_index("--refresh") @@ -122,33 +125,33 @@ def _updateRepo(self, reponame): # now set correct group (same as build user) usr = self.repos.get(reponame, "build_usr") p = subprocess.Popen(["chgrp", usr, "-R", "-f", localpath]) - p.wait() + p.wait() return repo - + def _build(self, reponame): usr = self.repos.get(reponame, "build_usr") cmd = self.repos.get(reponame, "build_cmd") args = self.repos.get(reponame, "build_args") cwd = os.path.join(self.repodir, reponame) - + logfilename = os.path.expanduser("~/.astrid/{}.build.log".format(reponame)) logfile = open(logfilename, "w") - p = subprocess.Popen(["sudo", "-u", usr, cmd] + shlex.split(args), cwd=cwd, stdout=logfile, stderr=logfile, stdin=open("/dev/null")) - p.wait() + p = subprocess.Popen([cmd] + shlex.split(args), cwd=cwd, stdout=logfile, stderr=logfile, stdin=open("/dev/null")) + p.wait() return p.returncode == 0 - + class InfoPage(BasePage): @cherrypy.expose def default(self, reponame): if reponame not in self.repos.sections(): raise cherrypy.HTTPError(404) - + self._checkAccess(reponame) - + logger = BuildLogger(self.repodir) - + msg = "" first = True for record in logger.getLogs(reponame): @@ -158,9 +161,9 @@ def default(self, reponame): first = False else: msg += """""".format(record[0], record[1], record[2]) - + msg += "
TimeMessageUser
{}{}{}
" - + template = Template("templates/info.html") template.assignData("reponame", reponame) for k, v in self.repos.items(reponame): @@ -168,7 +171,7 @@ def default(self, reponame): template.assignData("messages", msg) template.assignData("pagetitle", reponame + " info") - + return template.render() class BuildlogPage(BasePage): @@ -177,9 +180,9 @@ class BuildlogPage(BasePage): def default(self, reponame): if reponame not in self.repos.sections(): raise cherrypy.HTTPError(404) - + self._checkAccess(reponame) - + logger = BuildLogger(self.repodir) building = self._isBuilding(reponame) building = ', building…' if building else '' @@ -190,7 +193,7 @@ def default(self, reponame): template.assignData("buildlog", logger.getBuildlog(reponame)) template.assignData("pagetitle", reponame + " build log") - + return template.render() @@ -219,7 +222,7 @@ class DashboardPage(BasePage): } # 'tools.staticdir.index' : 'index.html', } - + @cherrypy.expose def index(self, **params): template = Template('templates/home.html') @@ -231,18 +234,10 @@ def index(self, **params): building = self._isBuilding(section) building = ', building…' if building else '' repos += """
  • {} (info{})
    • """.format(section, section, section, building) - + template.assignData("pagetitle", "Astrid") template.assignData("repos", repos) template.assignData("user", user) return template.render() - index._cp_config = {'tools.staticdir.on': False} - - - - - - - diff --git a/astrid/server.py b/astrid/server.py index 7aa1a5c..0217832 100644 --- a/astrid/server.py +++ b/astrid/server.py @@ -32,7 +32,7 @@ def htmldir( section="", dir="", path="", hdr=True, **kwargs ): fh += '

    Directory listing for: ' + url + \ '

    \n' fh += '' - + fh += """""".format("..", "..", "", "") for dpath, ddirs, dfiles in os.walk( path ): @@ -87,13 +87,13 @@ def staticdirindex(section, dir, root="", match="", content_types=None, index="" """ req = cherrypy.request response = cherrypy.response - + user = req.login reponame = req.path_info.split("/")[1] if reponame not in ["info", "build", "buildlog"]: if reponame not in astrid.repos.sections(): raise cherrypy.HTTPError(404) - + if user not in astrid.repos.get(reponame, "users"): raise cherrypy.HTTPError(401) @@ -113,7 +113,7 @@ def staticdirindex(section, dir, root="", match="", content_types=None, index="" # the following block of code directly copied from static.py staticdir if match and not re.search(match, cherrypy.request.path_info): return False - + # Allow the use of '~' to refer to a user's home directory. dir = os.path.expanduser(dir) @@ -123,7 +123,7 @@ def staticdirindex(section, dir, root="", match="", content_types=None, index="" msg = "Static dir requires an absolute dir (or root)." raise ValueError(msg) dir = os.path.join(root, dir) - + # Determine where we are in the object tree relative to 'section' # (where the static tool was defined). if section == 'global': @@ -131,10 +131,10 @@ def staticdirindex(section, dir, root="", match="", content_types=None, index="" section = section.rstrip(r"\/") branch = cherrypy.request.path_info[len(section) + 1:] branch = urllib.parse.unquote(branch.lstrip(r"\/")) - + # If branch is "", filename will end in a slash filename = os.path.join(dir, branch) - + # There's a chance that the branch pulled from the URL might # have ".." or similar uplevel attacks in it. Check that the final # filename is a child of dir. @@ -155,7 +155,7 @@ def staticdirindex(section, dir, root="", match="", content_types=None, index="" # paths become absolute by supplying a value for "tools.static.root". if not os.path.isabs(path): raise ValueError("'{}' is not an absolute path.".format(path)) - + try: st = os.stat(path) except OSError: @@ -179,5 +179,3 @@ def staticdirindex(section, dir, root="", match="", content_types=None, index="" # Replace the real staticdir with our version cherrypy.tools.staticdir = cherrypy._cptools.HandlerTool( staticdirindex ) - - diff --git a/astrid/templating.py b/astrid/templating.py index 268a123..ddd38bd 100644 --- a/astrid/templating.py +++ b/astrid/templating.py @@ -9,17 +9,17 @@ def __init__(self, filename): self.filename = filename self.includes = {} self.data = {} - + def assignData(self, name, data): self.data[name] = str(data) - + def render(self): # firstly include files template = pkg_resources.resource_stream('astrid', self.filename).read().decode() template = re.sub('\{include ([^\}]*)\}', self._include, template) # secondly expand variables return re.sub('\{(!)?([^\}]*)\}', self._expand, template) - + def _include(self, mo): filename = os.path.dirname(self.filename) + "/" + mo.group(1) if filename not in self.includes: @@ -29,16 +29,15 @@ def _include(self, mo): else: self.includes[filename] = f.read().decode() f.close() - + return self.includes[filename] - + def _expand(self, mo): if mo.group(2) in self.data: data = self.data[mo.group(2)] else: - data = "undefined" + data = "undefined" if mo.group(1) == "!": return html.escape(data) else: return data - diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..5158860 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.12 + +# install docker +RUN apt update && apt install -y podman + +# add astrid user +ARG DOCKER_USER=astrid +RUN useradd -m $DOCKER_USER + +# change workdir +WORKDIR /data/ + +# install python requirements +COPY requirements.txt . +RUN pip install -r requirements.txt + +# install astrid under astrid user +COPY . . +RUN chown -R $DOCKER_USER:$DOCKER_USER /data +USER $DOCKER_USER +#RUN ./build.sh + +# create config directory +RUN mkdir ~/.astrid + +ENTRYPOINT ./docker/entrypoint.sh diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..bf40931 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,11 @@ +# Docker container +## Build + - in root directory: + ``` + podman build . -f docker/Dockerfile --tag astrid + ``` + +## Run +``` +podman run --rm --name astrid --volume "./config:/home/astrid/.astrid" --volume "./ssh:/home/astrid/.ssh" -p 8080:8080 astrid +``` diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..ab04fc7 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,4 @@ +cp -n /data/config.ini.sample /home/astrid/.astrid/config.ini +cp -n /data/repos.ini.sample /home/astrid/.astrid/repos.ini + +python3 -u ./main diff --git a/bin/astrid b/main similarity index 91% rename from bin/astrid rename to main index aed5de8..2ddf9a2 100755 --- a/bin/astrid +++ b/main @@ -8,6 +8,8 @@ import os from cherrypy.process.plugins import Daemonizer from cherrypy.process.plugins import PIDFile +print("Starting Astrid service") + daemon = False argc = len(sys.argv) @@ -26,6 +28,8 @@ if daemon: Daemonizer(cherrypy.engine).subscribe() PIDFile(cherrypy.engine, pidfile=pidfile).subscribe() +print("Starting web server") + cherrypy.tree.mount(astrid.root, "/", config=astrid.config) cherrypy.engine.start() cherrypy.engine.block() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9e3f2a0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +CherryPy +GitPython diff --git a/setup.py b/setup.py index a335f32..d49bdac 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ 'cherrypy' ], scripts=[ - 'bin/astrid', + 'main', 'bin/touch-astrid.sample' ], package_dir={ From 050c1df8ca1c8f19670e9778e678c5d1edacddf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Tue, 26 Dec 2023 01:52:18 +0100 Subject: [PATCH 02/17] working build inside container --- .dockerignore | 3 ++- astrid/pages.py | 9 ++++----- docker/.gitignore | 4 ++++ docker/Dockerfile | 2 +- docker/docker-compose.yml | 21 +++++++++++++++++++++ docker/start.sh | 1 + repos.ini.sample | 32 +++++--------------------------- 7 files changed, 38 insertions(+), 34 deletions(-) create mode 100644 docker/.gitignore create mode 100644 docker/docker-compose.yml create mode 100755 docker/start.sh diff --git a/.dockerignore b/.dockerignore index 9a4b405..2c1cd4e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ -docker/Dockerfile +docker/* +!docker/entrypoint.sh diff --git a/astrid/pages.py b/astrid/pages.py index fdb5cd2..c497466 100644 --- a/astrid/pages.py +++ b/astrid/pages.py @@ -122,6 +122,7 @@ def _updateRepo(self, reponame): repo.git.submodule("foreach", "git", "fetch") repo.git.submodule("update") + print(f"Building {reponame}") # now set correct group (same as build user) usr = self.repos.get(reponame, "build_usr") p = subprocess.Popen(["chgrp", usr, "-R", "-f", localpath]) @@ -130,15 +131,13 @@ def _updateRepo(self, reponame): return repo def _build(self, reponame): - usr = self.repos.get(reponame, "build_usr") cmd = self.repos.get(reponame, "build_cmd") - args = self.repos.get(reponame, "build_args") - cwd = os.path.join(self.repodir, reponame) + cwd = os.path.join(self.repodir, reponame) # current working directory + image_version = self.repos.get(reponame, "image_version") logfilename = os.path.expanduser("~/.astrid/{}.build.log".format(reponame)) logfile = open(logfilename, "w") - p = subprocess.Popen([cmd] + shlex.split(args), cwd=cwd, stdout=logfile, stderr=logfile, stdin=open("/dev/null")) - p.wait() + p = subprocess.run(["podman", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) return p.returncode == 0 class InfoPage(BasePage): diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 0000000..d0549f0 --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,4 @@ +config +containers +repos +ssh diff --git a/docker/Dockerfile b/docker/Dockerfile index 5158860..6924cc3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.12 # install docker -RUN apt update && apt install -y podman +RUN apt update && apt install -y podman containers-storage # add astrid user ARG DOCKER_USER=astrid diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..2106062 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' + +services: + astrid: + image: astrid + build: + context: .. + dockerfile: docker/Dockerfile + restart: on-failure:3 + container_name: astrid + environment: + TZ: 'Europe/Prague' + privileged: true # needed for containers + volumes: + - ./config:/home/astrid/.astrid + - ./ssh:/home/astrid/.ssh + - ./containers:/home/astrid/.local/share/containers + - ./repos:/data/repos + ports: + - 8080:8080 # opened port mapping, not needed with proxy + user: 1000:1000 # expects the main user with uid:pid diff --git a/docker/start.sh b/docker/start.sh new file mode 100755 index 0000000..a36d856 --- /dev/null +++ b/docker/start.sh @@ -0,0 +1 @@ +docker run --rm --name astrid --volume "./config:/home/astrid/.astrid" --volume "./ssh:/home/astrid/.ssh" --volume "./containers:/home/astrid/.local/share/containers" --volume "./repos:/data/repos" -p 8080:8080 --privileged astrid diff --git a/repos.ini.sample b/repos.ini.sample index 60c9af5..9c658e8 100644 --- a/repos.ini.sample +++ b/repos.ini.sample @@ -1,28 +1,6 @@ -[fykos26] -path=/home/fksgit/data/fykos26.git +[fykos37] +path=gitea@fykos.cz:FYKOS/fykos37.git users=fykos,repo -build_usr=fksgit -build_cmd=make -build_args=all - -[fykos25] -path=/home/fksgit/data/fykos25.git -users=fykos,repo -build_usr=fksgit -build_cmd=make -build_args=all - -[vyfuk2] -path=/home/fksgit/data/vyfuk2.git -users=fykos,vyfuk,repo -build_usr=vyfuk -build_cmd=make -build_args=all - -[vyfuk1] -path=/home/fksgit/data/vyfuk1.git -users=fykos,vyfuk,repo -build_usr=vyfuk -build_cmd=make -build_args=all - +image_version=latest +build_usr=astrid +build_cmd=make -k all From 184e1d7f20ba0748e186af3d524114f195c2cc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Tue, 26 Dec 2023 21:48:11 +0100 Subject: [PATCH 03/17] update python syntax --- astrid/pages.py | 44 +++++++++++++++----------------------- astrid/server.py | 17 ++++++--------- astrid/templates/info.html | 2 +- astrid/templating.py | 9 ++++---- 4 files changed, 29 insertions(+), 43 deletions(-) diff --git a/astrid/pages.py b/astrid/pages.py index c497466..fbafcf6 100644 --- a/astrid/pages.py +++ b/astrid/pages.py @@ -1,14 +1,10 @@ -import cherrypy import os.path import os import subprocess -import shlex -import threading -import sys import time +import cherrypy from git import Repo, Git, GitCommandError -from configparser import ConfigParser from astrid import BuildLogger from astrid.templating import Template @@ -53,26 +49,26 @@ def default(self, reponame): repo = self._updateRepo(reponame) try: if self._build(reponame): - msg = "#{} Build succeeded.".format(repo.head.commit.hexsha[0:6]) + msg = f"#{repo.head.commit.hexsha[0:6]} Build succeeded." msg_class = "green" time_end = time.time() else: - msg = "#{} Build failed.".format(repo.head.commit.hexsha[0:6]) + msg = f"#{repo.head.commit.hexsha[0:6]} Build failed." msg_class = "red" + time_end = time.time() except: - msg = "#{} Build error.".format(repo.head.commit.hexsha[0:6]) + msg = f"#{repo.head.commit.hexsha[0:6]} Build error." msg_class = "red" raise except GitCommandError as e: - msg = "Pull error. ({})".format(str(e)) + msg = f"Pull error. ({e})" msg_class = "red" sendMail = True raise - if time_end != None: + if time_end is not None: msg += " ({:.2f} s)".format(time_end - time_start) - template = Template("templates/build.html") template.assignData("reponame", reponame) template.assignData("message", msg) @@ -102,8 +98,7 @@ def _updateRepo(self, reponame): repo.git.submodule("init") # not-working umask workaround - p = subprocess.Popen(["chmod", "g+w", localpath]) - p.wait() + subprocess.run(["chmod", "g+w", localpath], check=False) else: print(f"Pulling repository {reponame}") repo = Repo(localpath) @@ -122,12 +117,6 @@ def _updateRepo(self, reponame): repo.git.submodule("foreach", "git", "fetch") repo.git.submodule("update") - print(f"Building {reponame}") - # now set correct group (same as build user) - usr = self.repos.get(reponame, "build_usr") - p = subprocess.Popen(["chgrp", usr, "-R", "-f", localpath]) - p.wait() - return repo def _build(self, reponame): @@ -135,7 +124,8 @@ def _build(self, reponame): cwd = os.path.join(self.repodir, reponame) # current working directory image_version = self.repos.get(reponame, "image_version") - logfilename = os.path.expanduser("~/.astrid/{}.build.log".format(reponame)) + print(f"Building {reponame}") + logfilename = os.path.expanduser(f"~/.astrid/{reponame}.build.log") logfile = open(logfilename, "w") p = subprocess.run(["podman", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) return p.returncode == 0 @@ -156,10 +146,10 @@ def default(self, reponame): for record in logger.getLogs(reponame): record += ['']*(3-len(record)) if first: - msg += """""".format(record[0], reponame, record[1], record[2]) + msg += f'' first = False else: - msg += """""".format(record[0], record[1], record[2]) + msg += f"" msg += "
    FileSizeLast mod
    {}/{}{}
    {}{}{}
    {record[0]}{record[1]}{record[2]}
    {}{}{}
    {record[0]}{record[1]}{record[2]}
    " @@ -168,7 +158,7 @@ def default(self, reponame): for k, v in self.repos.items(reponame): template.assignData("repo." + k, v) - template.assignData("messages", msg) + template.assignData("messages", msg) template.assignData("pagetitle", reponame + " info") return template.render() @@ -185,7 +175,7 @@ def default(self, reponame): logger = BuildLogger(self.repodir) building = self._isBuilding(reponame) building = ', building…' if building else '' - + template = Template("templates/buildlog.html") template.assignData("reponame", reponame) template.assignData("building", building) @@ -219,7 +209,7 @@ class DashboardPage(BasePage): #'sample': ??? how about this stuff? #'Makefile': ??? how about Makefiles } -# 'tools.staticdir.index' : 'index.html', + #'tools.staticdir.index' : 'index.html', } @cherrypy.expose @@ -228,11 +218,11 @@ def index(self, **params): repos = "" user = cherrypy.request.login - for section in self.repos.sections(): + for section in self.repos.sections(): if user in self.repos.get(section, "users").split(","): building = self._isBuilding(section) building = ', building…' if building else '' - repos += """
  • {} (info{})
    • """.format(section, section, section, building) + repos += f'
  • {section} (info{building})
    • ' template.assignData("pagetitle", "Astrid") template.assignData("repos", repos) diff --git a/astrid/server.py b/astrid/server.py index 0217832..c649f75 100644 --- a/astrid/server.py +++ b/astrid/server.py @@ -1,18 +1,15 @@ -import cherrypy import os.path -import cherrypy.lib.auth_basic import re import stat import urllib.request, urllib.parse, urllib.error import datetime +import cherrypy +import cherrypy.lib.auth_basic from cherrypy.lib import cptools, httputil from cherrypy.lib.static import staticdir -from configparser import ConfigParser - import astrid - def htmldir( section="", dir="", path="", hdr=True, **kwargs ): fh = '' url = "http://" + cherrypy.request.headers.get('Host', '') + \ @@ -33,7 +30,7 @@ def htmldir( section="", dir="", path="", hdr=True, **kwargs ): '
    \n' fh += '' - fh += """""".format("..", "..", "", "") + fh += '' for dpath, ddirs, dfiles in os.walk( path ): for dn in sorted( ddirs ): @@ -42,7 +39,7 @@ def htmldir( section="", dir="", path="", hdr=True, **kwargs ): fdn = os.path.join( dpath, dn ) dmtime = os.path.getmtime( fdn ) dtim = datetime.datetime.fromtimestamp( dmtime ).isoformat(' ') - fh += """""".format(dn + '/', dn, "", dtim) + fh += f'' del ddirs[:] # limit to one level @@ -53,7 +50,7 @@ def htmldir( section="", dir="", path="", hdr=True, **kwargs ): siz = os.path.getsize( fn ) fmtime = os.path.getmtime( fn ) ftim = datetime.datetime.fromtimestamp( fmtime ).isoformat(' ') - fh += """""".format(fil, fil, siz, ftim) + fh += f'' fh += '
    FileSizeLast mod
    {}/{}{}
    ../
    {}/{}{}
    {dn}/{dtim}
    {}{}{}
    {fil}{siz}{ftim}
    ' # postamble @@ -92,7 +89,7 @@ def staticdirindex(section, dir, root="", match="", content_types=None, index="" reponame = req.path_info.split("/")[1] if reponame not in ["info", "build", "buildlog"]: if reponame not in astrid.repos.sections(): - raise cherrypy.HTTPError(404) + raise cherrypy.HTTPError(404) if user not in astrid.repos.get(reponame, "users"): raise cherrypy.HTTPError(401) @@ -154,7 +151,7 @@ def staticdirindex(section, dir, root="", match="", content_types=None, index="" # variety of paths). If using tools.static, you can make your relative # paths become absolute by supplying a value for "tools.static.root". if not os.path.isabs(path): - raise ValueError("'{}' is not an absolute path.".format(path)) + raise ValueError(f"'{path}' is not an absolute path.") try: st = os.stat(path) diff --git a/astrid/templates/info.html b/astrid/templates/info.html index 0286857..b31cfe9 100644 --- a/astrid/templates/info.html +++ b/astrid/templates/info.html @@ -5,7 +5,7 @@

    Information for repository {reponame}

    Address: {!repo.path}
    Permitted users: {!repo.users}
    Build user: {!repo.build_usr}
    -Build command: {!repo.build_cmd} {!repo.build_args} +Build command: {!repo.build_cmd}

    {messages} {include footer.inc} diff --git a/astrid/templating.py b/astrid/templating.py index ddd38bd..159e354 100644 --- a/astrid/templating.py +++ b/astrid/templating.py @@ -14,11 +14,11 @@ def assignData(self, name, data): self.data[name] = str(data) def render(self): - # firstly include files + # firstly include files template = pkg_resources.resource_stream('astrid', self.filename).read().decode() - template = re.sub('\{include ([^\}]*)\}', self._include, template) + template = re.sub('{include ([^}]*)}', self._include, template) # secondly expand variables - return re.sub('\{(!)?([^\}]*)\}', self._expand, template) + return re.sub('{(!)?([^}]*)}', self._expand, template) def _include(self, mo): filename = os.path.dirname(self.filename) + "/" + mo.group(1) @@ -39,5 +39,4 @@ def _expand(self, mo): data = "undefined" if mo.group(1) == "!": return html.escape(data) - else: - return data + return data From 96064a5cb5791609ca2c9c1b461035b617a26a75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Tue, 26 Dec 2023 22:00:11 +0100 Subject: [PATCH 04/17] build image on gh --- .github/workflows/deploy-image.yml | 46 ++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/deploy-image.yml diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml new file mode 100644 index 0000000..e874649 --- /dev/null +++ b/.github/workflows/deploy-image.yml @@ -0,0 +1,46 @@ +name: Create and publish a Docker image + +on: + push: + branchers: + - master + - dev-docker + paths: + - 'docker/**' + - '.github/workflows/deploy-image.yml' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: docker + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max From d86d3a2b8b03ec704b9215edfe58e60f1e83c0be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Tue, 26 Dec 2023 22:03:26 +0100 Subject: [PATCH 05/17] fix gh action --- .github/workflows/deploy-image.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index e874649..c4de4b4 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -38,7 +38,8 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@v5 with: - context: docker + context: . + file: ./docker/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From eab860dacfd479f4bb98cf42490a7a780b6c17c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Tue, 26 Dec 2023 22:51:49 +0100 Subject: [PATCH 06/17] fix dirpaths for container --- astrid/__init__.py | 4 ++-- config.ini.sample | 2 +- docker/Dockerfile | 7 +++---- docker/README.md | 16 +++++++--------- docker/docker-compose.prod.yml | 16 ++++++++++++++++ docker/docker-compose.yml | 4 ++-- docker/entrypoint.sh | 4 ++-- docker/start.sh | 1 - 8 files changed, 33 insertions(+), 21 deletions(-) create mode 100644 docker/docker-compose.prod.yml delete mode 100755 docker/start.sh diff --git a/astrid/__init__.py b/astrid/__init__.py index 8ce06e1..f985eef 100644 --- a/astrid/__init__.py +++ b/astrid/__init__.py @@ -68,8 +68,8 @@ def _sendMail(self, reponame, message): from configparser import ConfigParser -REPOS_INI = '~/.astrid/repos.ini' -CONFIG_INI = '~/.astrid/config.ini' +REPOS_INI = '/app/config/repos.ini' +CONFIG_INI = '/app/config/config.ini' repos = ConfigParser() repos.read(os.path.expanduser(REPOS_INI)) diff --git a/config.ini.sample b/config.ini.sample index 0c4c376..b8dcfe9 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -7,7 +7,7 @@ tools.auth_basic.on: True tools.auth_basic.realm: 'Astrid authorization' tools.auth_basic.checkpassword: cherrypy.lib.auth_basic.checkpassword_dict({'user': 'passwd'}) tools.expires.sec = 0 -repodir = 'data' +repodir = '/app/repos' [/style.css] tools.staticfile.on = True diff --git a/docker/Dockerfile b/docker/Dockerfile index 6924cc3..d42c42e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,7 +8,7 @@ ARG DOCKER_USER=astrid RUN useradd -m $DOCKER_USER # change workdir -WORKDIR /data/ +WORKDIR /app # install python requirements COPY requirements.txt . @@ -16,11 +16,10 @@ RUN pip install -r requirements.txt # install astrid under astrid user COPY . . -RUN chown -R $DOCKER_USER:$DOCKER_USER /data +RUN chown -R $DOCKER_USER:$DOCKER_USER /app USER $DOCKER_USER -#RUN ./build.sh # create config directory -RUN mkdir ~/.astrid +RUN mkdir /app/config /app/repos ENTRYPOINT ./docker/entrypoint.sh diff --git a/docker/README.md b/docker/README.md index bf40931..c0f9fa9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,11 +1,9 @@ # Docker container -## Build - - in root directory: - ``` - podman build . -f docker/Dockerfile --tag astrid - ``` -## Run -``` -podman run --rm --name astrid --volume "./config:/home/astrid/.astrid" --volume "./ssh:/home/astrid/.ssh" -p 8080:8080 astrid -``` +## Vývoj +- nutnost: nainstalovaný `docker` a `docker-compose-plugin` +- vytvoření aktuální image: `docker compose build` (potřeba spustit před prvním spuštěním) +- vytvoření mount složek (případný `chown` pro vlastnění správným uživatelem): `mkdir config containers repos ssh` +- spuštění: `docker compose up` +- spouštění příkazů uvnitř dockeru: `docker exec -it astrid ` + - stáhnutí `fykosak/buildtools` (potřeba před prvním buildem repozitáře): `docker exec -it astrid podman pull docker.io/fykosak/buildtools` diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml new file mode 100644 index 0000000..c26fbbd --- /dev/null +++ b/docker/docker-compose.prod.yml @@ -0,0 +1,16 @@ +version: '3' + +services: + astrid: + image: ghcr.io/fykosak/astrid:master + restart: on-failure:3 + container_name: astrid + environment: + TZ: 'Europe/Prague' + privileged: true # needed for containers + volumes: + - ./config:/home/astrid/.astrid + - ./ssh:/home/astrid/.ssh + - ./containers:/home/astrid/.local/share/containers + - ./repos:/data/repos + user: 1000:1000 # expects the main user with uid:pid diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2106062..27ec522 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,10 +12,10 @@ services: TZ: 'Europe/Prague' privileged: true # needed for containers volumes: - - ./config:/home/astrid/.astrid + - ./config:/app/config - ./ssh:/home/astrid/.ssh - ./containers:/home/astrid/.local/share/containers - - ./repos:/data/repos + - ./repos:/app/repos ports: - 8080:8080 # opened port mapping, not needed with proxy user: 1000:1000 # expects the main user with uid:pid diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index ab04fc7..720f64e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,4 +1,4 @@ -cp -n /data/config.ini.sample /home/astrid/.astrid/config.ini -cp -n /data/repos.ini.sample /home/astrid/.astrid/repos.ini +cp -n /app/config.ini.sample /app/config/config.ini +cp -n /app/repos.ini.sample /app/config/repos.ini python3 -u ./main diff --git a/docker/start.sh b/docker/start.sh deleted file mode 100755 index a36d856..0000000 --- a/docker/start.sh +++ /dev/null @@ -1 +0,0 @@ -docker run --rm --name astrid --volume "./config:/home/astrid/.astrid" --volume "./ssh:/home/astrid/.ssh" --volume "./containers:/home/astrid/.local/share/containers" --volume "./repos:/data/repos" -p 8080:8080 --privileged astrid From b58fbec9a38e80814d2b7c6fa059b373b0e9744a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Thu, 28 Dec 2023 17:14:52 +0100 Subject: [PATCH 07/17] make docker work under different user than 1000 --- .dockerignore | 4 ++++ .gitignore | 1 + astrid/__init__.py | 8 ++++---- astrid/pages.py | 3 ++- config.ini.sample | 4 ++-- docker/.gitignore | 4 ---- docker/Dockerfile | 11 ++++++++--- docker/README.md | 2 +- docker/docker-compose.prod.yml | 7 ++----- docker/docker-compose.yml | 7 ++----- docker/entrypoint.sh | 19 +++++++++++++++++-- docker/libpod.conf | 1 + docker/ssh.conf | 7 +++++++ docker/sshd.conf | 2 ++ docker/storage.conf | 3 +++ main | 7 ++++--- 16 files changed, 60 insertions(+), 30 deletions(-) delete mode 100644 docker/.gitignore create mode 100644 docker/libpod.conf create mode 100644 docker/ssh.conf create mode 100644 docker/sshd.conf create mode 100644 docker/storage.conf diff --git a/.dockerignore b/.dockerignore index 2c1cd4e..e780801 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,6 @@ docker/* !docker/entrypoint.sh +!docker/ssh.conf +!docker/sshd.conf +!docker/libpod.conf +!docker/storage.conf diff --git a/.gitignore b/.gitignore index 468b214..21332cf 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist astrid.egg-info/ *~ *.pyc +docker/data diff --git a/astrid/__init__.py b/astrid/__init__.py index f985eef..2318b44 100644 --- a/astrid/__init__.py +++ b/astrid/__init__.py @@ -13,10 +13,10 @@ def __init__(self, repodir): self.repodir = repodir def _getLogfile(self, reponame): - return os.path.expanduser('~/.astrid/{}.log'.format(reponame)) + return f'/data/log/{reponame}.log' def _getBuildlogfile(self, reponame): - return os.path.expanduser('~/.astrid/{}.build.log'.format(reponame)) + return f'/data/log/{reponame}.build.log' def log(self, reponame, message, sendMail = False): logfile = self._getLogfile(reponame) @@ -68,8 +68,8 @@ def _sendMail(self, reponame, message): from configparser import ConfigParser -REPOS_INI = '/app/config/repos.ini' -CONFIG_INI = '/app/config/config.ini' +REPOS_INI = '/data/config/repos.ini' +CONFIG_INI = '/data/config/config.ini' repos = ConfigParser() repos.read(os.path.expanduser(REPOS_INI)) diff --git a/astrid/pages.py b/astrid/pages.py index fbafcf6..653f29a 100644 --- a/astrid/pages.py +++ b/astrid/pages.py @@ -125,9 +125,10 @@ def _build(self, reponame): image_version = self.repos.get(reponame, "image_version") print(f"Building {reponame}") - logfilename = os.path.expanduser(f"~/.astrid/{reponame}.build.log") + logfilename = os.path.expanduser(f"/data/log/{reponame}.build.log") logfile = open(logfilename, "w") p = subprocess.run(["podman", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) + logfile.close() return p.returncode == 0 class InfoPage(BasePage): diff --git a/config.ini.sample b/config.ini.sample index b8dcfe9..a187c1f 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -1,13 +1,13 @@ [global] log.screen = False server.socket_port = 8080 -server.socket_host = '127.0.0.1' +server.socket_host = '0.0.0.0' server.thread_pool = 10 tools.auth_basic.on: True tools.auth_basic.realm: 'Astrid authorization' tools.auth_basic.checkpassword: cherrypy.lib.auth_basic.checkpassword_dict({'user': 'passwd'}) tools.expires.sec = 0 -repodir = '/app/repos' +repodir = '/data/repos' [/style.css] tools.staticfile.on = True diff --git a/docker/.gitignore b/docker/.gitignore deleted file mode 100644 index d0549f0..0000000 --- a/docker/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -config -containers -repos -ssh diff --git a/docker/Dockerfile b/docker/Dockerfile index d42c42e..c70e968 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,12 +14,17 @@ WORKDIR /app COPY requirements.txt . RUN pip install -r requirements.txt +# copy ssh config +COPY ./docker/ssh.conf /etc/ssh/ssh_config.d/99-astrid.conf +COPY ./docker/sshd.conf /etc/ssh/sshd_config.d/99-astrid.conf + +# copy containers config +COPY ./docker/libpod.conf /etc/containers/libpod.conf +COPY ./docker/storage.conf /etc/containers/storage.conf + # install astrid under astrid user COPY . . RUN chown -R $DOCKER_USER:$DOCKER_USER /app USER $DOCKER_USER -# create config directory -RUN mkdir /app/config /app/repos - ENTRYPOINT ./docker/entrypoint.sh diff --git a/docker/README.md b/docker/README.md index c0f9fa9..905d21a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -3,7 +3,7 @@ ## Vývoj - nutnost: nainstalovaný `docker` a `docker-compose-plugin` - vytvoření aktuální image: `docker compose build` (potřeba spustit před prvním spuštěním) -- vytvoření mount složek (případný `chown` pro vlastnění správným uživatelem): `mkdir config containers repos ssh` +- vytvoření mount složky `data` (případně `chown`, aby byla vlastněna uživatelem, pod kterým kontejner poběží) - spuštění: `docker compose up` - spouštění příkazů uvnitř dockeru: `docker exec -it astrid ` - stáhnutí `fykosak/buildtools` (potřeba před prvním buildem repozitáře): `docker exec -it astrid podman pull docker.io/fykosak/buildtools` diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index c26fbbd..21eed2b 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -9,8 +9,5 @@ services: TZ: 'Europe/Prague' privileged: true # needed for containers volumes: - - ./config:/home/astrid/.astrid - - ./ssh:/home/astrid/.ssh - - ./containers:/home/astrid/.local/share/containers - - ./repos:/data/repos - user: 1000:1000 # expects the main user with uid:pid + - ./data:/data + user: 1000:1000 # expects the main user as uid:pid diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 27ec522..485992e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,10 +12,7 @@ services: TZ: 'Europe/Prague' privileged: true # needed for containers volumes: - - ./config:/app/config - - ./ssh:/home/astrid/.ssh - - ./containers:/home/astrid/.local/share/containers - - ./repos:/app/repos + - ./data:/data ports: - 8080:8080 # opened port mapping, not needed with proxy - user: 1000:1000 # expects the main user with uid:pid + user: 1000:1000 # expects the main user as uid:pid diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 720f64e..85fff87 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,4 +1,19 @@ -cp -n /app/config.ini.sample /app/config/config.ini -cp -n /app/repos.ini.sample /app/config/repos.ini +#!/bin/bash + +DATA_OWNER=$(stat -c '%u' /data) + +if [ "$DATA_OWNER" -ne "$UID" ]; then + echo "Directory 'data' not owned by target user with $UID, instead owned by user with uid $DATA_OWNER" + exit 1 +fi + +mkdir -p /data/config /data/containers /data/log /data/repos /data/ssh + +cp -n /app/config.ini.sample /data/config/config.ini +cp -n /app/repos.ini.sample /data/config/repos.ini + +if [ $(ls "/data/ssh" | grep ".pub" | wc -l) -eq 0 ]; then + ssh-keygen -t ed25519 -f /data/ssh/id_ed25519 +fi python3 -u ./main diff --git a/docker/libpod.conf b/docker/libpod.conf new file mode 100644 index 0000000..86e8062 --- /dev/null +++ b/docker/libpod.conf @@ -0,0 +1 @@ +static_dir = "/data/containers/storage/libpod" diff --git a/docker/ssh.conf b/docker/ssh.conf new file mode 100644 index 0000000..9c3c96d --- /dev/null +++ b/docker/ssh.conf @@ -0,0 +1,7 @@ +# change .ssh files to /data/ssh +Host * + IdentityFile /data/ssh/id_rsa + IdentityFile /data/ssh/id_dsa + IdentityFile /data/ssh/id_ecdsa + IdentityFile /data/ssh/id_ed25519 + UserKnownHostsFile /data/ssh/known_hosts diff --git a/docker/sshd.conf b/docker/sshd.conf new file mode 100644 index 0000000..4ec32c7 --- /dev/null +++ b/docker/sshd.conf @@ -0,0 +1,2 @@ +# change .ssh files to /data/ssh +AuthorizedKeysFile /data/ssh/authorized_keys diff --git a/docker/storage.conf b/docker/storage.conf new file mode 100644 index 0000000..f5091cd --- /dev/null +++ b/docker/storage.conf @@ -0,0 +1,3 @@ +[storage] +driver = "overlay" +rootless_storage_path = "/data/containers/storage" diff --git a/main b/main index 2ddf9a2..ccffe5b 100755 --- a/main +++ b/main @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -import cherrypy -import astrid import sys import os +import cherrypy from cherrypy.process.plugins import Daemonizer from cherrypy.process.plugins import PIDFile +import astrid + print("Starting Astrid service") daemon = False @@ -16,7 +17,7 @@ argc = len(sys.argv) if argc == 1: daemon = False elif argc == 2 and sys.argv[1] == '-d': - pidfile = os.path.expanduser("~/.astrid/pidfile") + pidfile = os.path.expanduser("/data/pidfile") daemon = True elif argc == 3 and sys.argv[1] == '-d': pidfile = sys.argv[2] From 9b1d01ff7dab94e7822443e24a02b1f14732cd2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Thu, 28 Dec 2023 19:19:30 +0100 Subject: [PATCH 08/17] fix ssh and home dir --- docker/Dockerfile | 1 + docker/README.md | 4 ++-- docker/docker-compose.prod.yml | 1 + docker/docker-compose.yml | 1 + docker/entrypoint.sh | 5 +++++ 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c70e968..d798ac1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,6 +6,7 @@ RUN apt update && apt install -y podman containers-storage # add astrid user ARG DOCKER_USER=astrid RUN useradd -m $DOCKER_USER +RUN chmod 777 /home # change workdir WORKDIR /app diff --git a/docker/README.md b/docker/README.md index 905d21a..e20cdf5 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,9 +1,9 @@ # Docker container - -## Vývoj - nutnost: nainstalovaný `docker` a `docker-compose-plugin` - vytvoření aktuální image: `docker compose build` (potřeba spustit před prvním spuštěním) - vytvoření mount složky `data` (případně `chown`, aby byla vlastněna uživatelem, pod kterým kontejner poběží) - spuštění: `docker compose up` - spouštění příkazů uvnitř dockeru: `docker exec -it astrid ` - stáhnutí `fykosak/buildtools` (potřeba před prvním buildem repozitáře): `docker exec -it astrid podman pull docker.io/fykosak/buildtools` +- po fungování gitu potřeba přidat ssh klíč na cílový server + - je možné, že ssh bude vyhazovat chybu špatně nastaveného přístupu k privátnímu klíči, v takovém případě je potřeba přes `chmod` nastavit privátnímu klíči práva `600` diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 21eed2b..e1f0c38 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -10,4 +10,5 @@ services: privileged: true # needed for containers volumes: - ./data:/data + - /etc/passwd:/etc/passwd:ro # needed for ssh-keygen to work user: 1000:1000 # expects the main user as uid:pid diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 485992e..22f6d14 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -13,6 +13,7 @@ services: privileged: true # needed for containers volumes: - ./data:/data + - /etc/passwd:/etc/passwd:ro # needed for ssh-keygen to work ports: - 8080:8080 # opened port mapping, not needed with proxy user: 1000:1000 # expects the main user as uid:pid diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 85fff87..79d8fd4 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -7,6 +7,11 @@ if [ "$DATA_OWNER" -ne "$UID" ]; then exit 1 fi +# create home folder +export HOME="/home/$(id -u)" +mkdir -p $HOME + +# create needed files if missing mkdir -p /data/config /data/containers /data/log /data/repos /data/ssh cp -n /app/config.ini.sample /data/config/config.ini From f84f49cd7d60f370b90e1bbf0772b3b27b3667c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Thu, 28 Dec 2023 23:33:50 +0100 Subject: [PATCH 09/17] create docker as root for user creation --- astrid/pages.py | 3 +-- docker/Dockerfile | 9 +-------- docker/docker-compose.prod.yml | 3 ++- docker/docker-compose.yml | 4 ++-- docker/entrypoint.sh | 37 ++++++++++++++++++++++++---------- repos.ini.sample | 2 +- 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/astrid/pages.py b/astrid/pages.py index 653f29a..6cdd069 100644 --- a/astrid/pages.py +++ b/astrid/pages.py @@ -81,14 +81,13 @@ def default(self, reponame): lock.release() def _updateRepo(self, reponame): - print(f"Updating repository {reponame}") remotepath = self.repos.get(reponame, "path") submodules = self.repos.get(reponame, "submodules") if self.repos.has_option(reponame, "submodules") else False localpath = os.path.join(self.repodir, reponame) os.umask(0o007) # create repo content not readable to others if not os.path.isdir(localpath): - print(f"Repository {reponame} empty, creating") + print(f"Repository {reponame} empty, cloning") g = Git() g.clone(remotepath, localpath) diff --git a/docker/Dockerfile b/docker/Dockerfile index d798ac1..04e9e98 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,11 +3,6 @@ FROM python:3.12 # install docker RUN apt update && apt install -y podman containers-storage -# add astrid user -ARG DOCKER_USER=astrid -RUN useradd -m $DOCKER_USER -RUN chmod 777 /home - # change workdir WORKDIR /app @@ -23,9 +18,7 @@ COPY ./docker/sshd.conf /etc/ssh/sshd_config.d/99-astrid.conf COPY ./docker/libpod.conf /etc/containers/libpod.conf COPY ./docker/storage.conf /etc/containers/storage.conf -# install astrid under astrid user +# install astrid COPY . . -RUN chown -R $DOCKER_USER:$DOCKER_USER /app -USER $DOCKER_USER ENTRYPOINT ./docker/entrypoint.sh diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index e1f0c38..6ea7561 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -7,8 +7,9 @@ services: container_name: astrid environment: TZ: 'Europe/Prague' + PUID: 1000 + GUID: 1000 privileged: true # needed for containers volumes: - ./data:/data - /etc/passwd:/etc/passwd:ro # needed for ssh-keygen to work - user: 1000:1000 # expects the main user as uid:pid diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 22f6d14..b12e92b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,10 +10,10 @@ services: container_name: astrid environment: TZ: 'Europe/Prague' + PUID: 1000 + GUID: 1000 privileged: true # needed for containers volumes: - ./data:/data - - /etc/passwd:/etc/passwd:ro # needed for ssh-keygen to work ports: - 8080:8080 # opened port mapping, not needed with proxy - user: 1000:1000 # expects the main user as uid:pid diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 79d8fd4..a3e6d97 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,24 +1,39 @@ #!/bin/bash -DATA_OWNER=$(stat -c '%u' /data) +set -e -if [ "$DATA_OWNER" -ne "$UID" ]; then - echo "Directory 'data' not owned by target user with $UID, instead owned by user with uid $DATA_OWNER" +# check required variables +if [ -z "$PUID" ]; then + echo 'Environment variable $PUID not specified' exit 1 fi -# create home folder -export HOME="/home/$(id -u)" -mkdir -p $HOME +if [ -z "$GUID" ]; then + echo 'Environment variable $PUID not specified' + exit 1 +fi + +# create astrid user and group +if [ ! $(getent group astrid) ]; then + groupadd --gid $GUID astrid + echo "Group astrid with GID $GUID created." +fi +if [ ! $(getent passwd astrid) ]; then + useradd --uid $PUID --gid $GUID --create-home --add-subids-for-system astrid + echo "User astrid with UID $PUID created." +fi + +# set ownership of /data to target user +chown "$PUID:$GUID" /data # create needed files if missing -mkdir -p /data/config /data/containers /data/log /data/repos /data/ssh +su - astrid -c "mkdir -p /data/config /data/containers /data/log /data/repos /data/ssh" -cp -n /app/config.ini.sample /data/config/config.ini -cp -n /app/repos.ini.sample /data/config/repos.ini +su - astrid -c "cp -n /app/config.ini.sample /data/config/config.ini" +su - astrid -c "cp -n /app/repos.ini.sample /data/config/repos.ini" if [ $(ls "/data/ssh" | grep ".pub" | wc -l) -eq 0 ]; then - ssh-keygen -t ed25519 -f /data/ssh/id_ed25519 + su - astrid -c "ssh-keygen -t ed25519 -f /data/ssh/id_ed25519" fi -python3 -u ./main +su - astrid -c "python3 -u /app/main" diff --git a/repos.ini.sample b/repos.ini.sample index 9c658e8..fddac6e 100644 --- a/repos.ini.sample +++ b/repos.ini.sample @@ -1,6 +1,6 @@ [fykos37] path=gitea@fykos.cz:FYKOS/fykos37.git -users=fykos,repo +users=user image_version=latest build_usr=astrid build_cmd=make -k all From 9b4ef39279fe9f1e318572a54af8bb20faf4d086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Thu, 28 Dec 2023 23:46:26 +0100 Subject: [PATCH 10/17] fix group and user existance check --- docker/docker-compose.yml | 2 +- docker/entrypoint.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b12e92b..2180bee 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,7 +11,7 @@ services: environment: TZ: 'Europe/Prague' PUID: 1000 - GUID: 1000 + GUID: 65534 privileged: true # needed for containers volumes: - ./data:/data diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a3e6d97..34b421a 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -9,16 +9,16 @@ if [ -z "$PUID" ]; then fi if [ -z "$GUID" ]; then - echo 'Environment variable $PUID not specified' + echo 'Environment variable $GUID not specified' exit 1 fi # create astrid user and group -if [ ! $(getent group astrid) ]; then +if [ ! $(getent group astrid) ] && [ ! $(getent group $GUID) ]; then groupadd --gid $GUID astrid echo "Group astrid with GID $GUID created." fi -if [ ! $(getent passwd astrid) ]; then +if [ ! $(getent passwd astrid) ] && [ ! $(getent passwd $PUID) ]; then useradd --uid $PUID --gid $GUID --create-home --add-subids-for-system astrid echo "User astrid with UID $PUID created." fi From 0fa27705a8104fb573d035590d213ac0fece27ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Fri, 29 Dec 2023 00:22:22 +0100 Subject: [PATCH 11/17] fix subuid --- docker/README.md | 2 ++ docker/docker-compose.yml | 2 +- docker/entrypoint.sh | 29 ++++++++++++++++++++++++----- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/docker/README.md b/docker/README.md index e20cdf5..15cab8a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -7,3 +7,5 @@ - stáhnutí `fykosak/buildtools` (potřeba před prvním buildem repozitáře): `docker exec -it astrid podman pull docker.io/fykosak/buildtools` - po fungování gitu potřeba přidat ssh klíč na cílový server - je možné, že ssh bude vyhazovat chybu špatně nastaveného přístupu k privátnímu klíči, v takovém případě je potřeba přes `chmod` nastavit privátnímu klíči práva `600` +- Možno přes env_var `PUID` a `GUID` nastavit uživatele, pod kterým to pojede. Pozor ale na UID, které již můžou existovat v dockeru, potom se to rozbije. + - pokud je PUID nebo GUID změněno, je potřeba smazat starý kontejner (`docker rm astrid`) a znovu jej vytvořit diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2180bee..5288866 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,7 +10,7 @@ services: container_name: astrid environment: TZ: 'Europe/Prague' - PUID: 1000 + PUID: 950 GUID: 65534 privileged: true # needed for containers volumes: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 34b421a..a13e2e5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,6 +2,13 @@ set -e +function checksubid { + if [ $(grep -E "^$1:" $2 | wc -l) -eq 0 ]; then + exit 1 + fi + exit 0 +} + # check required variables if [ -z "$PUID" ]; then echo 'Environment variable $PUID not specified' @@ -23,17 +30,29 @@ if [ ! $(getent passwd astrid) ] && [ ! $(getent passwd $PUID) ]; then echo "User astrid with UID $PUID created." fi +USER=$(id -nu $PUID) + +# add to subuid and subgid +if ! $(checksubid $USER /etc/subuid); then + echo "$USER:100000:65536" >> /etc/subuid +fi + +if ! $(checksubid $USER /etc/subgid); then + echo "$USER:100000:65536" >> /etc/subgid +fi + # set ownership of /data to target user chown "$PUID:$GUID" /data + # create needed files if missing -su - astrid -c "mkdir -p /data/config /data/containers /data/log /data/repos /data/ssh" +su - $USER -c "mkdir -p /data/config /data/containers /data/log /data/repos /data/ssh" -su - astrid -c "cp -n /app/config.ini.sample /data/config/config.ini" -su - astrid -c "cp -n /app/repos.ini.sample /data/config/repos.ini" +su - $USER -c "cp -n /app/config.ini.sample /data/config/config.ini" +su - $USER -c "cp -n /app/repos.ini.sample /data/config/repos.ini" if [ $(ls "/data/ssh" | grep ".pub" | wc -l) -eq 0 ]; then - su - astrid -c "ssh-keygen -t ed25519 -f /data/ssh/id_ed25519" + su - $USER -c "ssh-keygen -t ed25519 -f /data/ssh/id_ed25519" fi -su - astrid -c "python3 -u /app/main" +su - $USER -c "python3 -u /app/main" From 7adecd7c293e5a6a1cbaf31b8a5658956abb1809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Fri, 29 Dec 2023 12:29:40 +0100 Subject: [PATCH 12/17] yeet podman --- .dockerignore | 3 +-- astrid/pages.py | 2 +- docker/Dockerfile | 5 ++--- docker/daemon.json | 6 ++++++ docker/docker-compose.yml | 4 ++-- docker/entrypoint.sh | 2 ++ docker/libpod.conf | 1 - docker/storage.conf | 3 --- 8 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 docker/daemon.json delete mode 100644 docker/libpod.conf delete mode 100644 docker/storage.conf diff --git a/.dockerignore b/.dockerignore index e780801..234ea3e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,5 +2,4 @@ docker/* !docker/entrypoint.sh !docker/ssh.conf !docker/sshd.conf -!docker/libpod.conf -!docker/storage.conf +!docker/daemon.json diff --git a/astrid/pages.py b/astrid/pages.py index 6cdd069..9059bf3 100644 --- a/astrid/pages.py +++ b/astrid/pages.py @@ -126,7 +126,7 @@ def _build(self, reponame): print(f"Building {reponame}") logfilename = os.path.expanduser(f"/data/log/{reponame}.build.log") logfile = open(logfilename, "w") - p = subprocess.run(["podman", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) + p = subprocess.run(["docker", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) logfile.close() return p.returncode == 0 diff --git a/docker/Dockerfile b/docker/Dockerfile index 04e9e98..6a93e43 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.12 # install docker -RUN apt update && apt install -y podman containers-storage +RUN apt update && apt install -y docker.io containers-storage # change workdir WORKDIR /app @@ -15,8 +15,7 @@ COPY ./docker/ssh.conf /etc/ssh/ssh_config.d/99-astrid.conf COPY ./docker/sshd.conf /etc/ssh/sshd_config.d/99-astrid.conf # copy containers config -COPY ./docker/libpod.conf /etc/containers/libpod.conf -COPY ./docker/storage.conf /etc/containers/storage.conf +COPY ./docker/daemon.json /etc/docker/daemon.json # install astrid COPY . . diff --git a/docker/daemon.json b/docker/daemon.json new file mode 100644 index 0000000..5996548 --- /dev/null +++ b/docker/daemon.json @@ -0,0 +1,6 @@ +{ + "data-root": "/data/containers", + "live-restore": true, + "log-driver": "json-file", + "init": true +} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5288866..b12e92b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,8 +10,8 @@ services: container_name: astrid environment: TZ: 'Europe/Prague' - PUID: 950 - GUID: 65534 + PUID: 1000 + GUID: 1000 privileged: true # needed for containers volumes: - ./data:/data diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a13e2e5..90205fc 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -31,6 +31,7 @@ if [ ! $(getent passwd astrid) ] && [ ! $(getent passwd $PUID) ]; then fi USER=$(id -nu $PUID) +usermod -a -G docker $USER # add to subuid and subgid if ! $(checksubid $USER /etc/subuid); then @@ -55,4 +56,5 @@ if [ $(ls "/data/ssh" | grep ".pub" | wc -l) -eq 0 ]; then su - $USER -c "ssh-keygen -t ed25519 -f /data/ssh/id_ed25519" fi +dockerd & su - $USER -c "python3 -u /app/main" diff --git a/docker/libpod.conf b/docker/libpod.conf deleted file mode 100644 index 86e8062..0000000 --- a/docker/libpod.conf +++ /dev/null @@ -1 +0,0 @@ -static_dir = "/data/containers/storage/libpod" diff --git a/docker/storage.conf b/docker/storage.conf deleted file mode 100644 index f5091cd..0000000 --- a/docker/storage.conf +++ /dev/null @@ -1,3 +0,0 @@ -[storage] -driver = "overlay" -rootless_storage_path = "/data/containers/storage" From b728cc155e170efb89f15700d5ef0b7196fd1e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Fri, 29 Dec 2023 13:26:51 +0100 Subject: [PATCH 13/17] run as current user --- astrid/pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrid/pages.py b/astrid/pages.py index 9059bf3..f2e8930 100644 --- a/astrid/pages.py +++ b/astrid/pages.py @@ -126,7 +126,7 @@ def _build(self, reponame): print(f"Building {reponame}") logfilename = os.path.expanduser(f"/data/log/{reponame}.build.log") logfile = open(logfilename, "w") - p = subprocess.run(["docker", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) + p = subprocess.run(["docker", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"--user={os.getuid()}", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) logfile.close() return p.returncode == 0 From af69841af3820c8749b37fe382f233aa69a37685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Fri, 29 Dec 2023 13:29:11 +0100 Subject: [PATCH 14/17] include docker dependencies --- .github/workflows/deploy-image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index c4de4b4..ee65abb 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -6,7 +6,10 @@ on: - master - dev-docker paths: + - 'astrid/**' - 'docker/**' + - 'main' + - 'requirements.txt' - '.github/workflows/deploy-image.yml' env: From 737a02d21a01ce32dc93229826c81307af6f3282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Fri, 29 Dec 2023 14:36:39 +0100 Subject: [PATCH 15/17] run as gid --- astrid/pages.py | 2 +- docker/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/astrid/pages.py b/astrid/pages.py index f2e8930..eb7133d 100644 --- a/astrid/pages.py +++ b/astrid/pages.py @@ -126,7 +126,7 @@ def _build(self, reponame): print(f"Building {reponame}") logfilename = os.path.expanduser(f"/data/log/{reponame}.build.log") logfile = open(logfilename, "w") - p = subprocess.run(["docker", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"--user={os.getuid()}", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) + p = subprocess.run(["docker", "run", "--rm", "-v", f"{cwd}:/usr/src/local", f"--user={os.getuid()}:{os.getgid()}", f"fykosak/buildtools:{image_version}"] + cmd.split(), cwd=cwd, stdout=logfile, stderr=logfile, check=False) logfile.close() return p.returncode == 0 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b12e92b..2180bee 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,7 +11,7 @@ services: environment: TZ: 'Europe/Prague' PUID: 1000 - GUID: 1000 + GUID: 65534 privileged: true # needed for containers volumes: - ./data:/data From fc52a938c65a6fdc3ac93bae041e10e00b731206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Sat, 30 Dec 2023 22:41:40 +0100 Subject: [PATCH 16/17] instant http response --- astrid/__init__.py | 12 +++--- astrid/pages.py | 94 +++++++++++++++++++++++++++++----------------- 2 files changed, 65 insertions(+), 41 deletions(-) diff --git a/astrid/__init__.py b/astrid/__init__.py index 2318b44..90d7a97 100644 --- a/astrid/__init__.py +++ b/astrid/__init__.py @@ -18,10 +18,9 @@ def _getLogfile(self, reponame): def _getBuildlogfile(self, reponame): return f'/data/log/{reponame}.build.log' - def log(self, reponame, message, sendMail = False): + def log(self, reponame, user, message, sendMail = False): logfile = self._getLogfile(reponame) f = open(logfile, "a") - user = cherrypy.request.login f.write(BuildLogger.separator.join([datetime.now().isoformat(' '), message, user]) + "\n") f.close() @@ -80,12 +79,13 @@ def _sendMail(self, reponame, message): # prepare locks for each repository locks = {section: threading.Lock() for section in repos.sections()} +waitLocks = {section: threading.Lock() for section in repos.sections()} from astrid.pages import BuilderPage, InfoPage, DashboardPage, BuildlogPage repodir = cherrypy.config.get("repodir") -root = DashboardPage(repodir, repos, locks) -root.build = BuilderPage(repodir, repos, locks) -root.info = InfoPage(repodir, repos, locks) -root.buildlog = BuildlogPage(repodir, repos, locks) +root = DashboardPage(repodir, repos, locks, waitLocks) +root.build = BuilderPage(repodir, repos, locks, waitLocks) +root.info = InfoPage(repodir, repos, locks, waitLocks) +root.buildlog = BuildlogPage(repodir, repos, locks, waitLocks) diff --git a/astrid/pages.py b/astrid/pages.py index eb7133d..8969856 100644 --- a/astrid/pages.py +++ b/astrid/pages.py @@ -2,6 +2,7 @@ import os import subprocess import time +from threading import Thread import cherrypy from git import Repo, Git, GitCommandError @@ -11,10 +12,11 @@ import astrid.server class BasePage(object): - def __init__(self, repodir, repos, locks): + def __init__(self, repodir, repos, locks, waitLocks): self.repos = repos self.repodir = repodir self.locks = locks + self.waitLocks = waitLocks def _checkAccess(self, reponame): user = cherrypy.request.login @@ -37,48 +39,70 @@ def default(self, reponame): raise cherrypy.HTTPError(404) self._checkAccess(reponame) + + user = cherrypy.request.login + + thread = Thread(target=self._queueJob, args=(reponame,user)) + thread.daemon = True + thread.start() + + raise cherrypy.HTTPRedirect("/") + + def _queueJob(self, reponame, user): lock = self.locks[reponame] + waitLock = self.waitLocks[reponame] + + # if job is not running + if lock.acquire(blocking=False): + # lock job running and run job + try: + self._runJob(reponame, user) + finally: # ensure lock release on exception + lock.release() # release lock for job run + return + + # if job is already running + if not waitLock.acquire(blocking=False): # if another process is already waiting + return # skip waiting + + # if it's not waiting + # we now own lock for waiting + lock.acquire() # wait for job lock to release and acquire it + waitLock.release() # relase lock for waiting + try: + self._runJob(reponame, user) # run job + finally: # ensure lock release on exception + lock.release() # release lock for job run + + def _runJob(self, reponame, user): msg = None sendMail = False - lock.acquire() + time_start = time.time() + time_end = None + try: + repo = self._updateRepo(reponame) try: - time_start = time.time() - time_end = None - repo = self._updateRepo(reponame) - try: - if self._build(reponame): - msg = f"#{repo.head.commit.hexsha[0:6]} Build succeeded." - msg_class = "green" - time_end = time.time() - else: - msg = f"#{repo.head.commit.hexsha[0:6]} Build failed." - msg_class = "red" - time_end = time.time() - except: - msg = f"#{repo.head.commit.hexsha[0:6]} Build error." - msg_class = "red" - raise - except GitCommandError as e: - msg = f"Pull error. ({e})" - msg_class = "red" - sendMail = True + if self._build(reponame): + msg = f"#{repo.head.commit.hexsha[0:6]} Build succeeded." + time_end = time.time() + else: + msg = f"#{repo.head.commit.hexsha[0:6]} Build failed." + time_end = time.time() + except: + msg = f"#{repo.head.commit.hexsha[0:6]} Build error." raise + except GitCommandError as e: + msg = f"Pull error. ({e})" + sendMail = True + raise - if time_end is not None: - msg += " ({:.2f} s)".format(time_end - time_start) - - template = Template("templates/build.html") - template.assignData("reponame", reponame) - template.assignData("message", msg) - template.assignData("msg_class", msg_class) - template.assignData("pagetitle", "Build") - return template.render() - finally: - logger = BuildLogger(self.repodir) - logger.log(reponame, msg, sendMail) - lock.release() + if time_end is not None: + msg += " ({:.2f} s)".format(time_end - time_start) + + logger = BuildLogger(self.repodir) + logger.log(reponame, user, msg, sendMail) def _updateRepo(self, reponame): remotepath = self.repos.get(reponame, "path") From 0f26856127bc27f3f52a951291dda33b508ae30c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kr=C5=A1ka?= Date: Sat, 6 Jan 2024 20:00:29 +0100 Subject: [PATCH 17/17] update production mounts --- docker/docker-compose.prod.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 6ea7561..1313e6e 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -12,4 +12,3 @@ services: privileged: true # needed for containers volumes: - ./data:/data - - /etc/passwd:/etc/passwd:ro # needed for ssh-keygen to work