From 6ba03a78a975e476d61f3e427d69a4b67c8371c7 Mon Sep 17 00:00:00 2001 From: Xenorf <90411648+Xenorf@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:46:20 +0000 Subject: [PATCH 01/15] Adding support for x11 forwarding --- exegol/model/ExegolContainer.py | 55 ++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index e49f4800..2f646059 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -1,4 +1,6 @@ import os +import subprocess +import random import shutil from datetime import datetime from typing import Optional, Dict, Sequence, Tuple, Union @@ -16,6 +18,7 @@ from exegol.utils.ContainerLogStream import ContainerLogStream from exegol.utils.ExeLog import logger, console from exegol.utils.imgsync.ImageScriptSync import ImageScriptSync +from exegol.utils.GuiUtils import GuiUtils class ExegolContainer(ExegolContainerTemplate, SelectableInterface): @@ -156,7 +159,7 @@ def spawnShell(self): logger.info(f"Shared host device: {device.split(':')[0]}") logger.success(f"Opening shell in Exegol '{self.name}'") # In case of multi-user environment, xhost must be set before opening each session to be sure - self.__applyXhostACL() + self.__applyX11ACLs() # Using system command to attach the shell to the user terminal (stdin / stdout / stderr) envs = self.config.getShellEnvs() options = "" @@ -281,7 +284,7 @@ def __preStartSetup(self): Operation to be performed before starting a container :return: """ - self.__applyXhostACL() + self.__applyX11ACLs() def __check_start_version(self): """ @@ -305,7 +308,7 @@ def postCreateSetup(self, is_temporary: bool = False): Operation to be performed after creating a container :return: """ - self.__applyXhostACL() + self.__applyX11ACLs() # if not a temporary container, apply custom config if not is_temporary: # Update entrypoint script in the container @@ -318,10 +321,11 @@ def postCreateSetup(self, is_temporary: bool = False): if "is not running" in e.explanation: logger.critical("An unexpected error occurred. Exegol cannot start the container after its creation...") - def __applyXhostACL(self): + def __applyX11ACLs(self): """ If X11 (GUI) is enabled, allow X11 access on host ACL (if not already allowed) for linux and mac. - On Windows host, WSLg X11 don't have xhost ACL. + If the host is accessed by SSH, propagate xauth cookie authentication if applicable. + On Windows host, WSLg X11 don't have xhost ACL. #TODO xauth remote x11 forwarding :return: """ if self.config.isGUIEnable() and not self.__xhost_applied and not EnvInfo.isWindowsHost(): @@ -334,16 +338,39 @@ def __applyXhostACL(self): logger.error(f"The [green]xhost[/green] command is not available on your [bold]host[/bold]. " f"Exegol was unable to allow your container to access your graphical environment ({debug_msg}).") return - - if EnvInfo.isMacHost(): - logger.debug(f"Adding xhost ACL to localhost") - # add xquartz inet ACL - with console.status(f"Starting XQuartz...", spinner_style="blue"): - os.system(f"xhost + localhost > /dev/null") + + # Another way of passing the argument without importing subprocess: os.system('xauth list | grep "${DISPLAY%.*}" > '+f"{self.config.getPrivateVolumePath()}/testyu") + logger.debug(f"DISPLAY variable: {GuiUtils.getDisplayEnv()}") + tmpXauthority = f"/tmp/.tmpXauthority{random.randint(0, 999)}" + output = os.system(f"xauth extract {tmpXauthority} $DISPLAY > /dev/null 2>&1") + xauthEntry = subprocess.check_output(f"xauth -f {tmpXauthority} list 2>/dev/null",shell=True).decode() + logger.debug(f"xauthEntry to propagate: {xauthEntry}") + display_host = GuiUtils.getDisplayEnv().split(':')[0] + if display_host=='': + logger.debug("Connecting to container from local GUI, no X11 forwarding to set up") + # TODO verify that the display format is the same on macOS, otherwise might not set up xauth and xhost correctly + if EnvInfo.isMacHost(): + logger.debug(f"Adding xhost ACL to localhost") + # add xquartz inet ACL + with console.status(f"Starting XQuartz...", spinner_style="blue"): + os.system(f"xhost + localhost > /dev/null") + else: + logger.debug(f"Adding xhost ACL to local:{self.config.getUsername()}") + # add linux local ACL + os.system(f"xhost +local:{self.config.getUsername()} > /dev/null") + elif display_host=="localhost" and self.config.getTextNetworkMode() == "bridge": + logger.warning("X11 forwarding won't work on a bridged container unless you specify \"X11UseLocalhost no\" in your host sshd_config") + elif display_host=="localhost": + logger.debug("X11UseLocalhost directive is set to \"yes\" or unspecified, X11 connection can be received only on loopback"); + # Modifing the entry to convert /unix: to localhost: + xauthEntry = f"localhost:{xauthEntry.split(':')[1]}" else: - logger.debug(f"Adding xhost ACL to local:{self.config.getUsername()}") - # add linux local ACL - os.system(f"xhost +local:{self.config.getUsername()} > /dev/null") + logger.debug("X11UseLocalhost directive is set to \"no\", X11 connection can be received from anywere"); + if xauthEntry: + logger.debug(f"Adding xauth cookie to container: {xauthEntry}") + self.exec(f"xauth add {xauthEntry}", as_daemon=False, quiet=True) + logger.debug(f"Removing {tmpXauthority}") + os.remove(tmpXauthority) def __updatePasswd(self): """ From 9cd86aac7716cc28eedc5ae2c7560cdad507a683 Mon Sep 17 00:00:00 2001 From: Xenorf <90411648+Xenorf@users.noreply.github.com> Date: Sat, 31 Aug 2024 19:26:55 +0000 Subject: [PATCH 02/15] improved messages and initialisation --- exegol/model/ExegolContainer.py | 34 ++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 2f646059..2264fbd7 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -339,12 +339,8 @@ def __applyX11ACLs(self): f"Exegol was unable to allow your container to access your graphical environment ({debug_msg}).") return - # Another way of passing the argument without importing subprocess: os.system('xauth list | grep "${DISPLAY%.*}" > '+f"{self.config.getPrivateVolumePath()}/testyu") logger.debug(f"DISPLAY variable: {GuiUtils.getDisplayEnv()}") - tmpXauthority = f"/tmp/.tmpXauthority{random.randint(0, 999)}" - output = os.system(f"xauth extract {tmpXauthority} $DISPLAY > /dev/null 2>&1") - xauthEntry = subprocess.check_output(f"xauth -f {tmpXauthority} list 2>/dev/null",shell=True).decode() - logger.debug(f"xauthEntry to propagate: {xauthEntry}") + display_host = GuiUtils.getDisplayEnv().split(':')[0] if display_host=='': logger.debug("Connecting to container from local GUI, no X11 forwarding to set up") @@ -358,19 +354,43 @@ def __applyX11ACLs(self): logger.debug(f"Adding xhost ACL to local:{self.config.getUsername()}") # add linux local ACL os.system(f"xhost +local:{self.config.getUsername()} > /dev/null") - elif display_host=="localhost" and self.config.getTextNetworkMode() == "bridge": + return + + if shutil.which("xauth") is None: + if EnvInfo.is_linux_shell: + debug_msg = "Try to install the package [green]xorg-xauth[/green] or maybe you don't have X11 on your host?" + else: + debug_msg = "or you don't have one" + logger.error(f"The [green]xauth[/green] command is not available on your [bold]host[/bold]. " + f"Exegol was unable to allow your container to access your graphical environment ({debug_msg}).") + return + + if display_host=="localhost" and self.config.getTextNetworkMode() != "host": logger.warning("X11 forwarding won't work on a bridged container unless you specify \"X11UseLocalhost no\" in your host sshd_config") - elif display_host=="localhost": + logger.warning("[red]Be aware[/red] changing \"X11UseLocalhost\" value can [red]expose your device[/red], correct firewalling is [red]required[/red]") + logger.warning("The following documentation can be usefull to limit the exposure of your x11 socket: https://studioware.com/wikislax/index.php?title=X11_over_the_network#X11_firewalling") + return + + # Setting up the temporary file to pass the xauth cookie to the container + tmpXauthority = f"/tmp/.tmpXauthority{random.randint(0, 999)}" + logger.debug(f"Extracting xauth entries to {tmpXauthority}") + os.system(f"xauth extract {tmpXauthority} $DISPLAY > /dev/null 2>&1") + xauthEntry = subprocess.check_output(f"xauth -f {tmpXauthority} list 2>/dev/null",shell=True).decode() + logger.debug(f"xauthEntry to propagate: {xauthEntry}") + if display_host=="localhost": logger.debug("X11UseLocalhost directive is set to \"yes\" or unspecified, X11 connection can be received only on loopback"); # Modifing the entry to convert /unix: to localhost: xauthEntry = f"localhost:{xauthEntry.split(':')[1]}" else: logger.debug("X11UseLocalhost directive is set to \"no\", X11 connection can be received from anywere"); + # Check if the host has a xauth entry corresponding to the current display. if xauthEntry: logger.debug(f"Adding xauth cookie to container: {xauthEntry}") self.exec(f"xauth add {xauthEntry}", as_daemon=False, quiet=True) logger.debug(f"Removing {tmpXauthority}") os.remove(tmpXauthority) + else: + logger.warning(f"No xauth cookie corresponding to the current display was found.") def __updatePasswd(self): """ From 33866efb8a27bb8fdaa907812deb6b6b72515c02 Mon Sep 17 00:00:00 2001 From: Xenorf <90411648+Xenorf@users.noreply.github.com> Date: Sun, 1 Sep 2024 09:41:17 +0000 Subject: [PATCH 03/15] better tmp file and logs + comments --- exegol/model/ExegolContainer.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 2264fbd7..45cdb617 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -1,6 +1,6 @@ import os import subprocess -import random +import tempfile import shutil from datetime import datetime from typing import Optional, Dict, Sequence, Tuple, Union @@ -328,6 +328,7 @@ def __applyX11ACLs(self): On Windows host, WSLg X11 don't have xhost ACL. #TODO xauth remote x11 forwarding :return: """ + # TODO check if the xauth propagation should be performed on Windows, if so remove the "and not EnvInfo.isWindowsHost()" if self.config.isGUIEnable() and not self.__xhost_applied and not EnvInfo.isWindowsHost(): self.__xhost_applied = True # Can be applied only once per execution if shutil.which("xhost") is None: @@ -340,17 +341,18 @@ def __applyX11ACLs(self): return logger.debug(f"DISPLAY variable: {GuiUtils.getDisplayEnv()}") - + # Extracts the left part of the display variable to determine if remote access is used display_host = GuiUtils.getDisplayEnv().split(':')[0] + # Left part is empty, local access is used to start Exegol if display_host=='': logger.debug("Connecting to container from local GUI, no X11 forwarding to set up") - # TODO verify that the display format is the same on macOS, otherwise might not set up xauth and xhost correctly + # TODO verify that the display format is the same on macOS, otherwise might not set up xauth and xhost correctly if EnvInfo.isMacHost(): logger.debug(f"Adding xhost ACL to localhost") # add xquartz inet ACL with console.status(f"Starting XQuartz...", spinner_style="blue"): os.system(f"xhost + localhost > /dev/null") - else: + elif not EnvInfo.isWindowsHost(): logger.debug(f"Adding xhost ACL to local:{self.config.getUsername()}") # add linux local ACL os.system(f"xhost +local:{self.config.getUsername()} > /dev/null") @@ -358,31 +360,37 @@ def __applyX11ACLs(self): if shutil.which("xauth") is None: if EnvInfo.is_linux_shell: - debug_msg = "Try to install the package [green]xorg-xauth[/green] or maybe you don't have X11 on your host?" + debug_msg = "Try to install the package [green]xorg-xauth[/green] to support X11 forwarding in your current environment?" else: - debug_msg = "or you don't have one" + debug_msg = "or it might not be supported for now" logger.error(f"The [green]xauth[/green] command is not available on your [bold]host[/bold]. " f"Exegol was unable to allow your container to access your graphical environment ({debug_msg}).") return - if display_host=="localhost" and self.config.getTextNetworkMode() != "host": + # If the left part of the display variable is "localhost", x11 socket is exposed only on loopback and remote access is used + # If the container is not in host mode, it won't be able to reach the loopback interface of the host + if display_host=="localhost" and self.config.getNetworkMode() != "host": logger.warning("X11 forwarding won't work on a bridged container unless you specify \"X11UseLocalhost no\" in your host sshd_config") logger.warning("[red]Be aware[/red] changing \"X11UseLocalhost\" value can [red]expose your device[/red], correct firewalling is [red]required[/red]") - logger.warning("The following documentation can be usefull to limit the exposure of your x11 socket: https://studioware.com/wikislax/index.php?title=X11_over_the_network#X11_firewalling") + # TODO Add documentation to restrict the exposure of the x11 socket to the docker subnet return - # Setting up the temporary file to pass the xauth cookie to the container - tmpXauthority = f"/tmp/.tmpXauthority{random.randint(0, 999)}" + # Extracting the xauth cookie corresponding to the current display to a temporary file and reading it from there (grep cannot be used because display names are not accurate enough) + _, tmpXauthority = tempfile.mkstemp() logger.debug(f"Extracting xauth entries to {tmpXauthority}") os.system(f"xauth extract {tmpXauthority} $DISPLAY > /dev/null 2>&1") xauthEntry = subprocess.check_output(f"xauth -f {tmpXauthority} list 2>/dev/null",shell=True).decode() logger.debug(f"xauthEntry to propagate: {xauthEntry}") + + # Replacing the hostname with localhost to support loopback exposed x11 socket and container in host mode (loopback is the same) if display_host=="localhost": - logger.debug("X11UseLocalhost directive is set to \"yes\" or unspecified, X11 connection can be received only on loopback"); + logger.debug("X11UseLocalhost directive is set to \"yes\" or unspecified, X11 connections can be received only on loopback"); # Modifing the entry to convert /unix: to localhost: xauthEntry = f"localhost:{xauthEntry.split(':')[1]}" else: - logger.debug("X11UseLocalhost directive is set to \"no\", X11 connection can be received from anywere"); + # TODO latter implement a check to see if the x11 socket is correctly firewalled and warn the user if it is not + logger.debug("X11UseLocalhost directive is set to \"no\", X11 connections can be received from anywere"); + # Check if the host has a xauth entry corresponding to the current display. if xauthEntry: logger.debug(f"Adding xauth cookie to container: {xauthEntry}") From eaab349ed2b2f62d8ca34a01b9591e04dc3c62b4 Mon Sep 17 00:00:00 2001 From: Xenorf <90411648+Xenorf@users.noreply.github.com> Date: Wed, 4 Sep 2024 09:21:32 +0000 Subject: [PATCH 04/15] x11 forwarding not supported macOS --- exegol/model/ExegolContainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 45cdb617..69c60f71 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -344,7 +344,7 @@ def __applyX11ACLs(self): # Extracts the left part of the display variable to determine if remote access is used display_host = GuiUtils.getDisplayEnv().split(':')[0] # Left part is empty, local access is used to start Exegol - if display_host=='': + if display_host=='' or EnvInfo.isMacHost(): logger.debug("Connecting to container from local GUI, no X11 forwarding to set up") # TODO verify that the display format is the same on macOS, otherwise might not set up xauth and xhost correctly if EnvInfo.isMacHost(): From 4178c5d6dc02ac4e69290c0987946e003d8f72f4 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 4 Sep 2024 17:31:26 +0200 Subject: [PATCH 05/15] Ignore PR pipeline on master --- .github/workflows/entrypoint_pull_request.yml | 1 + exegol/model/ExegolContainer.py | 20 +++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/entrypoint_pull_request.yml b/.github/workflows/entrypoint_pull_request.yml index f5016f9a..a437dcd9 100644 --- a/.github/workflows/entrypoint_pull_request.yml +++ b/.github/workflows/entrypoint_pull_request.yml @@ -13,6 +13,7 @@ on: push: branches-ignore: - "dev" + - "master" paths-ignore: - ".github/**" - "**.md" diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 69c60f71..bf3b108a 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -1,7 +1,7 @@ import os +import shutil import subprocess import tempfile -import shutil from datetime import datetime from typing import Optional, Dict, Sequence, Tuple, Union @@ -17,8 +17,8 @@ from exegol.model.SelectableInterface import SelectableInterface from exegol.utils.ContainerLogStream import ContainerLogStream from exegol.utils.ExeLog import logger, console -from exegol.utils.imgsync.ImageScriptSync import ImageScriptSync from exegol.utils.GuiUtils import GuiUtils +from exegol.utils.imgsync.ImageScriptSync import ImageScriptSync class ExegolContainer(ExegolContainerTemplate, SelectableInterface): @@ -339,12 +339,12 @@ def __applyX11ACLs(self): logger.error(f"The [green]xhost[/green] command is not available on your [bold]host[/bold]. " f"Exegol was unable to allow your container to access your graphical environment ({debug_msg}).") return - + logger.debug(f"DISPLAY variable: {GuiUtils.getDisplayEnv()}") # Extracts the left part of the display variable to determine if remote access is used display_host = GuiUtils.getDisplayEnv().split(':')[0] # Left part is empty, local access is used to start Exegol - if display_host=='' or EnvInfo.isMacHost(): + if display_host == '' or EnvInfo.isMacHost(): logger.debug("Connecting to container from local GUI, no X11 forwarding to set up") # TODO verify that the display format is the same on macOS, otherwise might not set up xauth and xhost correctly if EnvInfo.isMacHost(): @@ -357,7 +357,7 @@ def __applyX11ACLs(self): # add linux local ACL os.system(f"xhost +local:{self.config.getUsername()} > /dev/null") return - + if shutil.which("xauth") is None: if EnvInfo.is_linux_shell: debug_msg = "Try to install the package [green]xorg-xauth[/green] to support X11 forwarding in your current environment?" @@ -366,10 +366,10 @@ def __applyX11ACLs(self): logger.error(f"The [green]xauth[/green] command is not available on your [bold]host[/bold]. " f"Exegol was unable to allow your container to access your graphical environment ({debug_msg}).") return - + # If the left part of the display variable is "localhost", x11 socket is exposed only on loopback and remote access is used # If the container is not in host mode, it won't be able to reach the loopback interface of the host - if display_host=="localhost" and self.config.getNetworkMode() != "host": + if display_host == "localhost" and self.config.getNetworkMode() != "host": logger.warning("X11 forwarding won't work on a bridged container unless you specify \"X11UseLocalhost no\" in your host sshd_config") logger.warning("[red]Be aware[/red] changing \"X11UseLocalhost\" value can [red]expose your device[/red], correct firewalling is [red]required[/red]") # TODO Add documentation to restrict the exposure of the x11 socket to the docker subnet @@ -379,18 +379,18 @@ def __applyX11ACLs(self): _, tmpXauthority = tempfile.mkstemp() logger.debug(f"Extracting xauth entries to {tmpXauthority}") os.system(f"xauth extract {tmpXauthority} $DISPLAY > /dev/null 2>&1") - xauthEntry = subprocess.check_output(f"xauth -f {tmpXauthority} list 2>/dev/null",shell=True).decode() + xauthEntry = subprocess.check_output(f"xauth -f {tmpXauthority} list 2>/dev/null", shell=True).decode() logger.debug(f"xauthEntry to propagate: {xauthEntry}") # Replacing the hostname with localhost to support loopback exposed x11 socket and container in host mode (loopback is the same) - if display_host=="localhost": + if display_host == "localhost": logger.debug("X11UseLocalhost directive is set to \"yes\" or unspecified, X11 connections can be received only on loopback"); # Modifing the entry to convert /unix: to localhost: xauthEntry = f"localhost:{xauthEntry.split(':')[1]}" else: # TODO latter implement a check to see if the x11 socket is correctly firewalled and warn the user if it is not logger.debug("X11UseLocalhost directive is set to \"no\", X11 connections can be received from anywere"); - + # Check if the host has a xauth entry corresponding to the current display. if xauthEntry: logger.debug(f"Adding xauth cookie to container: {xauthEntry}") From 0f00d07b3314b41b50e2e7a275994900f7c7f84e Mon Sep 17 00:00:00 2001 From: Dramelac Date: Mon, 9 Sep 2024 11:34:38 +0200 Subject: [PATCH 06/15] Add package install test on every workflows Signed-off-by: Dramelac --- .github/workflows/entrypoint_prerelease.yml | 8 ++++++-- .github/workflows/entrypoint_pull_request.yml | 8 ++++++-- .github/workflows/entrypoint_release.yml | 8 ++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/entrypoint_prerelease.yml b/.github/workflows/entrypoint_prerelease.yml index e59909ea..1391657a 100644 --- a/.github/workflows/entrypoint_prerelease.yml +++ b/.github/workflows/entrypoint_prerelease.yml @@ -53,10 +53,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install requirements - run: python -m pip install --user build + run: python -m pip install --user build virtualenv setuptools - name: Cleaning run : python setup.py clean test - name: Build Exegol run: python -m build --sdist --outdir dist/ . + - name: Create testing venv + run: python -m venv vtest + - name: Install pip package + run: ./vtest/bin/pip install ./dist/* diff --git a/.github/workflows/entrypoint_pull_request.yml b/.github/workflows/entrypoint_pull_request.yml index a437dcd9..dd974dc2 100644 --- a/.github/workflows/entrypoint_pull_request.yml +++ b/.github/workflows/entrypoint_pull_request.yml @@ -44,10 +44,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install requirements - run: python -m pip install --user build + run: python -m pip install --user build virtualenv setuptools - name: Cleaning run : python setup.py clean - name: Build Exegol run: python -m build --sdist --outdir dist/ . + - name: Create testing venv + run: python -m venv vtest + - name: Install pip package locally + run: ./vtest/bin/pip install ./dist/* diff --git a/.github/workflows/entrypoint_release.yml b/.github/workflows/entrypoint_release.yml index 36e59027..2c92b2e0 100644 --- a/.github/workflows/entrypoint_release.yml +++ b/.github/workflows/entrypoint_release.yml @@ -25,13 +25,17 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install requirements - run: python -m pip install --user build + run: python -m pip install --user build virtualenv setuptools - name: Run package unitest run : python setup.py clean test - name: Build Exegol release run: python -m build --sdist --outdir dist/ . + - name: Create testing venv + run: python -m venv vtest + - name: Install pip package locally + run: ./vtest/bin/pip install ./dist/* #- name: Publish distribution 📦 to Test PyPI # uses: pypa/gh-action-pypi-publish@release/v1 # with: From d63dceb3d78192f52168d86dc3ca0be451778f41 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Mon, 9 Sep 2024 11:35:02 +0200 Subject: [PATCH 07/15] Add libs to ignore in stacktrace Signed-off-by: Dramelac --- exegol/manager/ExegolController.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/exegol/manager/ExegolController.py b/exegol/manager/ExegolController.py index 3c90ca4f..09163f3a 100644 --- a/exegol/manager/ExegolController.py +++ b/exegol/manager/ExegolController.py @@ -1,9 +1,11 @@ +import http import logging try: import docker - import requests import git + import requests + import urllib3 from exegol.utils.ExeLog import logger, ExeLog, console from exegol.utils.DockerUtils import DockerUtils @@ -83,5 +85,5 @@ def main(): logger.critical(f"A critical error occurred while running this git command: {' '.join(git_error.command)}") except Exception: print_exception_banner() - console.print_exception(show_locals=True, suppress=[docker, requests, git]) + console.print_exception(show_locals=True, suppress=[docker, requests, git, urllib3, http]) exit(1) From 7a2fded072309f6d5f6093d87a57e987214d7dfc Mon Sep 17 00:00:00 2001 From: Dramelac Date: Mon, 9 Sep 2024 11:36:12 +0200 Subject: [PATCH 08/15] Handle another docker timeout error Signed-off-by: Dramelac --- exegol/utils/DockerUtils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 6f446d2c..6b932ab1 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -4,6 +4,7 @@ from typing import List, Optional, Union, cast import docker +import requests.exceptions from docker import DockerClient from docker.errors import APIError, DockerException, NotFound, ImageNotFound from docker.models.images import Image @@ -58,6 +59,8 @@ def __init__(self): logger.critical( "Unable to connect to docker (from env config). Is docker operational and accessible? on your machine? " "Exiting.") + except (ReadTimeout, requests.exceptions.ConnectionError): + logger.critical("Docker daemon seems busy, Exegol receives timeout response. Try again later.") self.__images: Optional[List[ExegolImage]] = None self.__containers: Optional[List[ExegolContainer]] = None @@ -556,9 +559,9 @@ def __remove_image(self, image_name: str) -> bool: try: self.__client.images.remove(image_name, force=False, noprune=False) return True - except ReadTimeout: + except (ReadTimeout, requests.exceptions.ConnectionError): logger.warning("The deletion of the image has timeout. Docker is still processing the removal, please wait.") - max_retry = 5 + max_retry = 10 wait_time = 5 for i in range(5): try: @@ -569,7 +572,7 @@ def __remove_image(self, image_name: str) -> bool: return True else: logger.debug(f"Unexpected error after timeout: {err}") - except ReadTimeout: + except (ReadTimeout, requests.exceptions.ConnectionError): wait_time = wait_time + wait_time * i logger.info(f"Docker timeout again ({i + 1}/{max_retry}). Next retry in {wait_time} seconds...") sleep(wait_time) # Wait x seconds before retry From bf1d6f5d6379069913fe28812f0e66cdb0828795 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 10 Sep 2024 00:36:32 +0200 Subject: [PATCH 09/15] Add config to disable exegol resources Signed-off-by: Dramelac --- exegol/config/UserConfig.py | 11 ++++++++--- exegol/manager/UpdateManager.py | 7 +++++-- exegol/model/ContainerConfig.py | 3 ++- exegol/model/ExegolModules.py | 6 +++--- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/exegol/config/UserConfig.py b/exegol/config/UserConfig.py index d3a877aa..af38ee0e 100644 --- a/exegol/config/UserConfig.py +++ b/exegol/config/UserConfig.py @@ -24,6 +24,7 @@ def __init__(self): self.auto_remove_images: bool = True self.auto_update_workspace_fs: bool = False self.default_start_shell: str = "zsh" + self.enable_exegol_resources: bool = True self.shell_logging_method: str = "asciinema" self.shell_logging_compress: bool = True self.desktop_default_enable: bool = False @@ -61,6 +62,9 @@ def _build_file_content(self): # Default shell command to start default_start_shell: {self.default_start_shell} + # Enable Exegol resources + enable_exegol_resources: {self.enable_exegol_resources} + # Change the configuration of the shell logging functionality shell_logging: #Choice of the method used to record the sessions (script or asciinema) @@ -82,8 +86,6 @@ def _build_file_content(self): localhost_by_default: {self.desktop_default_localhost} """ - # TODO handle default image selection - # TODO handle default start container return config @staticmethod @@ -115,6 +117,7 @@ def _process_data(self): self.auto_remove_images = self._load_config_bool(config_data, 'auto_remove_image', self.auto_remove_images) self.auto_update_workspace_fs = self._load_config_bool(config_data, 'auto_update_workspace_fs', self.auto_update_workspace_fs) self.default_start_shell = self._load_config_str(config_data, 'default_start_shell', self.default_start_shell, choices=self.start_shell_options) + self.enable_exegol_resources = self._load_config_bool(config_data, 'enable_exegol_resources', self.enable_exegol_resources) # Shell_logging section shell_logging_data = config_data.get("shell_logging", {}) @@ -132,7 +135,9 @@ def get_configs(self) -> List[str]: configs = [ f"User config file: [magenta]{self._file_path}[/magenta]", f"Private workspace: [magenta]{self.private_volume_path}[/magenta]", - f"Exegol resources: [magenta]{self.exegol_resources_path}[/magenta]", + "Exegol resources: " + (f"[magenta]{self.exegol_resources_path}[/magenta]" + if self.enable_exegol_resources else + boolFormatter(self.enable_exegol_resources)), f"My resources: [magenta]{self.my_resources_path}[/magenta]", f"Auto-check updates: {boolFormatter(self.auto_check_updates)}", f"Auto-remove images: {boolFormatter(self.auto_remove_images)}", diff --git a/exegol/manager/UpdateManager.py b/exegol/manager/UpdateManager.py index 55626948..ef7fecde 100644 --- a/exegol/manager/UpdateManager.py +++ b/exegol/manager/UpdateManager.py @@ -1,12 +1,12 @@ import re -from datetime import datetime, timedelta +from pathlib import Path from typing import Optional, Dict, cast, Tuple, Sequence -from pathlib import Path, PurePath from rich.prompt import Prompt from exegol.config.ConstantConfig import ConstantConfig from exegol.config.DataCache import DataCache +from exegol.config.UserConfig import UserConfig from exegol.console.ExegolPrompt import Confirm from exegol.console.TUI import ExegolTUI from exegol.console.cli.ParametersManager import ParametersManager @@ -117,6 +117,9 @@ def updateImageSource(cls) -> bool: @classmethod def updateResources(cls) -> bool: """Update Exegol-resources from git (submodule)""" + if not UserConfig().enable_exegol_resources: + logger.info("Skipping disabled Exegol resources.") + return False try: if not ExegolModules().isExegolResourcesReady() and not Confirm('Do you want to update exegol resources.', default=True): return False diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index c34ba06b..8d16f8db 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -501,7 +501,8 @@ def enableExegolResources(self) -> bool: raise CancelOperation except CancelOperation: # Error during installation, skipping operation - logger.warning("Exegol resources have not been downloaded, the feature cannot be enabled") + if UserConfig().enable_exegol_resources: + logger.warning("Exegol resources have not been downloaded, the feature cannot be enabled yet") return False logger.verbose("Config: Enabling exegol resources volume") self.__exegol_resources = True diff --git a/exegol/model/ExegolModules.py b/exegol/model/ExegolModules.py index 95b6ba7e..794cb414 100644 --- a/exegol/model/ExegolModules.py +++ b/exegol/model/ExegolModules.py @@ -1,14 +1,14 @@ from pathlib import Path from typing import Optional, Union +from exegol.config.ConstantConfig import ConstantConfig +from exegol.config.UserConfig import UserConfig from exegol.console.ExegolPrompt import Confirm from exegol.console.cli.ParametersManager import ParametersManager from exegol.exceptions.ExegolExceptions import CancelOperation -from exegol.config.ConstantConfig import ConstantConfig from exegol.utils.ExeLog import logger from exegol.utils.GitUtils import GitUtils from exegol.utils.MetaSingleton import MetaSingleton -from exegol.config.UserConfig import UserConfig class ExegolModules(metaclass=MetaSingleton): @@ -51,7 +51,7 @@ def getResourcesGit(self, fast_load: bool = False, skip_install: bool = False) - if self.__git_resources is None: self.__git_resources = GitUtils(UserConfig().exegol_resources_path, "resources", "", skip_submodule_update=fast_load) - if not self.__git_resources.isAvailable and not skip_install: + if not self.__git_resources.isAvailable and not skip_install and UserConfig().enable_exegol_resources: self.__init_resources_repo() return self.__git_resources From 84074b19aa08e79e342e8dc552d1dbd221693a81 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Mon, 14 Oct 2024 20:05:57 +0200 Subject: [PATCH 10/15] Fix startup log timeout --- exegol/model/ExegolContainer.py | 2 +- exegol/utils/ContainerLogStream.py | 14 +++++--------- exegol/utils/DockerUtils.py | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index bf3b108a..a3050d3e 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -118,7 +118,7 @@ def __start_container(self): :return: """ with console.status(f"Waiting to start {self.name}", spinner_style="blue") as progress: - start_date = datetime.utcnow() + start_date = datetime.now() try: self.__container.start() except APIError as e: diff --git a/exegol/utils/ContainerLogStream.py b/exegol/utils/ContainerLogStream.py index 51b278f8..f5dc6b30 100644 --- a/exegol/utils/ContainerLogStream.py +++ b/exegol/utils/ContainerLogStream.py @@ -1,12 +1,8 @@ -import asyncio -import concurrent.futures -import threading import time from datetime import datetime, timedelta -from typing import Union, List, Any, Optional +from typing import Optional from docker.models.containers import Container -from docker.types import CancellableStream from exegol.utils.ExeLog import logger @@ -17,7 +13,7 @@ def __init__(self, container: Container, start_date: Optional[datetime] = None, # Container to extract logs from self.__container = container # Fetch more logs from this datetime - self.__start_date: datetime = datetime.utcnow() if start_date is None else start_date + self.__start_date: datetime = datetime.now() if start_date is None else start_date self.__since_date = self.__start_date self.__until_date: Optional[datetime] = None # The data stream is returned from the docker SDK. It can contain multiple line at the same. @@ -30,7 +26,7 @@ def __init__(self, container: Container, start_date: Optional[datetime] = None, # Hint message flag self.__tips_sent = False - self.__tips_timedelta = self.__start_date + timedelta(seconds=15) + self.__tips_timedelta = self.__start_date + timedelta(seconds=30) def __iter__(self): return self @@ -38,7 +34,7 @@ def __iter__(self): def __next__(self): """Get the next line of the stream""" if self.__until_date is None: - self.__until_date = datetime.utcnow() + self.__until_date = datetime.now() while True: # The data stream is fetch from the docker SDK once empty. if self.__data_stream is None: @@ -69,4 +65,4 @@ def __next__(self): self.__data_stream = None self.__since_date = self.__until_date time.sleep(0.5) # Wait for more logs - self.__until_date = datetime.utcnow() + self.__until_date = datetime.now() diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 6b932ab1..5cd512c2 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -603,7 +603,7 @@ def buildImage(self, tag: str, build_profile: Optional[str] = None, build_docker tag=f"{ConstantConfig.IMAGE_NAME}:{tag}", buildargs={"TAG": f"{build_profile}", "VERSION": "local", - "BUILD_DATE": datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')}, + "BUILD_DATE": datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ')}, platform="linux/" + ParametersManager().arch, rm=True, forcerm=True, From e7d38e85dbf4d17eac3ac634bac3c2141baab6e8 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 22 Oct 2024 18:20:10 +0200 Subject: [PATCH 11/15] Fix raw logger bytes --- exegol/model/ExegolContainer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index a3050d3e..d834d527 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -193,7 +193,9 @@ def exec(self, command: Union[str, Sequence[str]], as_daemon: bool = True, quiet # stream[0] : exit code # stream[1] : text stream for log in stream[1]: - logger.raw(log.decode("utf-8")) + if type(log) is bytes: + log = log.decode("utf-8") + logger.raw(log) if not quiet: logger.success("End of the command") except KeyboardInterrupt: From ba889aa35212ddc9ad4361d45cc57f154240d8ac Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 22 Oct 2024 18:23:21 +0200 Subject: [PATCH 12/15] Use original UID/GID when using sudo --- exegol/model/ContainerConfig.py | 4 ++-- exegol/utils/DataFileUtils.py | 8 +++++++- exegol/utils/FsUtils.py | 34 ++++++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 8d16f8db..03bdd87a 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -24,7 +24,7 @@ from exegol.model.ExegolModules import ExegolModules from exegol.utils import FsUtils from exegol.utils.ExeLog import logger, ExeLog -from exegol.utils.FsUtils import check_sysctl_value +from exegol.utils.FsUtils import check_sysctl_value, mkdir from exegol.utils.GuiUtils import GuiUtils if EnvInfo.is_windows_shell or EnvInfo.is_mac_shell: @@ -1039,7 +1039,7 @@ def addVolume(self, else: # If the directory is created by exegol, bypass user preference and enable shared perms (if available) execute_update_fs = force_sticky_group or enable_sticky_group - path.mkdir(parents=True, exist_ok=True) + mkdir(path) except PermissionError: logger.error("Unable to create the volume folder on the filesystem locally.") logger.critical(f"Insufficient permissions to create the folder: {host_path}") diff --git a/exegol/utils/DataFileUtils.py b/exegol/utils/DataFileUtils.py index c5bd4031..7ab6a46a 100644 --- a/exegol/utils/DataFileUtils.py +++ b/exegol/utils/DataFileUtils.py @@ -1,4 +1,6 @@ import json +import os +import sys from json import JSONEncoder, JSONDecodeError from pathlib import Path from typing import Union, Dict, cast, Optional, Set, Any @@ -8,6 +10,7 @@ from exegol.config.ConstantConfig import ConstantConfig from exegol.utils.ExeLog import logger +from exegol.utils.FsUtils import mkdir, get_user_id class DataFileUtils: @@ -47,7 +50,7 @@ def __load_file(self): """ if not self._file_path.parent.is_dir(): logger.verbose(f"Creating config folder: {self._file_path.parent}") - self._file_path.parent.mkdir(parents=True, exist_ok=True) + mkdir(self._file_path.parent) if not self._file_path.is_file(): logger.verbose(f"Creating default file: {self._file_path}") self._create_config_file() @@ -72,6 +75,9 @@ def _create_config_file(self): try: with open(self._file_path, 'w') as file: file.write(self._build_file_content()) + if sys.platform == "linux" and os.getuid() == 0: + user_uid, user_gid = get_user_id() + os.chown(self._file_path, user_uid, user_gid) except PermissionError as e: logger.critical(f"Unable to open the file '{self._file_path}' ({e}). Please fix your file permissions or run exegol with the correct rights.") diff --git a/exegol/utils/FsUtils.py b/exegol/utils/FsUtils.py index 5d81f58e..1e781a70 100644 --- a/exegol/utils/FsUtils.py +++ b/exegol/utils/FsUtils.py @@ -1,9 +1,11 @@ import logging +import os import re import stat import subprocess +import sys from pathlib import Path, PurePath -from typing import Optional +from typing import Optional, Tuple from exegol.config.EnvInfo import EnvInfo from exegol.utils.ExeLog import logger @@ -104,3 +106,33 @@ def check_sysctl_value(sysctl: str, compare_to: str) -> bool: except PermissionError: logger.debug(f"Unable to read sysctl {sysctl} permission!") return False + + +def get_user_id() -> Tuple[int, int]: + user_uid_raw = os.getenv("SUDO_UID") + if user_uid_raw is None: + user_uid = os.getuid() + else: + user_uid = int(user_uid_raw) + user_gid_raw = os.getenv("SUDO_GID") + if user_gid_raw is None: + user_gid = os.getgid() + else: + user_gid = int(user_gid_raw) + return user_uid, user_gid + + +def mkdir(path): + try: + path.mkdir(parents=False, exist_ok=False) + if sys.platform == "linux" and os.getuid() == 0: + user_uid, user_gid = get_user_id() + os.chown(path, user_uid, user_gid) + except FileExistsError: + # The directory already exist, this setup can be skipped + pass + except FileNotFoundError: + # Create parent directory first + mkdir(path.parent) + # Then create the targeted directory + mkdir(path) From d014d3d69a9408434e4a0d5585b466f993842af1 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sun, 27 Oct 2024 17:58:04 +0100 Subject: [PATCH 13/15] Handle windows system for get_user_id --- exegol/utils/FsUtils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/exegol/utils/FsUtils.py b/exegol/utils/FsUtils.py index 1e781a70..92f750aa 100644 --- a/exegol/utils/FsUtils.py +++ b/exegol/utils/FsUtils.py @@ -95,6 +95,7 @@ def setGidPermission(root_folder: Path): def check_sysctl_value(sysctl: str, compare_to: str) -> bool: + """Function to find a sysctl configured value and compare it to a desired value.""" sysctl_path = "/proc/sys/" + sysctl.replace('.', '/') try: with open(sysctl_path, 'r') as conf: @@ -109,6 +110,9 @@ def check_sysctl_value(sysctl: str, compare_to: str) -> bool: def get_user_id() -> Tuple[int, int]: + """On linux system, retrieve the original user id when using SUDO.""" + if sys.platform == "win32": + raise SystemError user_uid_raw = os.getenv("SUDO_UID") if user_uid_raw is None: user_uid = os.getuid() @@ -123,6 +127,7 @@ def get_user_id() -> Tuple[int, int]: def mkdir(path): + """Function to recursively create a directory and setting the right user and group id to allow host user access.""" try: path.mkdir(parents=False, exist_ok=False) if sys.platform == "linux" and os.getuid() == 0: From f423a30c68d5c124d504284352c27af018e87293 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sun, 27 Oct 2024 18:10:55 +0100 Subject: [PATCH 14/15] Release 4.3.7 configuration --- exegol-docker-build | 2 +- exegol-resources | 2 +- exegol/config/ConstantConfig.py | 2 +- tests/test_exegol.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exegol-docker-build b/exegol-docker-build index 1e87c584..28676a85 160000 --- a/exegol-docker-build +++ b/exegol-docker-build @@ -1 +1 @@ -Subproject commit 1e87c5843d11174e28b7b81e24f13804ceee663a +Subproject commit 28676a85642a4902e1a7a3a08d2bd231e36043bb diff --git a/exegol-resources b/exegol-resources index 314c14d8..a6e78aad 160000 --- a/exegol-resources +++ b/exegol-resources @@ -1 +1 @@ -Subproject commit 314c14d8e275f6d9111b7f434b3f846444fdbf60 +Subproject commit a6e78aade01083c4f50d3d5f0d55b8584d8cd7f0 diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index 7315741e..0c698243 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -5,7 +5,7 @@ class ConstantConfig: """Constant parameters information""" # Exegol Version - version: str = "4.3.6" + version: str = "4.3.7" # Exegol documentation link documentation: str = "https://exegol.rtfd.io/" diff --git a/tests/test_exegol.py b/tests/test_exegol.py index 15f0d804..af3aa00b 100644 --- a/tests/test_exegol.py +++ b/tests/test_exegol.py @@ -2,4 +2,4 @@ def test_version(): - assert __version__ == '4.3.6' + assert __version__ == '4.3.7' From 32307da45e37c57a5d62de5ab0d5e0d4bd9c8aef Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sun, 27 Oct 2024 18:37:25 +0100 Subject: [PATCH 15/15] Remove test suite following deprecation --- .github/workflows/entrypoint_prerelease.yml | 2 +- .github/workflows/entrypoint_release.yml | 2 +- setup.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/entrypoint_prerelease.yml b/.github/workflows/entrypoint_prerelease.yml index 1391657a..cb00b44b 100644 --- a/.github/workflows/entrypoint_prerelease.yml +++ b/.github/workflows/entrypoint_prerelease.yml @@ -57,7 +57,7 @@ jobs: - name: Install requirements run: python -m pip install --user build virtualenv setuptools - name: Cleaning - run : python setup.py clean test + run : python setup.py clean - name: Build Exegol run: python -m build --sdist --outdir dist/ . - name: Create testing venv diff --git a/.github/workflows/entrypoint_release.yml b/.github/workflows/entrypoint_release.yml index 2c92b2e0..bf2d8cfb 100644 --- a/.github/workflows/entrypoint_release.yml +++ b/.github/workflows/entrypoint_release.yml @@ -29,7 +29,7 @@ jobs: - name: Install requirements run: python -m pip install --user build virtualenv setuptools - name: Run package unitest - run : python setup.py clean test + run : python setup.py clean - name: Build Exegol release run: python -m build --sdist --outdir dist/ . - name: Create testing venv diff --git a/setup.py b/setup.py index 4cb8c89a..317fcf33 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,5 @@ 'Source': 'https://github.com/ThePorgs/Exegol', 'Documentation': 'https://exegol.readthedocs.io/', 'Funding': 'https://patreon.com/nwodtuhs', - }, - test_suite='tests' + } )